scala slick mysql 字段过多 tuple问题

原同步服务正常,因需,对方单表新增字段,超过22条

sbt assembly 编译出错

too many elements for tuple: 26, allowed: 22

scala case class 最多只支持22个构造参数

遂找解决办法

https://underscore.io/blog/posts/2016/10/11/twenty-two.html

https://github.com/slick/slick/issues/519#issuecomment-48327043

https://github.com/underscoreio/slickless/issues/16

 

最终应用slickless解决

部分示例

import slick.jdbc.H2Profile.api._
import shapeless.{ HList, ::, HNil, Generic }
import slickless._

case class User(id: Long, email: String)

class Users(tag: Tag) extends Table[User](tag, "users") {
  def id    = column[Long]( "id", O.PrimaryKey, O.AutoInc )
  def email = column[String]("email")

  def * = (id :: email :: HNil).mappedWith(Generic[User])
}

lazy val users = TableQuery[Users]

 

 

基于 tupled 需要作转换

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
case (BYOFFSET_GET,lastid:Int) => {
  println("sync start by",lastid)
  val words = TableQuery[Words]
  //      val action=ORIGIN_DB.run(Word.filter(_.id > lastid).take(PAGE_SIZE).result)
  val action=ORIGIN_DB.run(words.filter(_.id > lastid).take(PAGE_SIZE).result)
  action.onComplete(data=>{
    println("sync get data",data.get.length)
    if (data.isSuccess){
      val words=data.get.toList.map(a=>{
        Word.tupled(a)
      })
      if (words.length>0){
        Future {
          println(s"Blocking next page 1s start")
          TimeUnit.SECONDS.sleep(1)
          println(s"Blocking next page 1s finished")
          //同步时只考虑insert
          if(is_just_insert){
            self !(BYOFFSET_INSERT,words)
          }else{
            //如果会更新历史数据
            self !(BYOFFSET_INSERT_UPDATE,words)
          }
        }
      }else{
        //拿到最新数据 等待5分钟
        Future {
          println(s"Blocking future 5m start")
          TimeUnit.MINUTES.sleep(5)
          println(s"Blocking future 5m finished")
          self !(BYOFFSET_GET,lastid)
        }
      }
    }else{      Future {<br>        println(s"Blocking table "+tablename+"  future 5m start")<br>        TimeUnit.MINUTES.sleep(5)<br>        println(s"Blocking table "+tablename+"  future 5m finished")<br>        self !(BYOFFSET_GET,lastid)<br>      }<br>       }      })
}
//插入数据
case (BYOFFSET_INSERT,words:List[Word])=>{
  println("insert start",words.length)
  val word = TableQuery[Words]
  word.++=(words.map(a=>{
    Word.unapply(a).get
  }))
  //Word.+=(Word.unapply(words.head).get)
  val insertActions = DBIO.seq(
    word.++=(words.map(a=>{
      Word.unapply(a).get
    }))
  )
  DEST_DB.run(insertActions).onComplete(data=>{
   if (data.isSuccess){
    println("insert data result",data)
    //添加成功后更新last表
     val lastid=words.last.id
     Sync.lastActor !(BYOFFSET_UPSERT_OFFSET,tablename,lastid)
   }else{
      self !(BYOFFSET_INSERT,words)
    }
  })
}

 

基于HLists/Generics 则不必

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
case (BYOFFSET_GET,lastid:Int) => {
    println("table "+tablename+"  sync start by",lastid)
    val users = TableQuery[Users]
    //      val action=ORIGIN_DB.run(User.filter(_.id > lastid).take(PAGE_SIZE).result)
    val action=ORIGIN_DB.run(users.filter(_.id > lastid).take(PAGE_SIZE).result)
    action.onComplete(data=>{
      println("table "+tablename+"  sync get data",data.get.length)
      if (data.isSuccess){
        val users=data.get.toList
        if (users.length>0){
          Future {
            println(s"Blocking table "+tablename+"  next page 1s start")
            TimeUnit.SECONDS.sleep(1)
            println(s"Blocking table "+tablename+"  next page 1s finished")
            //同步时只考虑insert
            if(is_just_insert){
              self !(BYOFFSET_INSERT,users)
            }else{
              //如果会更新历史数据
              self !(BYOFFSET_INSERT_UPDATE,users)
            }
          }
        }else{
          Future {
            println(s"Blocking table "+tablename+"  future 5m start")
            TimeUnit.MINUTES.sleep(5)
            println(s"Blocking table "+tablename+"  future 5m finished")
            self !(BYOFFSET_GET,lastid)
          }
        }
      }else{
          Future {
            println(s"Blocking table "+tablename+"  future 5m start")
            TimeUnit.MINUTES.sleep(5)
            println(s"Blocking table "+tablename+"  future 5m finished")
            self !(BYOFFSET_GET,lastid)
          }
      }
    })
  }
  //  插入数据
  case (BYOFFSET_INSERT,users:List[User])=>{
    println("table "+tablename+"  insert start",users.length)
    val user = TableQuery[Users]
    //User.+=(User.unapply(users.head).get)
    val insertActions = DBIO.seq(
        user++=users
    )
    DEST_DB.run(insertActions).onComplete(data=>{
     if (data.isSuccess){
      println("table "+tablename+"  insert data result",data)
      //添加成功后更新last表
       val lastid=users.last.id
       Sync.lastActor !(BYOFFSET_UPSERT_OFFSET,tablename,lastid)
     }else{
        self !(BYOFFSET_INSERT,users)
      }           
    })
  }<em id="__mceDel" style="background-color: rgba(255, 255, 255, 1); font-family: "PingFang SC", "Helvetica Neue", Helvetica, Arial, sans-serif; font-size: 14px">  </em>

scala slick 异构表同步服务

https://github.com/cclient/ScalaMysqlSync

posted @   cclient  阅读(1181)  评论(0编辑  收藏  举报
编辑推荐:
· Linux glibc自带哈希表的用例及性能测试
· 深入理解 Mybatis 分库分表执行原理
· 如何打造一个高并发系统?
· .NET Core GC压缩(compact_phase)底层原理浅谈
· 现代计算机视觉入门之:什么是图片特征编码
阅读排行:
· 手把手教你在本地部署DeepSeek R1,搭建web-ui ,建议收藏!
· Spring AI + Ollama 实现 deepseek-r1 的API服务和调用
· 数据库服务器 SQL Server 版本升级公告
· C#/.NET/.NET Core技术前沿周刊 | 第 23 期(2025年1.20-1.26)
· 程序员常用高效实用工具推荐,办公效率提升利器!
点击右上角即可分享
微信分享提示