[Bug] In combined mode, an error will be reported if the number of buckets is not set #3254
Closed
1 of 2 tasks
Labels
bug
Something isn't working
Search before asking
Paimon version
0.8
Compute Engine
vvr-8.0.5-flink-1.17
maxcompute
Minimal reproduce step
Start a job on the vvp。
mysql_sync_database
--warehouse 'oss://xxx/yyy'
--database 't1'
--mysql_conf hostname=
--mysql_conf port=
--mysql_conf username=
--mysql_conf password=
--mysql_conf database-name=
--mode combined
--including_tables 'aaa|t1'
--catalog_conf 'fs.oss.endpoint='
--catalog_conf 'fs.oss.accessKeyId='
--catalog_conf 'fs.oss.accessKeySecret='
--catalog_conf 'metastore = maxcompute'
--catalog_conf 'maxcompute.endpoint = '
--catalog_conf 'maxcompute.project = '
--catalog_conf 'maxcompute.accessid = '
--catalog_conf 'maxcompute.accesskey = '
What doesn't meet your expectations?
java.lang.IllegalArgumentException: Num bucket is illegal: -1
at org.apache.paimon.utils.Preconditions.checkArgument(Preconditions.java:127)
at org.apache.paimon.table.sink.KeyAndBucketExtractor.bucket(KeyAndBucketExtractor.java:51)
at org.apache.paimon.flink.sink.cdc.CdcRecordKeyAndBucketExtractor.bucket(CdcRecordKeyAndBucketExtractor.java:101)
at org.apache.paimon.flink.sink.cdc.CdcFixedBucketChannelComputerBase.channel(CdcFixedBucketChannelComputerBase.java:52)
at org.apache.paimon.flink.sink.cdc.CdcMultiplexRecordChannelComputer.channel(CdcMultiplexRecordChannelComputer.java:62)
at org.apache.paimon.flink.sink.cdc.CdcMultiplexRecordChannelComputer.channel(CdcMultiplexRecordChannelComputer.java:34)
at org.apache.paimon.flink.sink.FlinkStreamPartitioner.selectChannel(FlinkStreamPartitioner.java:47)
at org.apache.paimon.flink.sink.FlinkStreamPartitioner.selectChannel(FlinkStreamPartitioner.java:31)
at org.apache.flink.runtime.io.network.api.writer.ChannelSelectorRecordWriter.emit(ChannelSelectorRecordWriter.java:55)
at org.apache.flink.streaming.runtime.io.RecordWriterOutput.pushToRecordWriter(RecordWriterOutput.java:125)
at org.apache.flink.streaming.runtime.io.RecordWriterOutput.collect(RecordWriterOutput.java:114)
at org.apache.flink.streaming.runtime.tasks.CopyingBroadcastingOutputCollector.collect(CopyingBroadcastingOutputCollector.java:60)
at org.apache.flink.streaming.api.operators.CountingOutput.collect(CountingOutput.java:65)
at org.apache.flink.streaming.api.operators.ProcessOperator$ContextImpl.output(ProcessOperator.java:103)
at org.apache.paimon.flink.sink.cdc.CdcDynamicTableParsingProcessFunction.lambda$processElement$1(CdcDynamicTableParsingProcessFunction.java:119)
at java.util.Collections$SingletonList.forEach(Collections.java:4824)
at org.apache.paimon.flink.sink.cdc.CdcDynamicTableParsingProcessFunction.processElement(CdcDynamicTableParsingProcessFunction.java:117)
at org.apache.flink.streaming.api.operators.ProcessOperator.processElement(ProcessOperator.java:66)
at org.apache.flink.streaming.runtime.tasks.ChainingOutput.pushToOperator(ChainingOutput.java:94)
at org.apache.flink.streaming.runtime.tasks.ChainingOutput.collect(ChainingOutput.java:75)
at org.apache.flink.streaming.runtime.tasks.ChainingOutput.collect(ChainingOutput.java:39)
at org.apache.flink.streaming.api.operators.TimestampedCollector.collect(TimestampedCollector.java:51)
at java.util.ArrayList.forEach(ArrayList.java:1259)
at org.apache.paimon.flink.action.cdc.mysql.MySqlRecordParser.flatMap(MySqlRecordParser.java:121)
at org.apache.paimon.flink.action.cdc.mysql.MySqlRecordParser.flatMap(MySqlRecordParser.java:72)
at org.apache.flink.streaming.api.operators.StreamFlatMap.processElement(StreamFlatMap.java:47)
at org.apache.flink.streaming.runtime.tasks.ChainingOutput.pushToOperator(ChainingOutput.java:94)
at org.apache.flink.streaming.runtime.tasks.ChainingOutput.collect(ChainingOutput.java:75)
at org.apache.flink.streaming.runtime.tasks.ChainingOutput.collect(ChainingOutput.java:39)
at org.apache.flink.streaming.runtime.tasks.SourceOperatorStreamTask$AsyncDataOutputToOutput.emitRecord(SourceOperatorStreamTask.java:309)
at org.apache.flink.streaming.api.operators.source.SourceOutputWithWatermarks.collect(SourceOutputWithWatermarks.java:110)
at org.apache.flink.streaming.api.operators.source.SourceOutputWithWatermarks.collect(SourceOutputWithWatermarks.java:101)
at com.ververica.cdc.connectors.mysql.source.reader.MySqlRecordEmitter$OutputCollector.collect(MySqlRecordEmitter.java:154)
at com.ververica.cdc.debezium.JsonDebeziumDeserializationSchema.deserialize(JsonDebeziumDeserializationSchema.java:72)
at com.ververica.cdc.connectors.mysql.source.reader.MySqlRecordEmitter.emitElement(MySqlRecordEmitter.java:129)
at com.ververica.cdc.connectors.mysql.source.reader.MySqlRecordEmitter.processElement(MySqlRecordEmitter.java:111)
at com.ververica.cdc.connectors.mysql.source.reader.MySqlRecordEmitter.emitRecord(MySqlRecordEmitter.java:83)
at com.ververica.cdc.connectors.mysql.source.reader.MySqlRecordEmitter.emitRecord(MySqlRecordEmitter.java:55)
at org.apache.flink.connector.base.source.reader.SourceReaderBase.pollNext(SourceReaderBase.java:160)
at org.apache.flink.streaming.api.operators.SourceOperator.pollNext(SourceOperator.java:708)
at org.apache.flink.streaming.api.operators.SourceOperator.emitNext(SourceOperator.java:425)
at org.apache.flink.streaming.runtime.io.StreamTaskSourceInput.emitNext(StreamTaskSourceInput.java:70)
at org.apache.flink.streaming.runtime.io.StreamOneInputProcessor.processInput(StreamOneInputProcessor.java:68)
at org.apache.flink.streaming.runtime.tasks.StreamTask.processInput(StreamTask.java:615)
at org.apache.flink.streaming.runtime.tasks.mailbox.MailboxProcessor.runMailboxLoop(MailboxProcessor.java:231)
at org.apache.flink.streaming.runtime.tasks.StreamTask.runMailboxLoop(StreamTask.java:1070)
at org.apache.flink.streaming.runtime.tasks.StreamTask.invoke(StreamTask.java:1019)
at org.apache.flink.runtime.taskmanager.Task.runWithSystemExitMonitoring(Task.java:959)
at org.apache.flink.runtime.taskmanager.Task.restoreAndInvoke(Task.java:938)
at org.apache.flink.runtime.taskmanager.Task.doRun(Task.java:751)
at org.apache.flink.runtime.taskmanager.Task.run(Task.java:567)
at java.lang.Thread.run(Thread.java:879)
Anything else?
No response
Are you willing to submit a PR?
The text was updated successfully, but these errors were encountered: