Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[Improve][Jdbc/CDC] Improve user permissions in table metadata load #6649

Open
wants to merge 2 commits into
base: dev
Choose a base branch
from
Open

[Improve][Jdbc/CDC] Improve user permissions in table metadata load #6649

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
a9a60fd
[Improve][Jdbc/CDC] Improve user permissions in table metadata load
hailin0 Apr 7, 2024
42f7fb1
Merge branch 'dev' into dev-jdbc-improve-load-metadata
hailin0 May 15, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
13 changes: 13 additions & 0 deletions ...l/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/mysql/utils/MySqlUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import org.apache.seatunnel.api.table.type.SeaTunnelRowType;
import org.apache.seatunnel.common.utils.SeaTunnelException;
import org.apache.seatunnel.connectors.seatunnel.cdc.mysql.source.offset.BinlogOffset;
import org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.SQLUtils;

import org.apache.kafka.connect.source.SourceRecord;

Expand Down Expand Up @@ -81,6 +82,18 @@ public static Object[] queryMinMax(JdbcConnection jdbc, TableId tableId, String

public static long queryApproximateRowCnt(JdbcConnection jdbc, TableId tableId)
throws SQLException {
try {
return analyzeRowCnt(jdbc, tableId);
} catch (SQLException e) {
log.warn(
"Failed to get approximate row count for table {}. Will use exact row count instead.",
tableId,
e);
return SQLUtils.countForTable(jdbc.connection(), quote(tableId));
}
}

public static long analyzeRowCnt(JdbcConnection jdbc, TableId tableId) throws SQLException {
// The statement used to get approximate row count which is less
// accurate than COUNT(*), but is more efficient for large table.
final String useDatabaseStatement = String.format("USE %s;", quote(tableId.catalog()));
Expand Down
13 changes: 13 additions & 0 deletions ...src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/oracle/utils/OracleUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import org.apache.seatunnel.common.utils.SeaTunnelException;
import org.apache.seatunnel.connectors.cdc.base.utils.SourceRecordUtils;
import org.apache.seatunnel.connectors.seatunnel.cdc.oracle.source.offset.RedoLogOffset;
import org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.SQLUtils;

import org.apache.kafka.connect.source.SourceRecord;

Expand Down Expand Up @@ -82,6 +83,18 @@ public static Object[] queryMinMax(JdbcConnection jdbc, TableId tableId, String

public static long queryApproximateRowCnt(JdbcConnection jdbc, TableId tableId)
throws SQLException {
try {
return analyzeRowCnt(jdbc, tableId);
} catch (SQLException e) {
log.warn(
"Failed to get approximate row count for table {}. Will use exact row count instead.",
tableId,
e);
return SQLUtils.countForTable(jdbc.connection(), quoteSchemaAndTable(tableId));
}
}

public static long analyzeRowCnt(JdbcConnection jdbc, TableId tableId) throws SQLException {
final String analyzeTable =
String.format(
"analyze table %s compute statistics for table",
Expand Down
13 changes: 13 additions & 0 deletions ...main/java/org/apache/seatunnel/connectors/seatunnel/cdc/postgres/utils/PostgresUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
import org.apache.seatunnel.connectors.cdc.base.utils.SourceRecordUtils;
import org.apache.seatunnel.connectors.seatunnel.cdc.postgres.source.offset.LsnOffset;
import org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.JdbcDialect;
import org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.SQLUtils;
import org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.psql.PostgresDialect;

import org.apache.kafka.connect.source.SourceRecord;
Expand Down Expand Up @@ -83,6 +84,18 @@ public static Object[] queryMinMax(

public static long queryApproximateRowCnt(JdbcConnection jdbc, TableId tableId)
throws SQLException {
try {
return analyzeRowCnt(jdbc, tableId);
} catch (SQLException e) {
log.warn(
"Failed to get approximate row count for table {}. Will use exact row count instead.",
tableId,
e);
return SQLUtils.countForTable(jdbc.connection(), quote(tableId));
}
}

public static long analyzeRowCnt(JdbcConnection jdbc, TableId tableId) throws SQLException {
// The statement used to get approximate row count which is less
// accurate than COUNT(*), but is more efficient for large table.
final String rowCountQuery =
Expand Down
13 changes: 13 additions & 0 deletions .../org/apache/seatunnel/connectors/seatunnel/cdc/sqlserver/source/utils/SqlServerUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import org.apache.seatunnel.connectors.cdc.base.source.offset.Offset;
import org.apache.seatunnel.connectors.cdc.base.utils.SourceRecordUtils;
import org.apache.seatunnel.connectors.seatunnel.cdc.sqlserver.source.source.offset.LsnOffset;
import org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.SQLUtils;

import org.apache.kafka.connect.source.SourceRecord;

Expand Down Expand Up @@ -82,6 +83,18 @@ public static Object[] queryMinMax(JdbcConnection jdbc, TableId tableId, String

public static long queryApproximateRowCnt(JdbcConnection jdbc, TableId tableId)
throws SQLException {
try {
return analyzeRowCnt(jdbc, tableId);
} catch (SQLException e) {
log.warn(
"Failed to get approximate row count for table {}. Will use exact row count instead.",
tableId,
e);
return SQLUtils.countForTable(jdbc.connection(), quote(tableId));
}
}

public static long analyzeRowCnt(JdbcConnection jdbc, TableId tableId) throws SQLException {
// The statement used to get approximate row count which is less
// accurate than COUNT(*), but is more efficient for large table.
final String useDatabaseStatement = String.format("USE %s;", quote(tableId.catalog()));
Expand Down
5 changes: 5 additions & 0 deletions ...a/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/mysql/MysqlDialect.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -209,6 +209,11 @@ public Long approximateRowCntStatement(Connection connection, JdbcSourceTable ta
}
return rs.getLong(5);
}
} catch (SQLException e) {
log.warn(
"Failed to get approximate row count from table status, fallback to count rows",
e);
return SQLUtils.countForTable(connection, tableIdentifier(table.getTablePath()));
}
}

Expand Down
5 changes: 5 additions & 0 deletions ...org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/oracle/OracleDialect.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -218,6 +218,11 @@ public Long approximateRowCntStatement(Connection connection, JdbcSourceTable ta
}
return rs.getLong(1);
}
} catch (SQLException e) {
log.warn(
"Failed to get approximate row count from table status, fallback to count rows",
e);
return SQLUtils.countForTable(connection, tableIdentifier(table.getTablePath()));
}
}
return SQLUtils.countForSubquery(connection, table.getQuery());
Expand Down
5 changes: 5 additions & 0 deletions ...org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/psql/PostgresDialect.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -251,6 +251,11 @@ public Long approximateRowCntStatement(Connection connection, JdbcSourceTable ta
}
return rs.getLong(1);
}
} catch (SQLException e) {
log.warn(
"Failed to get approximate row count from table status, fallback to count rows",
e);
return SQLUtils.countForTable(connection, tableIdentifier(table.getTablePath()));
}
}
return SQLUtils.countForSubquery(connection, table.getQuery());
Expand Down
5 changes: 5 additions & 0 deletions ...ache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/sqlserver/SqlServerDialect.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -201,6 +201,11 @@ public Long approximateRowCntStatement(Connection connection, JdbcSourceTable ta
}
return rs.getLong(1);
}
} catch (SQLException e) {
log.warn(
"Failed to get approximate row count from table status, fallback to count rows",
e);
return SQLUtils.countForTable(connection, tableIdentifier(table.getTablePath()));
}
}
return SQLUtils.countForSubquery(connection, table.getQuery());
Expand Down