package org.jeecg; import java.io.FileInputStream; import java.io.FileOutputStream; import java.sql.*; import java.text.SimpleDateFormat; import java.util.*; import java.util.Date; /** * Oracle数据库同步工具类 * 功能:根据指定的字段类型(日期或ID)分批同步数据,并记录同步位置 */ public class OracleSync { // 数据库连接信息 private static final String SOURCE_URL = "jdbc:oracle:thin:@//127.0.0.1:1521/orcl"; private static final String TARGET_URL = "jdbc:oracle:thin:@//127.0.0.1:1521/orcl"; private static final String USERNAME = "idctest"; private static final String PASSWORD = "12345678"; private static final String TARGET_USER = "REB"; private static final String TARGET_PASSWORD = "REB"; // 分批同步配置 private static final int BATCH_SIZE_ID = 10000; // ID每次同步数量 private static final int BATCH_SIZE_DAYS = 30; // 日期每次同步天数 // 同步状态记录文件路径 private static final String SYNC_STATUS_FILE = "sync_status.properties"; /** * 表配置类,用于存储表名、依据字段名和字段类型 */ static class TableConfig { String tableName; // 表名 String fieldName; // 依据字段名 FieldType fieldType; // 字段类型(日期或ID) public TableConfig(String tableName, String fieldName, FieldType fieldType) { this.tableName = tableName; this.fieldName = fieldName; this.fieldType = fieldType; } } /** * 字段类型枚举 */ enum FieldType { DATE, // 日期类型字段 ID // ID类型字段 } /** * 日期范围类,用于存储最小和最大日期 */ static class DateRange { Date minDate; // 最小日期 Date maxDate; // 最大日期 public DateRange(Date minDate, Date maxDate) { this.minDate = minDate; this.maxDate = maxDate; } } /** * ID范围类,用于存储最小和最大ID */ static class IdRange { long minId; // 最小ID long maxId; // 最大ID public IdRange(long minId, long maxId) { this.minId = minId; this.maxId = maxId; } } /** * 主方法,程序入口 * @param args 命令行参数 */ public static void main(String[] args) { // 指定要同步的表名和依据字段 List tableConfigs = Arrays.asList( new TableConfig("TABLE1", "CREATE_DATE", FieldType.DATE), new TableConfig("TABLE2", "ID", FieldType.ID) ); // 加载上次同步状态 Properties syncStatus = loadSyncStatus(); try { // 1. 验证指定的表是否存在 List validTables = validateTables(tableConfigs); if (validTables.isEmpty()) { System.out.println("没有有效的表需要同步"); return; } // 2. 在目标库创建表结构 createTargetTables(validTables); // 3. 同步数据(根据字段类型分批) syncDataByFieldType(validTables, syncStatus); System.out.println("所有指定表同步完成"); } catch (SQLException e) { System.err.println("数据库操作错误: " + e.getMessage()); e.printStackTrace(); } catch (Exception e) { System.err.println("系统错误: " + e.getMessage()); e.printStackTrace(); } finally { // 保存同步状态 saveSyncStatus(syncStatus); } } /** * 验证表是否存在且包含指定字段 * @param tableConfigs 表配置列表 * @return 有效的表配置列表 * @throws SQLException 数据库异常 */ private static List validateTables(List tableConfigs) throws SQLException { List validTables = new ArrayList<>(); try (Connection sourceConn = DriverManager.getConnection(SOURCE_URL, USERNAME, PASSWORD)) { for (TableConfig config : tableConfigs) { if (isTableExists(sourceConn, USERNAME, config.tableName)) { if (isColumnExists(sourceConn, USERNAME, config.tableName, config.fieldName)) { validTables.add(config); System.out.println("表 " + config.tableName + " 存在,将根据字段 " + config.fieldName + " 分批同步"); } else { System.out.println("警告: 表 " + config.tableName + " 中不存在字段 " + config.fieldName + ",已跳过"); } } else { System.out.println("警告: 表 " + config.tableName + " 在源数据库中不存在,已跳过"); } } } return validTables; } /** * 在目标库创建表结构 * @param tableConfigs 表配置列表 * @throws SQLException 数据库异常 */ private static void createTargetTables(List tableConfigs) throws SQLException { try (Connection sourceConn = DriverManager.getConnection(SOURCE_URL, USERNAME, PASSWORD); Connection targetConn = DriverManager.getConnection(TARGET_URL, TARGET_USER, TARGET_PASSWORD)) { for (TableConfig config : tableConfigs) { try { // 检查表是否已存在(区分大小写) if (!isTableExists(targetConn, TARGET_USER, config.tableName)) { // 获取源表结构 String createSql = getCreateTableSql(sourceConn, config.tableName); // 在目标库创建表 try (Statement stmt = targetConn.createStatement()) { stmt.execute(createSql); System.out.println("表 " + config.tableName + " 创建成功"); } catch (Exception e) { e.printStackTrace(); } } else { System.out.println("表 " + config.tableName + " 已存在,跳过创建"); } } catch (SQLException e) { System.err.println("处理表 " + config.tableName + " 时出错: " + e.getMessage()); throw e; } } } } /** * 根据字段类型同步数据 * @param tableConfigs 表配置列表 * @param syncStatus 同步状态 * @throws SQLException 数据库异常 */ private static void syncDataByFieldType(List tableConfigs, Properties syncStatus) throws SQLException { try (Connection sourceConn = DriverManager.getConnection(SOURCE_URL, USERNAME, PASSWORD); Connection targetConn = DriverManager.getConnection(TARGET_URL, TARGET_USER, TARGET_PASSWORD)) { // 设置目标连接为批量提交模式 targetConn.setAutoCommit(false); for (TableConfig config : tableConfigs) { System.out.println("开始同步表: " + config.tableName + " (依据字段: " + config.fieldName + ")"); long startTime = System.currentTimeMillis(); try { if (config.fieldType == FieldType.DATE) { syncByDateRange(sourceConn, targetConn, config, syncStatus); } else if (config.fieldType == FieldType.ID) { syncByIdRange(sourceConn, targetConn, config, syncStatus); } } catch (SQLException e) { System.err.println("同步表 " + config.tableName + " 时出错: " + e.getMessage()); targetConn.rollback(); throw e; } long endTime = System.currentTimeMillis(); System.out.println("表 " + config.tableName + " 同步耗时: " + (endTime - startTime) + " 毫秒"); } } } /** * 按日期范围同步数据 * @param sourceConn 源数据库连接 * @param targetConn 目标数据库连接 * @param config 表配置 * @param syncStatus 同步状态 * @throws SQLException 数据库异常 */ private static void syncByDateRange(Connection sourceConn, Connection targetConn, TableConfig config, Properties syncStatus) throws SQLException { // 获取最小和最大日期 DateRange dateRange = getDateRange(sourceConn, config); System.out.println("日期范围: " + dateRange.minDate + " 至 " + dateRange.maxDate); SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd"); // 获取上次同步的位置 Date lastSyncedDate = getLastSyncedDate(config.tableName, syncStatus); Date currentStart = (lastSyncedDate != null && lastSyncedDate.after(dateRange.minDate)) ? lastSyncedDate : dateRange.minDate; while (currentStart.before(dateRange.maxDate)) { Date currentEnd = addDays(currentStart, BATCH_SIZE_DAYS); if (currentEnd.after(dateRange.maxDate)) { currentEnd = dateRange.maxDate; } String whereClause = config.fieldName + " BETWEEN TO_DATE('" + sdf.format(currentStart) + "', 'YYYY-MM-DD') AND TO_DATE('" + sdf.format(currentEnd) + "', 'YYYY-MM-DD')"; System.out.println("同步日期范围: " + sdf.format(currentStart) + " 至 " + sdf.format(currentEnd)); int rowsSynced = syncDataBatch(sourceConn, targetConn, config.tableName, whereClause); System.out.println("已同步 " + rowsSynced + " 行数据"); // 记录同步位置 syncStatus.setProperty(config.tableName + ".lastSyncedDate", sdf.format(currentEnd)); saveSyncStatus(syncStatus); // 实时保存状态 currentStart = currentEnd; } } /** * 按ID范围同步数据 * @param sourceConn 源数据库连接 * @param targetConn 目标数据库连接 * @param config 表配置 * @param syncStatus 同步状态 * @throws SQLException 数据库异常 */ private static void syncByIdRange(Connection sourceConn, Connection targetConn, TableConfig config, Properties syncStatus) throws SQLException { // 获取最小和最大ID IdRange idRange = getIdRange(sourceConn, config); System.out.println("ID范围: " + idRange.minId + " 至 " + idRange.maxId); // 获取上次同步的位置 long lastSyncedId = getLastSyncedId(config.tableName, syncStatus); long currentStart = (lastSyncedId > 0 && lastSyncedId > idRange.minId) ? lastSyncedId : idRange.minId; while (currentStart <= idRange.maxId) { long currentEnd = currentStart + BATCH_SIZE_ID - 1; if (currentEnd > idRange.maxId) { currentEnd = idRange.maxId; } String whereClause = config.fieldName + " BETWEEN " + currentStart + " AND " + currentEnd; System.out.println("同步ID范围: " + currentStart + " 至 " + currentEnd); int rowsSynced = syncDataBatch(sourceConn, targetConn, config.tableName, whereClause); System.out.println("已同步 " + rowsSynced + " 行数据"); // 记录同步位置 syncStatus.setProperty(config.tableName + ".lastSyncedId", String.valueOf(currentEnd)); saveSyncStatus(syncStatus); // 实时保存状态 currentStart = currentEnd + 1; } } /** * 获取表的日期范围 * @param conn 数据库连接 * @param config 表配置 * @return 日期范围对象 * @throws SQLException 数据库异常 */ private static DateRange getDateRange(Connection conn, TableConfig config) throws SQLException { String sql = "SELECT MIN(" + config.fieldName + ") as min_date, MAX(" + config.fieldName + ") as max_date FROM " + USERNAME + "." + config.tableName; try (Statement stmt = conn.createStatement(); ResultSet rs = stmt.executeQuery(sql)) { if (rs.next()) { return new DateRange(rs.getDate("min_date"), rs.getDate("max_date")); } } throw new SQLException("无法获取日期范围"); } /** * 获取表的ID范围 * @param conn 数据库连接 * @param config 表配置 * @return ID范围对象 * @throws SQLException 数据库异常 */ private static IdRange getIdRange(Connection conn, TableConfig config) throws SQLException { String sql = "SELECT MIN(" + config.fieldName + ") as min_id, MAX(" + config.fieldName + ") as max_id FROM " + USERNAME + "." + config.tableName; try (Statement stmt = conn.createStatement(); ResultSet rs = stmt.executeQuery(sql)) { if (rs.next()) { return new IdRange(rs.getLong("min_id"), rs.getLong("max_id")); } } throw new SQLException("无法获取ID范围"); } /** * 同步一批数据 * @param sourceConn 源数据库连接 * @param targetConn 目标数据库连接 * @param tableName 表名 * @param whereClause 条件子句 * @return 同步的行数 * @throws SQLException 数据库异常 */ private static int syncDataBatch(Connection sourceConn, Connection targetConn, String tableName, String whereClause) throws SQLException { int totalRows = 0; // 从源表读取数据(使用带引号的表名保持大小写) String selectSql = "SELECT * FROM \"" + USERNAME.toUpperCase() + "\".\"" + tableName + "\" WHERE " + whereClause; try (Statement sourceStmt = sourceConn.createStatement( ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY); ResultSet rs = sourceStmt.executeQuery(selectSql)) { // 设置获取大小为1000,优化大表读取 sourceStmt.setFetchSize(1000); // 获取列信息 ResultSetMetaData metaData = rs.getMetaData(); int columnCount = metaData.getColumnCount(); // 准备插入语句(使用带引号的表名保持大小写) StringBuilder insertSql = new StringBuilder("INSERT INTO \"") .append(TARGET_USER.toUpperCase()).append("\".\"").append(tableName).append("\" VALUES ("); for (int i = 1; i <= columnCount; i++) { if (i > 1) insertSql.append(", "); insertSql.append("?"); } insertSql.append(")"); // 批量插入 try (PreparedStatement pstmt = targetConn.prepareStatement(insertSql.toString())) { int batchSize = 0; while (rs.next()) { for (int i = 1; i <= columnCount; i++) { pstmt.setObject(i, rs.getObject(i)); } pstmt.addBatch(); batchSize++; totalRows++; if (batchSize % 1000 == 0) { pstmt.executeBatch(); targetConn.commit(); batchSize = 0; } } if (batchSize > 0) { pstmt.executeBatch(); targetConn.commit(); } } } return totalRows; } /** * 检查表是否存在 * @param conn 数据库连接 * @param schema 模式名 * @param tableName 表名 * @return 表是否存在 * @throws SQLException 数据库异常 */ private static boolean isTableExists(Connection conn, String schema, String tableName) throws SQLException { String sql = "SELECT COUNT(*) FROM all_tables WHERE owner = ? AND (table_name = ? OR table_name = ?)"; try (PreparedStatement pstmt = conn.prepareStatement(sql)) { pstmt.setString(1, schema.toUpperCase()); pstmt.setString(2, tableName.toUpperCase()); pstmt.setString(3, tableName); try (ResultSet rs = pstmt.executeQuery()) { return rs.next() && rs.getInt(1) > 0; } } } /** * 检查列是否存在 * @param conn 数据库连接 * @param schema 模式名 * @param tableName 表名 * @param columnName 列名 * @return 列是否存在 * @throws SQLException 数据库异常 */ private static boolean isColumnExists(Connection conn, String schema, String tableName, String columnName) throws SQLException { String sql = "SELECT COUNT(*) FROM all_tab_columns WHERE owner = ? AND table_name = ? AND column_name = ?"; try (PreparedStatement pstmt = conn.prepareStatement(sql)) { pstmt.setString(1, schema.toUpperCase()); pstmt.setString(2, tableName.toUpperCase()); pstmt.setString(3, columnName.toUpperCase()); try (ResultSet rs = pstmt.executeQuery()) { return rs.next() && rs.getInt(1) > 0; } } } /** * 获取创建表的SQL语句 * @param conn 数据库连接 * @param table 表名 * @return 创建表的SQL语句 * @throws SQLException 数据库异常 */ private static String getCreateTableSql(Connection conn, String table) throws SQLException { if (!isTableExists(conn, USERNAME, table)) { throw new SQLException("表 " + table + " 在源数据库中不存在"); } return "CREATE TABLE \"" + table + "\" AS SELECT * FROM \"" + USERNAME.toUpperCase() + "\".\"" + table + "\" WHERE 1=0"; } /** * 加载同步状态 * @return 同步状态Properties对象 */ private static Properties loadSyncStatus() { Properties props = new Properties(); try { props.load(new FileInputStream(SYNC_STATUS_FILE)); } catch (Exception e) { // 文件不存在或其他错误,返回空Properties } return props; } /** * 保存同步状态 * @param syncStatus 同步状态Properties对象 */ private static void saveSyncStatus(Properties syncStatus) { try { syncStatus.store(new FileOutputStream(SYNC_STATUS_FILE), "Oracle Sync Status - " + new Date()); } catch (Exception e) { System.err.println("无法保存同步状态: " + e.getMessage()); } } /** * 获取上次同步的日期 * @param tableName 表名 * @param syncStatus 同步状态 * @return 上次同步的日期,如果没有则返回null */ private static Date getLastSyncedDate(String tableName, Properties syncStatus) { try { String dateStr = syncStatus.getProperty(tableName + ".lastSyncedDate"); if (dateStr != null) { return new SimpleDateFormat("yyyy-MM-dd").parse(dateStr); } } catch (Exception e) { System.err.println("解析上次同步日期失败: " + e.getMessage()); } return null; } /** * 获取上次同步的ID * @param tableName 表名 * @param syncStatus 同步状态 * @return 上次同步的ID,如果没有则返回0 */ private static long getLastSyncedId(String tableName, Properties syncStatus) { try { String idStr = syncStatus.getProperty(tableName + ".lastSyncedId"); if (idStr != null) { return Long.parseLong(idStr); } } catch (Exception e) { System.err.println("解析上次同步ID失败: " + e.getMessage()); } return 0; } /** * 计算指定日期加上指定天数后的日期 * @param date 基准日期 * @param days 要添加的天数 * @return 计算后的日期 */ private static Date addDays(Date date, int days) { long time = date.getTime() + (long)days * 24 * 60 * 60 * 1000; return new Date(time); } }