diff --git a/runtime/datax/package.xml b/runtime/datax/package.xml index 3c8825af..4c28790d 100644 --- a/runtime/datax/package.xml +++ b/runtime/datax/package.xml @@ -67,27 +67,27 @@ - - postgresqlreader/target/datax/ - - **/*.* - - datax - - + - + + + rdbmsreader/target/datax/ + + **/*.* + + datax + @@ -257,13 +257,13 @@ - - gaussdbreader/target/datax/ - - **/*.* - - datax - + + + + + + + nfsreader/target/datax/ @@ -294,13 +294,13 @@ - - mysqlwriter/target/datax/ - - **/*.* - - datax - + + + + + + + @@ -399,13 +399,13 @@ - - postgresqlwriter/target/datax/ - - **/*.* - - datax - + + + + + + + @@ -581,13 +581,13 @@ - - gaussdbwriter/target/datax/ - - **/*.* - - datax - + + + + + + + diff --git a/runtime/datax/pom.xml b/runtime/datax/pom.xml index de431257..f33a7ecd 100644 --- a/runtime/datax/pom.xml +++ b/runtime/datax/pom.xml @@ -51,13 +51,13 @@ mysqlreader - postgresqlreader + - + rdbmsreader @@ -88,13 +88,13 @@ glusterfsreader s3reader - mysqlwriter + starrockswriter - postgresqlwriter + @@ -137,8 +137,8 @@ nfswriter glusterfswriter s3writer - gaussdbreader - gaussdbwriter + + apireader plugin-rdbms-util diff --git a/runtime/datax/rdbmsreader/pom.xml b/runtime/datax/rdbmsreader/pom.xml new file mode 100644 index 00000000..d7a13b43 --- /dev/null +++ b/runtime/datax/rdbmsreader/pom.xml @@ -0,0 +1,132 @@ + + + + datax-all + com.alibaba.datax + 0.0.1-SNAPSHOT + + 4.0.0 + + rdbmsreader + rdbmsreader + jar + + + + com.alibaba.datax + datax-common + ${datax-project-version} + + + slf4j-log4j12 + org.slf4j + + + + + + + com.dameng + Dm7JdbcDriver17 + 7.6.0.142 + + + + com.sybase + jconn3 + 1.0.0-SNAPSHOT + system + ${basedir}/src/main/libs/jconn3-1.0.0-SNAPSHOT.jar + + + + ppas + ppas + 16 + system + ${basedir}/src/main/libs/edb-jdbc16.jar + + + + com.ibm.db2.jcc + db2jcc + db2jcc4 + + + + com.microsoft.sqlserver + sqljdbc4 + 4.0 + + + + org.postgresql + postgresql + 42.3.3 + + + + mysql + mysql-connector-java + ${mysql.driver.version} + + + + org.opengauss + opengauss-jdbc + 3.0.0 + + + + org.slf4j + slf4j-api + + + + ch.qos.logback + logback-classic + + + + com.alibaba.datax + plugin-rdbms-util + ${datax-project-version} + + + + + + + + + maven-compiler-plugin + + ${jdk-version} + ${jdk-version} + ${project-sourceEncoding} + + + + + maven-assembly-plugin + + + src/main/assembly/package.xml + + datax + + + + dwzip + package + + single + + + + + + + + diff --git a/runtime/datax/rdbmsreader/src/main/resources/plugin.json b/runtime/datax/rdbmsreader/src/main/resources/plugin.json new file mode 100644 index 00000000..dfaac747 --- /dev/null +++ b/runtime/datax/rdbmsreader/src/main/resources/plugin.json @@ -0,0 +1,7 @@ +{ + "name": "rdbmsreader", + "class": "com.alibaba.datax.plugin.reader.rdbmsreader.RdbmsReader", + "description": "useScene: prod. mechanism: Jdbc connection using the database, execute select sql, retrieve data from the ResultSet. warn: The more you know about the database, the less problems you encounter.", + "developer": "alibaba", + "drivers":["dm.jdbc.driver.DmDriver", "com.sybase.jdbc3.jdbc.SybDriver", "com.edb.Driver", "com.ibm.db2.jcc.DB2Driver", "com.microsoft.sqlserver.jdbc.SQLServerDriver", "org.postgresql.Driver", "org.opengauss.Driver", "com.mysql.cj.jdbc.Driver"] +} diff --git a/scripts/db/data-collection-init.sql b/scripts/db/data-collection-init.sql index ee934923..072f3762 100644 --- a/scripts/db/data-collection-init.sql +++ b/scripts/db/data-collection-init.sql @@ -233,7 +233,20 @@ INSERT INTO t_dc_collection_templates ( '{"parameter": {}, "reader": {"api": {"name": "接口地址","description": "API的访问地址。","type": "input", "required": true, "index": 1}, "method": {"name": "请求方式","description": "API的请求方式(默认为GET)。","type": "option", "options": ["GET", "POST"], "required": false, "index": 2}, "body": {"name": "请求体参数","description": "请求体参数,主要针对POST请求。示例:\n {\"query\": \"value\"}","type": "jsonobject", "required": false, "index": 3}, "headers": {"name": "请求头参数","description": "需要设置的请求头参数。示例:\n {\"Authorization\": \"***\"}","type": "jsonobject", "required": false, "index": 4}, "schema": {"name": "数据解析schema","description": "会依据schema解析API返回结果。示例:\n {\"dataPath\": \"返回体内定位到具体数据的路径,形如$.data\", \"fields\": [{\"name\": \"属性名\", \"alias\": \"别名\", \"path\": \"属性的路径\"}]}","type": "jsonobject", "required": true, "index": 5}}, "writer": {}}', TRUE, 'system', - 'system') + 'system'), + ( + '7', + '通用关系型数据库归集模板', + '将关系型数据库中的数据以csv文件的形式归集到DataMate平台上。当前支持postgres、opengauss、sqlserver、mysql、达梦、db2格式、PPAS格式等关系型数据库。', + 'rdbmsreader', + 'rdbmsreader', + 'txtfilewriter', + 'txtfilewriter', + '{"parameter": {}, "reader": {"username": {"name": "用户名","description": "数据库的用户名。","type": "input", "required": true, "index": 2}, "password": {"name": "密码","description": "数据库的密码。","type": "password", "required": true, "index": 3}, "connection": {"name": "数据库连接信息", "description": "数据库连接信息。", "type": "multipleList", "size": 1, "index": 1, "properties": {"jdbcUrl": {"type": "inputList", "name": "数据库连接", "description": "数据库连接url。", "required": true, "index": 1}, "querySql": {"type": "inputList", "name": "查询sql", "description": "输入符合语法的sql查询语句。", "required": true, "index": 2}}}}, "writer": {"header": {"name": "列名","description": "查询结果的列名,最终会体现为csv文件的表头。","type": "selectTag", "required": false}}}', + TRUE, + 'system', + 'system' + ) ON CONFLICT (id) DO UPDATE SET name = EXCLUDED.name, description = EXCLUDED.description,