Skip to content

feature: added universal relational database collection template #302

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
hefanli merged 1 commit into from
Jan 30, 2026
Merged

feature: added universal relational database collection template #302

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
74 changes: 37 additions & 37 deletions runtime/datax/package.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,27 +67,27 @@
<!-- </includes>-->
<!-- <outputDirectory>datax</outputDirectory>-->
<!-- </fileSet>-->
<fileSet>
<directory>postgresqlreader/target/datax/</directory>
<includes>
<include>**/*.*</include>
</includes>
<outputDirectory>datax</outputDirectory>
</fileSet>
<!-- <fileSet>-->
<!-- <directory>kingbaseesreader/target/datax/</directory>-->
<!-- <directory>postgresqlreader/target/datax/</directory>-->
<!-- <includes>-->
<!-- <include>**/*.*</include>-->
<!-- </includes>-->
<!-- <outputDirectory>datax</outputDirectory>-->
<!-- </fileSet>-->
<!-- <fileSet>-->
<!-- <directory>rdbmsreader/target/datax/</directory>-->
<!-- <directory>kingbaseesreader/target/datax/</directory>-->
<!-- <includes>-->
<!-- <include>**/*.*</include>-->
<!-- </includes>-->
<!-- <outputDirectory>datax</outputDirectory>-->
<!-- </fileSet>-->
<fileSet>
<directory>rdbmsreader/target/datax/</directory>
<includes>
<include>**/*.*</include>
</includes>
<outputDirectory>datax</outputDirectory>
</fileSet>

<!-- <fileSet>-->
<!-- <directory>odpsreader/target/datax/</directory>-->
Expand Down Expand Up @@ -257,13 +257,13 @@
<!-- </includes>-->
<!-- <outputDirectory>datax</outputDirectory>-->
<!-- </fileSet>-->
<fileSet>
<directory>gaussdbreader/target/datax/</directory>
<includes>
<include>**/*.*</include>
</includes>
<outputDirectory>datax</outputDirectory>
</fileSet>
<!-- <fileSet>-->
<!-- <directory>gaussdbreader/target/datax/</directory>-->
<!-- <includes>-->
<!-- <include>**/*.*</include>-->
<!-- </includes>-->
<!-- <outputDirectory>datax</outputDirectory>-->
<!-- </fileSet>-->
<fileSet>
<directory>nfsreader/target/datax/</directory>
<includes>
Expand Down Expand Up @@ -294,13 +294,13 @@
</fileSet>

<!-- writer -->
<fileSet>
<directory>mysqlwriter/target/datax/</directory>
<includes>
<include>**/*.*</include>
</includes>
<outputDirectory>datax</outputDirectory>
</fileSet>
<!-- <fileSet>-->
<!-- <directory>mysqlwriter/target/datax/</directory>-->
<!-- <includes>-->
<!-- <include>**/*.*</include>-->
<!-- </includes>-->
<!-- <outputDirectory>datax</outputDirectory>-->
<!-- </fileSet>-->
<!-- <fileSet>-->
<!-- <directory>tdenginewriter/target/datax/</directory>-->
<!-- <includes>-->
Expand Down Expand Up @@ -399,13 +399,13 @@
<!-- </includes>-->
<!-- <outputDirectory>datax</outputDirectory>-->
<!-- </fileSet>-->
<fileSet>
<directory>postgresqlwriter/target/datax/</directory>
<includes>
<include>**/*.*</include>
</includes>
<outputDirectory>datax</outputDirectory>
</fileSet>
<!-- <fileSet>-->
<!-- <directory>postgresqlwriter/target/datax/</directory>-->
<!-- <includes>-->
<!-- <include>**/*.*</include>-->
<!-- </includes>-->
<!-- <outputDirectory>datax</outputDirectory>-->
<!-- </fileSet>-->
<!-- <fileSet>-->
<!-- <directory>kingbaseeswriter/target/datax/</directory>-->
<!-- <includes>-->
Expand Down Expand Up @@ -581,13 +581,13 @@
<!-- </includes>-->
<!-- <outputDirectory>datax</outputDirectory>-->
<!-- </fileSet>-->
<fileSet>
<directory>gaussdbwriter/target/datax/</directory>
<includes>
<include>**/*.*</include>
</includes>
<outputDirectory>datax</outputDirectory>
</fileSet>
<!-- <fileSet>-->
<!-- <directory>gaussdbwriter/target/datax/</directory>-->
<!-- <includes>-->
<!-- <include>**/*.*</include>-->
<!-- </includes>-->
<!-- <outputDirectory>datax</outputDirectory>-->
<!-- </fileSet>-->
<!-- <fileSet>-->
<!-- <directory>milvuswriter/target/datax/</directory>-->
<!-- <includes>-->
Expand Down
12 changes: 6 additions & 6 deletions runtime/datax/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,13 +51,13 @@
<module>mysqlreader</module>
<!-- <module>drdsreader</module>-->
<!-- <module>sqlserverreader</module>-->
<module>postgresqlreader</module>
<!-- <module>postgresqlreader</module>-->
<!-- <module>kingbaseesreader</module>-->
<!-- <module>oraclereader</module>-->
<!-- <module>cassandrareader</module>-->
<!-- <module>oceanbasev10reader</module>-->
<!-- <module>obhbasereader</module>-->
<!-- <module>rdbmsreader</module>-->
<module>rdbmsreader</module>

<!-- <module>odpsreader</module>-->
<!-- <module>otsreader</module>-->
Expand Down Expand Up @@ -88,13 +88,13 @@
<module>glusterfsreader</module>
<module>s3reader</module>
<!-- writer -->
<module>mysqlwriter</module>
<!-- <module>mysqlwriter</module>-->
<module>starrockswriter</module>
<!-- <module>drdswriter</module>-->
<!-- <module>databendwriter</module>-->
<!-- <module>oraclewriter</module>-->
<!-- <module>sqlserverwriter</module>-->
<module>postgresqlwriter</module>
<!-- <module>postgresqlwriter</module>-->
<!-- <module>kingbaseeswriter</module>-->
<!-- <module>adswriter</module>-->
<!-- <module>oceanbasev10writer</module>-->
Expand Down Expand Up @@ -137,8 +137,8 @@
<module>nfswriter</module>
<module>glusterfswriter</module>
<module>s3writer</module>
<module>gaussdbreader</module>
<module>gaussdbwriter</module>
<!-- <module>gaussdbreader</module>-->
<!-- <module>gaussdbwriter</module>-->
<module>apireader</module>
<!-- common support module -->
<module>plugin-rdbms-util</module>
Expand Down
132 changes: 132 additions & 0 deletions runtime/datax/rdbmsreader/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,132 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>datax-all</artifactId>
<groupId>com.alibaba.datax</groupId>
<version>0.0.1-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>

<artifactId>rdbmsreader</artifactId>
<name>rdbmsreader</name>
<packaging>jar</packaging>

<dependencies>
<dependency>
<groupId>com.alibaba.datax</groupId>
<artifactId>datax-common</artifactId>
<version>${datax-project-version}</version>
<exclusions>
<exclusion>
<artifactId>slf4j-log4j12</artifactId>
<groupId>org.slf4j</groupId>
</exclusion>
</exclusions>
</dependency>

<!-- dm7 dm8 driver -->
<dependency>
<groupId>com.dameng</groupId>
<artifactId>Dm7JdbcDriver17</artifactId>
<version>7.6.0.142</version>
</dependency>
<!-- sybase driver -->
<dependency>
<groupId>com.sybase</groupId>
<artifactId>jconn3</artifactId>
<version>1.0.0-SNAPSHOT</version>
<scope>system</scope>
<systemPath>${basedir}/src/main/libs/jconn3-1.0.0-SNAPSHOT.jar</systemPath>
</dependency>
<!-- ppas driver -->
<dependency>
<groupId>ppas</groupId>
<artifactId>ppas</artifactId>
<version>16</version>
<scope>system</scope>
<systemPath>${basedir}/src/main/libs/edb-jdbc16.jar</systemPath>
</dependency>
<!-- db2 driver -->
<dependency>
<groupId>com.ibm.db2.jcc</groupId>
<artifactId>db2jcc</artifactId>
<version>db2jcc4</version>
</dependency>
<!-- sqlserver driver -->
<dependency>
<groupId>com.microsoft.sqlserver</groupId>
<artifactId>sqljdbc4</artifactId>
<version>4.0</version>
</dependency>
<!-- postgresql driver -->
<dependency>
<groupId>org.postgresql</groupId>
<artifactId>postgresql</artifactId>
<version>42.3.3</version>
</dependency>
<!-- mysql driver -->
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
<version>${mysql.driver.version}</version>
</dependency>
<!-- opengauss driver -->
<dependency>
<groupId>org.opengauss</groupId>
<artifactId>opengauss-jdbc</artifactId>
<version>3.0.0</version>
</dependency>

<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</dependency>

<dependency>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
</dependency>

<dependency>
<groupId>com.alibaba.datax</groupId>
<artifactId>plugin-rdbms-util</artifactId>
<version>${datax-project-version}</version>
</dependency>

</dependencies>

<build>
<plugins>
<!-- compiler plugin -->
<plugin>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
<source>${jdk-version}</source>
<target>${jdk-version}</target>
<encoding>${project-sourceEncoding}</encoding>
</configuration>
</plugin>
<!-- assembly plugin -->
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<configuration>
<descriptors>
<descriptor>src/main/assembly/package.xml</descriptor>
</descriptors>
<finalName>datax</finalName>
</configuration>
<executions>
<execution>
<id>dwzip</id>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>

</project>
7 changes: 7 additions & 0 deletions runtime/datax/rdbmsreader/src/main/resources/plugin.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
{
"name": "rdbmsreader",
"class": "com.alibaba.datax.plugin.reader.rdbmsreader.RdbmsReader",
"description": "useScene: prod. mechanism: Jdbc connection using the database, execute select sql, retrieve data from the ResultSet. warn: The more you know about the database, the less problems you encounter.",
"developer": "alibaba",
"drivers":["dm.jdbc.driver.DmDriver", "com.sybase.jdbc3.jdbc.SybDriver", "com.edb.Driver", "com.ibm.db2.jcc.DB2Driver", "com.microsoft.sqlserver.jdbc.SQLServerDriver", "org.postgresql.Driver", "org.opengauss.Driver", "com.mysql.cj.jdbc.Driver"]
}
15 changes: 14 additions & 1 deletion scripts/db/data-collection-init.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -233,7 +233,20 @@ INSERT INTO t_dc_collection_templates (
'{"parameter": {}, "reader": {"api": {"name": "接口地址","description": "API的访问地址。","type": "input", "required": true, "index": 1}, "method": {"name": "请求方式","description": "API的请求方式（默认为GET）。","type": "option", "options": ["GET", "POST"], "required": false, "index": 2}, "body": {"name": "请求体参数","description": "请求体参数，主要针对POST请求。示例：\n {\"query\": \"value\"}","type": "jsonobject", "required": false, "index": 3}, "headers": {"name": "请求头参数","description": "需要设置的请求头参数。示例：\n {\"Authorization\": \"***\"}","type": "jsonobject", "required": false, "index": 4}, "schema": {"name": "数据解析schema","description": "会依据schema解析API返回结果。示例：\n {\"dataPath\": \"返回体内定位到具体数据的路径，形如$.data\", \"fields\": [{\"name\": \"属性名\", \"alias\": \"别名\", \"path\": \"属性的路径\"}]}","type": "jsonobject", "required": true, "index": 5}}, "writer": {}}',
TRUE,
'system',
'system')
'system'),
(
'7',
'通用关系型数据库归集模板',
'将关系型数据库中的数据以csv文件的形式归集到DataMate平台上。当前支持postgres、opengauss、sqlserver、mysql、达梦、db2格式、PPAS格式等关系型数据库。',
'rdbmsreader',
'rdbmsreader',
'txtfilewriter',
'txtfilewriter',
'{"parameter": {}, "reader": {"username": {"name": "用户名","description": "数据库的用户名。","type": "input", "required": true, "index": 2}, "password": {"name": "密码","description": "数据库的密码。","type": "password", "required": true, "index": 3}, "connection": {"name": "数据库连接信息", "description": "数据库连接信息。", "type": "multipleList", "size": 1, "index": 1, "properties": {"jdbcUrl": {"type": "inputList", "name": "数据库连接", "description": "数据库连接url。", "required": true, "index": 1}, "querySql": {"type": "inputList", "name": "查询sql", "description": "输入符合语法的sql查询语句。", "required": true, "index": 2}}}}, "writer": {"header": {"name": "列名","description": "查询结果的列名，最终会体现为csv文件的表头。","type": "selectTag", "required": false}}}',
TRUE,
'system',
'system'
)
ON CONFLICT (id) DO UPDATE SET
name = EXCLUDED.name,
description = EXCLUDED.description,
Expand Down
Loading