Did you find this summary helpful?
Thank you for your feedback
Description Available ETL job nodes
Common Syntax class ETLJobNode {
private id: Long;
private name: String;
private JobNodeType type;
private List<Long> parentNodeIds;
enum Operator { EQ, NE, GT, LT, GTE, LTE }
enum ConditionOperator { HEAD, AND, OR }
}
enum JobNodeType(JobNodeCategory category) {
SOURCE_CATALOG(JobNodeCategory.SOURCE),
SOURCE_CDB_MYSQL(JobNodeCategory.SOURCE),
SOURCE_CDB_POSTGRESQL(JobNodeCategory.SOURCE),
SOURCE_OBS(JobNodeCategory.SOURCE),
TRANSFORM_DEFINE(JobNodeCategory.TRANSFORM),
TRANSFORM_SELECT(JobNodeCategory.TRANSFORM),
TRANSFORM_JOIN(JobNodeCategory.TRANSFORM),
TRANSFORM_FILTER(JobNodeCategory.TRANSFORM),
TRANSFORM_UNION(JobNodeCategory.TRANSFORM),
TRANSFORM_AGGREGATE(JobNodeCategory.TRANSFORM),
TRANSFORM_RENAME(JobNodeCategory.TRANSFORM),
TRANSFORM_DROP_NULL(JobNodeCategory.TRANSFORM),
TRANSFORM_DROP_DUPLICATION(JobNodeCategory.TRANSFORM),
TRANSFORM_FILL_MISSING_VALUES(JobNodeCategory.TRANSFORM),
TRANSFORM_SPLIT_FIELD(JobNodeCategory.TRANSFORM),
TRANSFORM_SQL(JobNodeCategory.TRANSFORM),
TARGET_CDB_MYSQL(JobNodeCategory.TARGET),
TARGET_CDB_POSTGRESQL(JobNodeCategory.TARGET),
TARGET_OBS(JobNodeCategory.TARGET),
TARGET_CATALOG(JobNodeCategory.TARGET)
}
enum JobNodeCategory {
SOURCE, TRANSFORM, TARGET
}
class Field {
private String name;
private Type type;
private List<Field> properties;
enum Type {
Void, Boolean, Tinyint, Smallint, Int, Bigint, Float, Double, String, Char, Varchar, Date, Datetime, Timestamp, Decimal,
Binary, TimestampWithTimeZone, Array, Map, Struct, Uniontype;
}
}
Field Field name Required Type Restrictions Description id Yes Long ID name Yes String - English letters, starting with an underscore () ) and hyphens (-) are allowed Job name type Yes JobNodeType See syntax Node type parentNodeIds Yes List Parent node ID
Source / DataCatalog Syntax private String database;
private String table;
private String schemaVersion;
private List<Field> fieldList;
Field Field name Required Type Restrictions Description database Yes String Database name table Yes String Table name schemaVersion Yes String Schema version fieldList Yes List Field list
Source / ObjectStorage Syntax private String regionNo;
private String bucketName;
private String prefix;
private String dataType;
private List<Field> fieldList;
Field Field name Required Type Restrictions Description regionNo Yes String Region number bucketName Yes String Bucket name prefix Yes String Prefix dataType Yes String JSON, CSV, PARQUET, NDJSON Data type fieldList Yes List Field list
Source / Cloud DB for MySQL Syntax private String connectionId;
private String table;
private List<Field> fieldList;
Field Field name Required Type Restrictions Description connectionId Yes String Connection ID table Yes String Table name fieldList Yes List Field list
Source / Cloud DB for POSTGRESQL Syntax private String connectionId;
private String table;
private List<Field> fieldList;
Field Field name Required Type Restrictions Description connectionId Yes String Connection ID table Yes String Table name fieldList Yes List Field list
Syntax private List<DefineField> defineFieldList;
Field Field name Required Type Restrictions Description defineFieldList[].name Yes String Existing field name defineFieldList[].newType Yes String Type to change to defineFieldList[].newName Yes String Field name to change to
Syntax private List<Field> fieldList;
Field Field name Required Type Restrictions Description fieldList Yes List Field list
Syntax private String joinType;
private long leftNodeId;
private long rightNodeId;
private List<JoinCondition> joinConditionList;
private String prefix;
Field Field name Required Type Restrictions Description joinType Yes String INNER, LEFT_OUTER, RIGHT_OUTER, FULL_OUTER Join type leftNodeId Yes long Left node ID rightNodeId Yes long Right node ID joinConditionList[].leftField Yes String Left field joinConditionList[].leftField Yes String Right field joinConditionList[].joinOperator Yes Operator Operator joinConditionList[].conditionOperator Yes ConditionOperator Condition prefix No String Field prefixes to be changed on join
Syntax private String filterType;
private List<FilterCondition> filterConditionList;
Field Field name Required Type Restrictions Description filterType Yes String AND, OR Filter type filterCondtionList[].name Yes String Filter condition name filterCondtionList[].operator Yes String Filter condition operator filterCondtionList[].value Yes String Filter condition value
Syntax private String unionType;
Field Field name Required Type Restrictions Description unionType Yes String ALL, DISTINCT Union type
Syntax private List<String> groupByList;
private List<AggregateField> aggregateFieldList;
Field Field name Required Type Restrictions Description groupByList Yes List Field name aggregateFieldList[].name Yes String Aggregation condition name aggregateFieldList[].ResultName Yes String Aggregated field name aggregateFieldList[].function Yes String SUM, AVG, MAX, MIN Aggregation function
Syntax private List<RenameField> renameFieldList;
Field Field name Required Type Restrictions Description renameFieldList[].name Yes String Existing field name renameFieldList[].newName Yes String Field name to change to
Syntax private List<String> dropFieldList;
Field Field name Required Type Restrictions Description dropFieldList Yes List Field name to check for duplicates
Syntax private List<MissingValueField> missingValueFieldList;
Field Field name Required Type Restrictions Description missingValueFieldList[].name Yes String Name of the field whose empty value you want to fill missingValueFieldList[].defaultValue Yes String Value of the field whose empty value you want to fill
Target / Data Catalog Syntax private String database;
private String table;
private String schemaVersion;
private List<Field> fieldList;
private UpdateMode updateType;
private Integer coalesce;
Field Field name Required Type Restrictions Description database Yes String Database name table Yes String Table name schemaVersion Yes String Schema version fieldList Yes List Field list updateType No String OVERWRITE, APPEND, IGNORE Update option coalesce No Int Number of output files
Target / Object Storage Syntax private String regionNo;
private String bucketName;
private String prefix;
private String dataType;
private List<Field> fieldList;
private UpdateMode updateType;
private Integer coalesce;
Field Field name Required Type Restrictions Description regionNo Yes String Region number bucketName Yes String Bucket name prefix Yes String Target path dataType Yes String JSON, CSV, PARQUET Data type fieldList Yes List Field list updateType No String OVERWRITE, APPEND, IGNORE Update option coalesce No Int Number of output files
Target / Cloud DB for MySQL Syntax private String connectionId;
private String table;
private List<Field> fieldList;
private UpdateMode updateType;
Field Field name Required Type Restrictions Description connectionId Yes String Connection ID table Yes String Table name fieldList Yes List Field list updateType No String OVERWRITE, APPEND, IGNORE Update option
Target / Cloud DB for POSTGRESQL Syntax private String connectionId;
private String table;
private List<Field> fieldList;
private UpdateMode updateType;
Field Field name Required Type Restrictions Description connectionId Yes String Connection ID table Yes String Table name fieldList Yes List Field list updateType No String OVERWRITE, APPEND, IGNORE Update option
Was this article helpful?
Thank you for your feedback! Our team will get back to you
