Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
069a8e3
[fix][CGS][engineconn] fix sr task retry causing init_sql loading exc…
v-kkhuang Mar 31, 2026
8f41952
#AI commit# 开发阶段: spark支持第二队列选择
v-kkhuang Apr 10, 2026
6cad4de
#AI commit# 开发阶段: 优化第二队列逻辑
v-kkhuang Apr 13, 2026
3be456a
#AI commit# 开发阶段: 优化 wds.linkis.rm.secondary.yarnqueue.enable默认值
v-kkhuang Apr 13, 2026
47fb4e6
#AI commit# 开发阶段: refactor: translate Chinese logs to English in smar…
v-kkhuang Apr 13, 2026
b9de770
Merge branch 'dev-1.18.2-webank' into dev-1.18.0-secondary-queue
v-kkhuang Apr 13, 2026
d037626
Revert "#AI commit# 开发阶段: refactor: translate Chinese logs to English…
v-kkhuang Apr 13, 2026
821ddf1
#AI commit# 开发阶段: 优化日志英文打印
v-kkhuang Apr 13, 2026
e54d56e
#AI commit# 开发阶段: 优化日志英文打印
v-kkhuang Apr 13, 2026
ebf3ab9
Merge branch 'dev-1.18.2-webank' into dev-1.18.0-secondary-queue
v-kkhuang Apr 14, 2026
05fbdc2
#AI commit# 开发阶段: 优化spark备用队列执行逻辑
v-kkhuang Apr 14, 2026
08dad25
#AI commit# feat: add permission check for secondary queue selection
v-kkhuang Apr 14, 2026
f91be62
#AI commit# refactor: implement Yarn API-based permission check for s…
v-kkhuang Apr 14, 2026
0feb857
Revert "#AI commit# refactor: implement Yarn API-based permission che…
v-kkhuang Apr 14, 2026
92b5f8c
Revert "#AI commit# feat: add permission check for secondary queue se…
v-kkhuang Apr 14, 2026
8fb5aed
#AI commit# 开发阶段: 优化获取配置队列方式
v-kkhuang Apr 14, 2026
ba8707d
#AI commit# 开发阶段: 优化spark引擎队列获取方式
v-kkhuang Apr 16, 2026
fa226a9
Merge branch 'dev-1.18.2-webank' into dev-1.18.0-secondary-queue
v-kkhuang Apr 16, 2026
67e3b06
#AI commit# 开发阶段: spark备用队列逻辑迁移
v-kkhuang Apr 16, 2026
f5ceb29
#AI commit# 开发阶段: spark备用队列逻辑迁移
v-kkhuang Apr 17, 2026
f4dbdb1
Merge remote-tracking branch 'origin/dev-1.18.2-webank' into dev-1.18…
v-kkhuang Apr 17, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,9 @@ object Configuration extends Logging {
val CLOUD_CONSOLE_VARIABLE_SPRING_APPLICATION_NAME =
CommonVars("wds.linkis.console.variable.application.name", "linkis-ps-publicservice")

val MANAGER_SPRING_APPLICATION_NAME =
CommonVars("wds.linkis.engineconn.manager.name", "linkis-cg-linkismanager")

val JOBHISTORY_SPRING_APPLICATION_NAME =
CommonVars("wds.linkis.jobhistory.application.name", "linkis-ps-jobhistory")

Expand Down Expand Up @@ -104,6 +107,34 @@ object Configuration extends Logging {
val METRICS_INCREMENTAL_UPDATE_ENABLE =
CommonVars[Boolean]("linkis.jobhistory.metrics.incremental.update.enable", false)

/**
* Whether to enable secondary queue feature Default: true Description: true enables smart queue
* selection, false disables the feature
*/
val SECONDARY_QUEUE_ENABLED: CommonVars[Boolean] =
CommonVars.apply("wds.linkis.rm.secondary.yarnqueue.enable", false)

/**
* Secondary queue resource usage threshold Default: 0.9 (90%) Description: Use secondary queue
* when usage <= this value, use primary queue when usage > this value
*/
val SECONDARY_QUEUE_THRESHOLD: CommonVars[Double] =
CommonVars.apply("wds.linkis.rm.secondary.yarnqueue.threshold", 0.9)

/**
* Supported engine type list (comma-separated) Default: spark Description: Only engines in this
* list will execute smart queue selection Case-insensitive
*/
val SECONDARY_QUEUE_ENGINES: CommonVars[String] =
CommonVars.apply("wds.linkis.rm.secondary.yarnqueue.engines", "spark")

/**
* Supported Creator list (comma-separated) Default: IDE Description: Only Creators in this list
* will execute smart queue selection Case-insensitive
*/
val SECONDARY_QUEUE_CREATORS: CommonVars[String] =
CommonVars.apply("wds.linkis.rm.secondary.yarnqueue.creators", "IDE")

val EXECUTE_ERROR_CODE_INDEX =
CommonVars("execute.error.code.index", "-1")

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,6 @@ import org.apache.linkis.common.variable.DateTypeUtils.{
getCurHour,
getMonthDay,
getToday,
getWeekBegin,
getWeekEnd,
getYesterday
}

Expand Down Expand Up @@ -250,16 +248,11 @@ object VariableUtils extends Logging {
// Initialize week variables (with feature switch and exception handling)
if (WEEK_VARIABLE_ENABLED.getValue) {
Utils.tryAndWarn {
val weekBegin = getWeekBegin(std = false, run_date.getDate)
val weekBeginStd = getWeekBegin(std = true, run_date.getDate)
val weekEnd = getWeekEnd(std = false, run_date.getDate)
val weekEndStd = getWeekEnd(std = true, run_date.getDate)

// Use WeekType for week-based arithmetic (unit = weeks, not days)
nameAndType("run_week_begin") = WeekType(new CustomDateType(weekBegin, false))
nameAndType("run_week_begin_std") = WeekType(new CustomDateType(weekBeginStd, true))
nameAndType("run_week_end") = WeekType(new CustomDateType(weekEnd, false))
nameAndType("run_week_end_std") = WeekType(new CustomDateType(weekEndStd, true))
// Use CustomWeekType following the same pattern as CustomMonthType
nameAndType("run_week_begin") = WeekType(new CustomWeekType(run_date_str, false, false))
nameAndType("run_week_begin_std") = WeekType(new CustomWeekType(run_date_str, true, false))
nameAndType("run_week_end") = WeekType(new CustomWeekType(run_date_str, false, true))
nameAndType("run_week_end_std") = WeekType(new CustomWeekType(run_date_str, true, true))
logger.info("Week variables initialized successfully")
}
} else {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -255,3 +255,32 @@ class CustomHourType(dateH: String, std: Boolean = true) {
}

}

/**
* CustomWeekType: Week-based date type following the same pattern as CustomMonType
*
* @param date
* Base date string
* @param std
* Whether to use standard format (true: yyyy-MM-dd, false: yyyyMMdd)
* @param isEnd
* Whether to calculate week end (Sunday) instead of week begin (Monday)
*/
class CustomWeekType(date: String, std: Boolean = true, isEnd: Boolean = false) {

def -(weeks: Int): String = {
val dateFormat = DateTypeUtils.dateFormatLocal.get()
DateTypeUtils.getWeek(std, isEnd, DateUtils.addWeeks(dateFormat.parse(date), -weeks))
}

def +(weeks: Int): String = {
val dateFormat = DateTypeUtils.dateFormatLocal.get()
DateTypeUtils.getWeek(std, isEnd, DateUtils.addWeeks(dateFormat.parse(date), weeks))
}

override def toString: String = {
val dateFormat = DateTypeUtils.dateFormatLocal.get()
DateTypeUtils.getWeek(std, isEnd, dateFormat.parse(date))
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -253,16 +253,18 @@ object DateTypeUtils {
}

/**
* Get the start date of the week (Monday)
* Get week date (Monday or Sunday)
*
* @param std
* Whether to use standard format (true: yyyy-MM-dd, false: yyyyMMdd)
* @param isEnd
* Whether to get week end (Sunday) instead of week begin (Monday)
* @param date
* Base date
* @return
* Monday date string
* Monday or Sunday date string depending on isEnd
*/
def getWeekBegin(std: Boolean = true, date: Date): String = {
def getWeek(std: Boolean = true, isEnd: Boolean = false, date: Date): String = {
try {
val dateFormat = dateFormatLocal.get()
val dateFormat_std = dateFormatStdLocal.get()
Expand All @@ -272,75 +274,21 @@ object DateTypeUtils {
// Get current day of week (Calendar.SUNDAY=1, Calendar.MONDAY=2, ..., Calendar.SATURDAY=7)
val dayOfWeek = cal.get(Calendar.DAY_OF_WEEK)

// Calculate days to Monday
// Sunday(1) needs to go back 6 days to this week's Monday
// Monday(2) doesn't need adjustment
// Tuesday(3) needs to go back 1 day
// ...
// Saturday(7) needs to go back 5 days
val daysToMonday = if (dayOfWeek == Calendar.SUNDAY) {
-6 // Sunday goes back 6 days to this week's Monday
} else {
Calendar.MONDAY - dayOfWeek // Other dates go back to this week's Monday
}

cal.add(Calendar.DAY_OF_MONTH, daysToMonday)

if (std) {
dateFormat_std.format(cal.getTime)
} else {
dateFormat.format(cal.getTime)
}
} catch {
case e: Exception =>
// Return current date as fallback on error
val fallbackFormat = if (std) dateFormatStdLocal.get() else dateFormatLocal.get()
fallbackFormat.format(date)
}
}

/**
* Get the end date of the week (Sunday)
*
* @param std
* Whether to use standard format (true: yyyy-MM-dd, false: yyyyMMdd)
* @param date
* Base date
* @return
* Sunday date string
*/
def getWeekEnd(std: Boolean = true, date: Date): String = {
try {
val dateFormat = dateFormatLocal.get()
val dateFormat_std = dateFormatStdLocal.get()
val cal: Calendar = Calendar.getInstance()
cal.setTime(date)

// Get current day of week
val dayOfWeek = cal.get(Calendar.DAY_OF_WEEK)

// Calculate days to Sunday
// Sunday(1) doesn't need adjustment
// Monday(2) needs to go forward 6 days
// Tuesday(3) needs to go forward 5 days
// ...
// Saturday(7) needs to go forward 1 day
val daysToSunday = if (dayOfWeek == Calendar.SUNDAY) {
0 // Sunday doesn't need adjustment
// Calculate days to target day (Monday or Sunday)
val daysToTarget = if (isEnd) {
// Calculate days to Sunday
if (dayOfWeek == Calendar.SUNDAY) 0 else Calendar.SUNDAY - dayOfWeek + 7
} else {
Calendar.SUNDAY - dayOfWeek + 7 // Other dates go forward to this week's Sunday
// Calculate days to Monday
if (dayOfWeek == Calendar.SUNDAY) -6 else Calendar.MONDAY - dayOfWeek
}

cal.add(Calendar.DAY_OF_MONTH, daysToSunday)
cal.add(Calendar.DAY_OF_MONTH, daysToTarget)

if (std) {
dateFormat_std.format(cal.getTime)
} else {
dateFormat.format(cal.getTime)
}
if (std) dateFormat_std.format(cal.getTime)
else dateFormat.format(cal.getTime)
} catch {
case e: Exception =>
// Return current date as fallback on error
val fallbackFormat = if (std) dateFormatStdLocal.get() else dateFormatLocal.get()
fallbackFormat.format(date)
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,19 +41,21 @@ case class DateType(value: CustomDateType) extends VariableType {
}

/**
* WeekType: A date type that operates in weeks instead of days The arithmetic operations (+/-) are
* performed in units of weeks (1 week = 7 days) Example: run_week_begin - 1 means subtract 1 week
* (7 days), not 1 day
* WeekType: A week-based date type following the same pattern as MonthType
*
* Arithmetic operations (+/-) are performed in units of weeks:
* - run_week_begin - 1 means subtract 1 week (previous Monday)
* - run_week_end + 2 means add 2 weeks (Sunday after 2 weeks)
*
* Uses CustomWeekType internally for week-specific calculations
*/
case class WeekType(value: CustomDateType) extends VariableType {
case class WeekType(value: CustomWeekType) extends VariableType {
override def getValue: String = value.toString

def calculator(signal: String, bValue: String): String = {
// Convert weeks to days for the underlying date calculation
val days = bValue.toInt * 7
signal match {
case "+" => value + days
case "-" => value - days
case "+" => value + bValue.toInt
case "-" => value - bValue.toInt
case _ =>
throw new LinkisCommonErrorException(20046, s"WeekType is not supported to use:$signal")
}
Expand Down
Loading
Loading