You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@inlong.apache.org by "justinwwhuang (via GitHub)" <gi...@apache.org> on 2023/10/27 03:15:37 UTC
[PR] [INLONG-9134][Agent] Add file related utils [inlong]
justinwwhuang opened a new pull request, #9135:
URL: https://github.com/apache/inlong/pull/9135
### Prepare a Pull Request
[INLONG-9134][Agent] Add file related utils
- Fixes #9134
### Motivation
Add file related utils
### Modifications
Add file related utils
### Verifying this change
*(Please pick either of the following options)*
- [ ] This change is a trivial rework/code cleanup without any test coverage.
- [ ] This change is already covered by existing tests, such as:
*(please describe tests)*
- [ ] This change added tests and can be verified as follows:
*(example:)*
- *Added integration tests for end-to-end deployment with large payloads (10MB)*
- *Extended integration test for recovery after broker failure*
### Documentation
- Does this pull request introduce a new feature? (yes / no)
- If yes, how is the feature documented? (not applicable / docs / JavaDocs / not documented)
- If a feature is not applicable for documentation, explain why?
- If a feature is not documented yet in this PR, please create a follow-up issue for adding the documentation
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@inlong.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] [INLONG-9134][Agent] Add file related utils [inlong]
Posted by "EMsnap (via GitHub)" <gi...@apache.org>.
EMsnap commented on code in PR #9135:
URL: https://github.com/apache/inlong/pull/9135#discussion_r1374110971
##########
inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/utils/file/NewDateUtils.java:
##########
@@ -0,0 +1,937 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.agent.plugin.utils.file;
+
+import hirondelle.date4j.DateTime;
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Calendar;
+import java.util.Date;
+import java.util.List;
+import java.util.StringTokenizer;
+import java.util.TimeZone;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class NewDateUtils {
+
+ public static final String FULL_FORMAT = "yyyyMMddHHmmss";
+ public static final String NULL_DATA_TIME = "000000000000";
+ public static final String DEFAULT_FORMAT = "yyyyMMddHHmm";
+ public static final String DEFAULT_TIME_ZONE = "Asia/Shanghai";
+ private static final Logger logger = LoggerFactory.getLogger(NewDateUtils.class);
+ private static final String TIME_REGEX = "YYYY(?:.MM|MM)?(?:.DD|DD)?(?:.hh|hh)?(?:.mm|mm)?(?:"
+ + ".ss|ss)?";
+ private static final String LIMIT_SEP = "(?<=[a-zA-Z])";
+ private static final String LETTER_STR = "\\D+";
+ private static final String DIGIT_STR = "[0-9]+";
+ private static final Pattern pattern = Pattern.compile(TIME_REGEX,
+ Pattern.CASE_INSENSITIVE | Pattern.DOTALL | Pattern.MULTILINE);
+ private static final Pattern bracePatt = Pattern.compile("\\{(.*?)\\}");
+ private static final int DEFAULT_LENGTH = "yyyyMMddHHmm".length();
+ public static long DAY_TIMEOUT_INTERVAL = 2 * 24 * 3600 * 1000;
+ public static long HOUR_TIMEOUT_INTERVAL = 2 * 3600 * 1000;
+ // data source config error */
+ public static final String DATA_SOURCE_CONFIG_ERROR = "ERROR-0-TDAgent|10001|ERROR"
+ + "|ERROR_DATA_SOURCE_CONFIG|";
+
+ /* Return the time in milliseconds for a data time. */
+ /*
+ * public static long getTimeInMillis(String dataTime) { if (dataTime == null) { return 0; }
+ *
+ * try { SimpleDateFormat dateFormat = new SimpleDateFormat(DEFAULT_FORMAT); Date date = dateFormat.parse(dataTime);
+ *
+ * return date.getTime(); } catch (ParseException e) { return 0; } }
+ */
+ /* Return the format data time string from milliseconds. */
+ /*
+ * public static String getDataTimeFromTimeMillis(long dataTime) { SimpleDateFormat dateFormat = new
+ * SimpleDateFormat(DEFAULT_FORMAT); return dateFormat.format(new Date(dataTime)); }
+ */
+ /* Return the should start time for a data time log file. */
+ public static String getShouldStartTime(String dataTime, String cycleUnit,
+ String offset) {
+ if (dataTime == null || dataTime.length() > 12) {
+ return null;
+ }
+
+ SimpleDateFormat dateFormat = new SimpleDateFormat(DEFAULT_FORMAT);
+ TimeZone timeZone = TimeZone.getTimeZone(NewDateUtils.DEFAULT_TIME_ZONE);
+ dateFormat.setTimeZone(timeZone);
+
+ if (dataTime.length() < DEFAULT_LENGTH) {
+ StringBuffer sb = new StringBuffer();
+ for (int i = 0; i < DEFAULT_LENGTH - dataTime.length(); i++) {
+ sb.append("0");
+ }
+ dataTime = dataTime + sb.toString();
+ }
+
+ Calendar calendar = Calendar.getInstance();
+ try {
+ calendar.setTimeInMillis(dateFormat.parse(dataTime).getTime());
+ } catch (ParseException e) {
+ return null;
+ }
+
+ /*
+ * The delay should be added to the data time, so remove the - from offset.
+ */
+ if (offset.startsWith("-")) {
+ offset = offset.substring(1, offset.length());
+ } else { // positive，read file earlier
+ offset = "-" + offset;
+ }
+
+ return dateFormat
+ .format(new Date(getDateTime(calendar, cycleUnit, offset).getTimeInMillis()));
+ }
+
+ private static Calendar getCurDate(String cycleUnit, String offset) {
+ if (cycleUnit == null || cycleUnit.length() == 0) {
+ return null;
+ }
+
+ Calendar calendar = Calendar.getInstance();
+ calendar.setTimeInMillis(System.currentTimeMillis());
+
+ return getDateTime(calendar, cycleUnit, offset);
+ }
+
+ public static String getDateTime(String dataTime, String cycleUnit, String offset) {
+ String retTime = NewDateUtils.millSecConvertToTimeStr(
+ System.currentTimeMillis(), cycleUnit);
+ try {
+ long time = NewDateUtils.timeStrConvertTomillSec(dataTime, cycleUnit);
+
+ Calendar calendar = Calendar.getInstance();
+ calendar.setTimeInMillis(time);
+ Calendar retCalendar = getDateTime(calendar, cycleUnit, offset);
+ if (retCalendar == null) {
+ return dataTime;
+ }
+
+ retTime = NewDateUtils.millSecConvertToTimeStr(retCalendar.getTime().getTime(),
+ cycleUnit);
+ } catch (Exception e) {
+ logger.error("getDateTime error: ", e);
+ }
+ return retTime;
+ }
+
+ public static String getDateTime(long time, String cycleUnit, String offset) {
+ Calendar calendar = Calendar.getInstance();
+ calendar.setTimeInMillis(time);
+ Calendar retCalendar = getDateTime(calendar, cycleUnit, offset);
+ return NewDateUtils.millSecConvertToTimeStr(retCalendar.getTime().getTime(), cycleUnit);
+ }
+
+ private static Calendar getDateTime(Calendar calendar, String cycleUnit, String offset) {
+ int cycleNumber = (cycleUnit.length() <= 1
+ ? 1
+ : Integer.parseInt(cycleUnit.substring(0, cycleUnit.length() - 1)));
+
+ String offsetUnit = offset.substring(offset.length() - 1, offset.length());
+ int offsetNumber = Integer.parseInt(offset.substring(0, offset.length() - 1));
+
+ /*
+ * For day task, the offset cycle unit can only be day; for hourly task, the offset can't be minute; for
+ * minutely task, the offset cycle unit can be day, hour and minute, but if the offset cycle unit is minute, the
+ * offset must be divided by cycle number.
+ */
+ if (cycleUnit.length() > 1 && (StringUtils.endsWithIgnoreCase(cycleUnit, "M"))) {
+ calendar.set(Calendar.SECOND, 0);
+ int minTime = calendar.get(Calendar.MINUTE);
+
+ int leftMin = minTime % cycleNumber;
+ minTime = minTime - leftMin;
+ calendar.set(Calendar.MINUTE, minTime);
+
+ /* Calculate the offset. */
+ if ("D".equalsIgnoreCase(offsetUnit)) {
+ calendar.add(Calendar.DAY_OF_YEAR, offsetNumber);
+ }
+
+ if ("H".equalsIgnoreCase(offsetUnit)) {
+ calendar.add(Calendar.HOUR_OF_DAY, offsetNumber);
+ }
+ } else if (cycleUnit.length() == 1) {
+ if ("D".equalsIgnoreCase(cycleUnit)) {
+ calendar.set(Calendar.HOUR_OF_DAY, 0);
+ calendar.set(Calendar.MINUTE, 0);
+ calendar.set(Calendar.SECOND, 0);
+ } else if ("h".equalsIgnoreCase(cycleUnit)) {
+ calendar.set(Calendar.MINUTE, 0);
+ calendar.set(Calendar.SECOND, 0);
+ }
+ }
+
+ /* Calculate the offset. */
+ if ("D".equalsIgnoreCase(offsetUnit)) {
+ calendar.add(Calendar.DAY_OF_YEAR, offsetNumber);
+ }
+
+ if ("h".equalsIgnoreCase(offsetUnit)) {
+ calendar.add(Calendar.HOUR_OF_DAY, offsetNumber);
+ }
+
+ if ("m".equals(offsetUnit)) {
+ calendar.add(Calendar.MINUTE, offsetNumber);
+ }
+
+ return calendar;
+ }
+
+ public static boolean isValidCreationTime(String dataTime, String cycleUnit,
+ String timeOffset) {
+ long timeInterval = 0;
+ if ("Y".equalsIgnoreCase(cycleUnit)) {
+ timeInterval = DAY_TIMEOUT_INTERVAL;
+ } else if ("M".equals(cycleUnit)) {
+ timeInterval = HOUR_TIMEOUT_INTERVAL;
+ } else if ("D".equalsIgnoreCase(cycleUnit)) {
+ timeInterval = DAY_TIMEOUT_INTERVAL;
+ } else if ("h".equalsIgnoreCase(cycleUnit)) {
+ timeInterval = HOUR_TIMEOUT_INTERVAL;
+ } else if (cycleUnit.contains("m")) {
+ timeInterval = HOUR_TIMEOUT_INTERVAL;
+ } else {
+ logger.error("cycleUnit {} can't parse!", cycleUnit);
+ timeInterval = DAY_TIMEOUT_INTERVAL;
+ }
+
+ // 处理偏移量，超时周期要加上时间偏移偏移量
+ if (timeOffset.startsWith("-")) {
+ timeInterval += caclOffset(timeOffset);
+ } else { // 处理向后偏移
+ timeInterval -= caclOffset(timeOffset);
+ }
+
+ return isValidCreationTime(dataTime, timeInterval);
+ }
+
+ /**
+ * 根据偏移量计算偏移时间
Review Comment:
chinese
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@inlong.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] [INLONG-9134][Agent] Add file related utils [inlong]
Posted by "EMsnap (via GitHub)" <gi...@apache.org>.
EMsnap commented on code in PR #9135:
URL: https://github.com/apache/inlong/pull/9135#discussion_r1374037001
##########
inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/utils/file/NewDateUtils.java:
##########
@@ -0,0 +1,955 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.agent.plugin.utils.file;
+
+import hirondelle.date4j.DateTime;
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Calendar;
+import java.util.Date;
+import java.util.List;
+import java.util.StringTokenizer;
+import java.util.TimeZone;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class NewDateUtils {
+
+ public static final String FULL_FORMAT = "yyyyMMddHHmmss";
+ public static final String NULL_DATA_TIME = "000000000000";
+ public static final String DEFAULT_FORMAT = "yyyyMMddHHmm";
+ public static final String DEFAULT_TIME_ZONE = "Asia/Shanghai";
+ private static final Logger logger = LoggerFactory.getLogger(NewDateUtils.class);
+ private static final String TIME_REGEX = "YYYY(?:.MM|MM)?(?:.DD|DD)?(?:.hh|hh)?(?:.mm|mm)?(?:"
+ + ".ss|ss)?";
+ private static final String LIMIT_SEP = "(?<=[a-zA-Z])";
+ private static final String LETTER_STR = "\\D+";
+ private static final String DIGIT_STR = "[0-9]+";
+ private static final Pattern pattern = Pattern.compile(TIME_REGEX,
+ Pattern.CASE_INSENSITIVE | Pattern.DOTALL | Pattern.MULTILINE);
+ private static final Pattern bracePatt = Pattern.compile("\\{(.*?)\\}");
+ private static final int DEFAULT_LENGTH = "yyyyMMddHHmm".length();
+ public static long DAY_TIMEOUT_INTERVAL = 2 * 24 * 3600 * 1000;
+ public static long HOUR_TIMEOUT_INTERVAL = 2 * 3600 * 1000;
+ // 数据源配置异常 */
+ public static final String DATA_SOURCE_CONFIG_ERROR = "ERROR-0-TDAgent|10001|ERROR"
+ + "|ERROR_DATA_SOURCE_CONFIG|";
+
+ /* Return the time in milliseconds for a data time. */
+ /*
+ * public static long getTimeInMillis(String dataTime) { if (dataTime == null) { return 0; }
+ *
+ * try { SimpleDateFormat dateFormat = new SimpleDateFormat(DEFAULT_FORMAT); Date date = dateFormat.parse(dataTime);
+ *
+ * return date.getTime(); } catch (ParseException e) { return 0; } }
+ */
+ /* Return the format data time string from milliseconds. */
+ /*
+ * public static String getDataTimeFromTimeMillis(long dataTime) { SimpleDateFormat dateFormat = new
+ * SimpleDateFormat(DEFAULT_FORMAT); return dateFormat.format(new Date(dataTime)); }
+ */
+ /* Return the should start time for a data time log file. */
+ public static String getShouldStartTime(String dataTime, String cycleUnit,
+ String offset) {
+ if (dataTime == null || dataTime.length() > 12) {
+ return null;
+ }
+
+ SimpleDateFormat dateFormat = new SimpleDateFormat(DEFAULT_FORMAT);
+ TimeZone timeZone = TimeZone.getTimeZone(NewDateUtils.DEFAULT_TIME_ZONE);
+ dateFormat.setTimeZone(timeZone);
+
+ if (dataTime.length() < DEFAULT_LENGTH) {
+ StringBuffer sb = new StringBuffer();
+ for (int i = 0; i < DEFAULT_LENGTH - dataTime.length(); i++) {
+ sb.append("0");
+ }
+ dataTime = dataTime + sb.toString();
+ }
+
+ Calendar calendar = Calendar.getInstance();
+ try {
+ calendar.setTimeInMillis(dateFormat.parse(dataTime).getTime());
+ } catch (ParseException e) {
+ return null;
+ }
+
+ /*
+ * The delay should be added to the data time, so remove the - from offset.
+ */
+ if (offset.startsWith("-")) {
+ offset = offset.substring(1, offset.length());
+ } else { // 为正，配置提前读取文件
Review Comment:
ditto
##########
inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/utils/file/NewDateUtils.java:
##########
@@ -0,0 +1,955 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.agent.plugin.utils.file;
+
+import hirondelle.date4j.DateTime;
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Calendar;
+import java.util.Date;
+import java.util.List;
+import java.util.StringTokenizer;
+import java.util.TimeZone;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class NewDateUtils {
+
+ public static final String FULL_FORMAT = "yyyyMMddHHmmss";
+ public static final String NULL_DATA_TIME = "000000000000";
+ public static final String DEFAULT_FORMAT = "yyyyMMddHHmm";
+ public static final String DEFAULT_TIME_ZONE = "Asia/Shanghai";
+ private static final Logger logger = LoggerFactory.getLogger(NewDateUtils.class);
+ private static final String TIME_REGEX = "YYYY(?:.MM|MM)?(?:.DD|DD)?(?:.hh|hh)?(?:.mm|mm)?(?:"
+ + ".ss|ss)?";
+ private static final String LIMIT_SEP = "(?<=[a-zA-Z])";
+ private static final String LETTER_STR = "\\D+";
+ private static final String DIGIT_STR = "[0-9]+";
+ private static final Pattern pattern = Pattern.compile(TIME_REGEX,
+ Pattern.CASE_INSENSITIVE | Pattern.DOTALL | Pattern.MULTILINE);
+ private static final Pattern bracePatt = Pattern.compile("\\{(.*?)\\}");
+ private static final int DEFAULT_LENGTH = "yyyyMMddHHmm".length();
+ public static long DAY_TIMEOUT_INTERVAL = 2 * 24 * 3600 * 1000;
+ public static long HOUR_TIMEOUT_INTERVAL = 2 * 3600 * 1000;
+ // 数据源配置异常 */
+ public static final String DATA_SOURCE_CONFIG_ERROR = "ERROR-0-TDAgent|10001|ERROR"
+ + "|ERROR_DATA_SOURCE_CONFIG|";
+
+ /* Return the time in milliseconds for a data time. */
+ /*
+ * public static long getTimeInMillis(String dataTime) { if (dataTime == null) { return 0; }
+ *
+ * try { SimpleDateFormat dateFormat = new SimpleDateFormat(DEFAULT_FORMAT); Date date = dateFormat.parse(dataTime);
+ *
+ * return date.getTime(); } catch (ParseException e) { return 0; } }
+ */
+ /* Return the format data time string from milliseconds. */
+ /*
+ * public static String getDataTimeFromTimeMillis(long dataTime) { SimpleDateFormat dateFormat = new
+ * SimpleDateFormat(DEFAULT_FORMAT); return dateFormat.format(new Date(dataTime)); }
+ */
+ /* Return the should start time for a data time log file. */
+ public static String getShouldStartTime(String dataTime, String cycleUnit,
+ String offset) {
+ if (dataTime == null || dataTime.length() > 12) {
+ return null;
+ }
+
+ SimpleDateFormat dateFormat = new SimpleDateFormat(DEFAULT_FORMAT);
+ TimeZone timeZone = TimeZone.getTimeZone(NewDateUtils.DEFAULT_TIME_ZONE);
+ dateFormat.setTimeZone(timeZone);
+
+ if (dataTime.length() < DEFAULT_LENGTH) {
+ StringBuffer sb = new StringBuffer();
+ for (int i = 0; i < DEFAULT_LENGTH - dataTime.length(); i++) {
+ sb.append("0");
+ }
+ dataTime = dataTime + sb.toString();
+ }
+
+ Calendar calendar = Calendar.getInstance();
+ try {
+ calendar.setTimeInMillis(dateFormat.parse(dataTime).getTime());
+ } catch (ParseException e) {
+ return null;
+ }
+
+ /*
+ * The delay should be added to the data time, so remove the - from offset.
+ */
+ if (offset.startsWith("-")) {
+ offset = offset.substring(1, offset.length());
+ } else { // 为正，配置提前读取文件
+ offset = "-" + offset;
+ }
+
+ return dateFormat
+ .format(new Date(getDateTime(calendar, cycleUnit, offset).getTimeInMillis()));
+ }
+
+ private static Calendar getCurDate(String cycleUnit, String offset) {
+ if (cycleUnit == null || cycleUnit.length() == 0) {
+ return null;
+ }
+
+ Calendar calendar = Calendar.getInstance();
+ calendar.setTimeInMillis(System.currentTimeMillis());
+
+ return getDateTime(calendar, cycleUnit, offset);
+ }
+
+ public static String getDateTime(String dataTime, String cycleUnit, String offset) {
+ String retTime = NewDateUtils.millSecConvertToTimeStr(
+ System.currentTimeMillis(), cycleUnit);
+ try {
+ long time = NewDateUtils.timeStrConvertTomillSec(dataTime, cycleUnit);
+
+ Calendar calendar = Calendar.getInstance();
+ calendar.setTimeInMillis(time);
+ Calendar retCalendar = getDateTime(calendar, cycleUnit, offset);
+ if (retCalendar == null) {
+ return dataTime;
+ }
+
+ retTime = NewDateUtils.millSecConvertToTimeStr(retCalendar.getTime().getTime(),
+ cycleUnit);
+ } catch (Exception e) {
+ logger.error("getDateTime error: ", e);
+ }
+ return retTime;
+ }
+
+ public static String getDateTime(long time, String cycleUnit, String offset) {
+ // String retTime = NewDateUtils.millSecConvertToTimeStr(System.currentTimeMillis(),
+ // cycleUnit);
+
+ Calendar calendar = Calendar.getInstance();
+ calendar.setTimeInMillis(time);
+ Calendar retCalendar = getDateTime(calendar, cycleUnit, offset);
+ return NewDateUtils.millSecConvertToTimeStr(retCalendar.getTime().getTime(), cycleUnit);
+
+ // return retTime;
+ }
+
+ private static Calendar getDateTime(Calendar calendar, String cycleUnit, String offset) {
+ int cycleNumber = (cycleUnit.length() <= 1
+ ? 1
+ : Integer.parseInt(cycleUnit.substring(0, cycleUnit.length() - 1)));
+
+ String offsetUnit = offset.substring(offset.length() - 1, offset.length());
+ int offsetNumber = Integer.parseInt(offset.substring(0, offset.length() - 1));
+
+ /*
+ * For day task, the offset cycle unit can only be day; for hourly task, the offset can't be minute; for
+ * minutely task, the offset cycle unit can be day, hour and minute, but if the offset cycle unit is minute, the
+ * offset must be divided by cycle number.
+ */
+ // if (("D".equalsIgnoreCase(cycleUnit) && !"D".equalsIgnoreCase(offsetUnit))) {
Review Comment:
ditto
##########
inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/utils/file/NewDateUtils.java:
##########
@@ -0,0 +1,955 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.agent.plugin.utils.file;
+
+import hirondelle.date4j.DateTime;
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Calendar;
+import java.util.Date;
+import java.util.List;
+import java.util.StringTokenizer;
+import java.util.TimeZone;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class NewDateUtils {
+
+ public static final String FULL_FORMAT = "yyyyMMddHHmmss";
+ public static final String NULL_DATA_TIME = "000000000000";
+ public static final String DEFAULT_FORMAT = "yyyyMMddHHmm";
+ public static final String DEFAULT_TIME_ZONE = "Asia/Shanghai";
+ private static final Logger logger = LoggerFactory.getLogger(NewDateUtils.class);
+ private static final String TIME_REGEX = "YYYY(?:.MM|MM)?(?:.DD|DD)?(?:.hh|hh)?(?:.mm|mm)?(?:"
+ + ".ss|ss)?";
+ private static final String LIMIT_SEP = "(?<=[a-zA-Z])";
+ private static final String LETTER_STR = "\\D+";
+ private static final String DIGIT_STR = "[0-9]+";
+ private static final Pattern pattern = Pattern.compile(TIME_REGEX,
+ Pattern.CASE_INSENSITIVE | Pattern.DOTALL | Pattern.MULTILINE);
+ private static final Pattern bracePatt = Pattern.compile("\\{(.*?)\\}");
+ private static final int DEFAULT_LENGTH = "yyyyMMddHHmm".length();
+ public static long DAY_TIMEOUT_INTERVAL = 2 * 24 * 3600 * 1000;
+ public static long HOUR_TIMEOUT_INTERVAL = 2 * 3600 * 1000;
+ // 数据源配置异常 */
Review Comment:
chinese
##########
inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/utils/file/NewDateUtils.java:
##########
@@ -0,0 +1,955 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.agent.plugin.utils.file;
+
+import hirondelle.date4j.DateTime;
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Calendar;
+import java.util.Date;
+import java.util.List;
+import java.util.StringTokenizer;
+import java.util.TimeZone;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class NewDateUtils {
+
+ public static final String FULL_FORMAT = "yyyyMMddHHmmss";
+ public static final String NULL_DATA_TIME = "000000000000";
+ public static final String DEFAULT_FORMAT = "yyyyMMddHHmm";
+ public static final String DEFAULT_TIME_ZONE = "Asia/Shanghai";
+ private static final Logger logger = LoggerFactory.getLogger(NewDateUtils.class);
+ private static final String TIME_REGEX = "YYYY(?:.MM|MM)?(?:.DD|DD)?(?:.hh|hh)?(?:.mm|mm)?(?:"
+ + ".ss|ss)?";
+ private static final String LIMIT_SEP = "(?<=[a-zA-Z])";
+ private static final String LETTER_STR = "\\D+";
+ private static final String DIGIT_STR = "[0-9]+";
+ private static final Pattern pattern = Pattern.compile(TIME_REGEX,
+ Pattern.CASE_INSENSITIVE | Pattern.DOTALL | Pattern.MULTILINE);
+ private static final Pattern bracePatt = Pattern.compile("\\{(.*?)\\}");
+ private static final int DEFAULT_LENGTH = "yyyyMMddHHmm".length();
+ public static long DAY_TIMEOUT_INTERVAL = 2 * 24 * 3600 * 1000;
+ public static long HOUR_TIMEOUT_INTERVAL = 2 * 3600 * 1000;
+ // 数据源配置异常 */
+ public static final String DATA_SOURCE_CONFIG_ERROR = "ERROR-0-TDAgent|10001|ERROR"
+ + "|ERROR_DATA_SOURCE_CONFIG|";
+
+ /* Return the time in milliseconds for a data time. */
+ /*
+ * public static long getTimeInMillis(String dataTime) { if (dataTime == null) { return 0; }
+ *
+ * try { SimpleDateFormat dateFormat = new SimpleDateFormat(DEFAULT_FORMAT); Date date = dateFormat.parse(dataTime);
+ *
+ * return date.getTime(); } catch (ParseException e) { return 0; } }
+ */
+ /* Return the format data time string from milliseconds. */
+ /*
+ * public static String getDataTimeFromTimeMillis(long dataTime) { SimpleDateFormat dateFormat = new
+ * SimpleDateFormat(DEFAULT_FORMAT); return dateFormat.format(new Date(dataTime)); }
+ */
+ /* Return the should start time for a data time log file. */
+ public static String getShouldStartTime(String dataTime, String cycleUnit,
+ String offset) {
+ if (dataTime == null || dataTime.length() > 12) {
+ return null;
+ }
+
+ SimpleDateFormat dateFormat = new SimpleDateFormat(DEFAULT_FORMAT);
+ TimeZone timeZone = TimeZone.getTimeZone(NewDateUtils.DEFAULT_TIME_ZONE);
+ dateFormat.setTimeZone(timeZone);
+
+ if (dataTime.length() < DEFAULT_LENGTH) {
+ StringBuffer sb = new StringBuffer();
+ for (int i = 0; i < DEFAULT_LENGTH - dataTime.length(); i++) {
+ sb.append("0");
+ }
+ dataTime = dataTime + sb.toString();
+ }
+
+ Calendar calendar = Calendar.getInstance();
+ try {
+ calendar.setTimeInMillis(dateFormat.parse(dataTime).getTime());
+ } catch (ParseException e) {
+ return null;
+ }
+
+ /*
+ * The delay should be added to the data time, so remove the - from offset.
+ */
+ if (offset.startsWith("-")) {
+ offset = offset.substring(1, offset.length());
+ } else { // 为正，配置提前读取文件
+ offset = "-" + offset;
+ }
+
+ return dateFormat
+ .format(new Date(getDateTime(calendar, cycleUnit, offset).getTimeInMillis()));
+ }
+
+ private static Calendar getCurDate(String cycleUnit, String offset) {
+ if (cycleUnit == null || cycleUnit.length() == 0) {
+ return null;
+ }
+
+ Calendar calendar = Calendar.getInstance();
+ calendar.setTimeInMillis(System.currentTimeMillis());
+
+ return getDateTime(calendar, cycleUnit, offset);
+ }
+
+ public static String getDateTime(String dataTime, String cycleUnit, String offset) {
+ String retTime = NewDateUtils.millSecConvertToTimeStr(
+ System.currentTimeMillis(), cycleUnit);
+ try {
+ long time = NewDateUtils.timeStrConvertTomillSec(dataTime, cycleUnit);
+
+ Calendar calendar = Calendar.getInstance();
+ calendar.setTimeInMillis(time);
+ Calendar retCalendar = getDateTime(calendar, cycleUnit, offset);
+ if (retCalendar == null) {
+ return dataTime;
+ }
+
+ retTime = NewDateUtils.millSecConvertToTimeStr(retCalendar.getTime().getTime(),
+ cycleUnit);
+ } catch (Exception e) {
+ logger.error("getDateTime error: ", e);
+ }
+ return retTime;
+ }
+
+ public static String getDateTime(long time, String cycleUnit, String offset) {
+ // String retTime = NewDateUtils.millSecConvertToTimeStr(System.currentTimeMillis(),
Review Comment:
remove useless code
##########
inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/utils/file/NewDateUtils.java:
##########
@@ -0,0 +1,955 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.agent.plugin.utils.file;
+
+import hirondelle.date4j.DateTime;
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Calendar;
+import java.util.Date;
+import java.util.List;
+import java.util.StringTokenizer;
+import java.util.TimeZone;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class NewDateUtils {
+
+ public static final String FULL_FORMAT = "yyyyMMddHHmmss";
+ public static final String NULL_DATA_TIME = "000000000000";
+ public static final String DEFAULT_FORMAT = "yyyyMMddHHmm";
+ public static final String DEFAULT_TIME_ZONE = "Asia/Shanghai";
+ private static final Logger logger = LoggerFactory.getLogger(NewDateUtils.class);
+ private static final String TIME_REGEX = "YYYY(?:.MM|MM)?(?:.DD|DD)?(?:.hh|hh)?(?:.mm|mm)?(?:"
+ + ".ss|ss)?";
+ private static final String LIMIT_SEP = "(?<=[a-zA-Z])";
+ private static final String LETTER_STR = "\\D+";
+ private static final String DIGIT_STR = "[0-9]+";
+ private static final Pattern pattern = Pattern.compile(TIME_REGEX,
+ Pattern.CASE_INSENSITIVE | Pattern.DOTALL | Pattern.MULTILINE);
+ private static final Pattern bracePatt = Pattern.compile("\\{(.*?)\\}");
+ private static final int DEFAULT_LENGTH = "yyyyMMddHHmm".length();
+ public static long DAY_TIMEOUT_INTERVAL = 2 * 24 * 3600 * 1000;
+ public static long HOUR_TIMEOUT_INTERVAL = 2 * 3600 * 1000;
+ // 数据源配置异常 */
+ public static final String DATA_SOURCE_CONFIG_ERROR = "ERROR-0-TDAgent|10001|ERROR"
+ + "|ERROR_DATA_SOURCE_CONFIG|";
+
+ /* Return the time in milliseconds for a data time. */
+ /*
+ * public static long getTimeInMillis(String dataTime) { if (dataTime == null) { return 0; }
+ *
+ * try { SimpleDateFormat dateFormat = new SimpleDateFormat(DEFAULT_FORMAT); Date date = dateFormat.parse(dataTime);
+ *
+ * return date.getTime(); } catch (ParseException e) { return 0; } }
+ */
+ /* Return the format data time string from milliseconds. */
+ /*
+ * public static String getDataTimeFromTimeMillis(long dataTime) { SimpleDateFormat dateFormat = new
+ * SimpleDateFormat(DEFAULT_FORMAT); return dateFormat.format(new Date(dataTime)); }
+ */
+ /* Return the should start time for a data time log file. */
+ public static String getShouldStartTime(String dataTime, String cycleUnit,
+ String offset) {
+ if (dataTime == null || dataTime.length() > 12) {
+ return null;
+ }
+
+ SimpleDateFormat dateFormat = new SimpleDateFormat(DEFAULT_FORMAT);
+ TimeZone timeZone = TimeZone.getTimeZone(NewDateUtils.DEFAULT_TIME_ZONE);
+ dateFormat.setTimeZone(timeZone);
+
+ if (dataTime.length() < DEFAULT_LENGTH) {
+ StringBuffer sb = new StringBuffer();
+ for (int i = 0; i < DEFAULT_LENGTH - dataTime.length(); i++) {
+ sb.append("0");
+ }
+ dataTime = dataTime + sb.toString();
+ }
+
+ Calendar calendar = Calendar.getInstance();
+ try {
+ calendar.setTimeInMillis(dateFormat.parse(dataTime).getTime());
+ } catch (ParseException e) {
+ return null;
+ }
+
+ /*
+ * The delay should be added to the data time, so remove the - from offset.
+ */
+ if (offset.startsWith("-")) {
+ offset = offset.substring(1, offset.length());
+ } else { // 为正，配置提前读取文件
+ offset = "-" + offset;
+ }
+
+ return dateFormat
+ .format(new Date(getDateTime(calendar, cycleUnit, offset).getTimeInMillis()));
+ }
+
+ private static Calendar getCurDate(String cycleUnit, String offset) {
+ if (cycleUnit == null || cycleUnit.length() == 0) {
+ return null;
+ }
+
+ Calendar calendar = Calendar.getInstance();
+ calendar.setTimeInMillis(System.currentTimeMillis());
+
+ return getDateTime(calendar, cycleUnit, offset);
+ }
+
+ public static String getDateTime(String dataTime, String cycleUnit, String offset) {
+ String retTime = NewDateUtils.millSecConvertToTimeStr(
+ System.currentTimeMillis(), cycleUnit);
+ try {
+ long time = NewDateUtils.timeStrConvertTomillSec(dataTime, cycleUnit);
+
+ Calendar calendar = Calendar.getInstance();
+ calendar.setTimeInMillis(time);
+ Calendar retCalendar = getDateTime(calendar, cycleUnit, offset);
+ if (retCalendar == null) {
+ return dataTime;
+ }
+
+ retTime = NewDateUtils.millSecConvertToTimeStr(retCalendar.getTime().getTime(),
+ cycleUnit);
+ } catch (Exception e) {
+ logger.error("getDateTime error: ", e);
+ }
+ return retTime;
+ }
+
+ public static String getDateTime(long time, String cycleUnit, String offset) {
+ // String retTime = NewDateUtils.millSecConvertToTimeStr(System.currentTimeMillis(),
+ // cycleUnit);
+
+ Calendar calendar = Calendar.getInstance();
+ calendar.setTimeInMillis(time);
+ Calendar retCalendar = getDateTime(calendar, cycleUnit, offset);
+ return NewDateUtils.millSecConvertToTimeStr(retCalendar.getTime().getTime(), cycleUnit);
+
+ // return retTime;
Review Comment:
ditto
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@inlong.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] [INLONG-9134][Agent] Add file related utils [inlong]
Posted by "doleyzi (via GitHub)" <gi...@apache.org>.
doleyzi commented on code in PR #9135:
URL: https://github.com/apache/inlong/pull/9135#discussion_r1374029015
##########
inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/utils/file/MatchPoint.java:
##########
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.agent.plugin.utils.file;
+
+/**
+ * Created by lamberliu on 2015/11/13.
Review Comment:
The information here needs to be updated
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@inlong.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] [INLONG-9134][Agent] Add file related utils [inlong]
Posted by "justinwwhuang (via GitHub)" <gi...@apache.org>.
justinwwhuang commented on code in PR #9135:
URL: https://github.com/apache/inlong/pull/9135#discussion_r1374036529
##########
inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/utils/file/FilePathUtil.java:
##########
@@ -0,0 +1,170 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.agent.plugin.utils.file;
+
+import org.apache.commons.lang.StringUtils;
+
+import java.io.File;
+import java.util.ArrayList;
+
+public class FilePathUtil {
+
+ private static final String YEAR = "YYYY";
+ private static final String MONTH = "MM";
+ private static final String DAY = "DD";
+ private static final String HOUR = "hh";
+
+ public static ArrayList<String> cutDirectory(String directory) {
+ String baseDirectory;
+ String regixDirecotry;
+ String fileName;
+
+ File file = new File(directory);
+ fileName = file.getName();
+
+ int fileNameIndex = directory.length() - fileName.length() - 1;
+ String sign = "\\^$*+?{(|[.";
+ String range = "+?*{";
+
+ int regixDirecotryIndex = StringUtils.indexOfAny(directory, sign);
+ if (regixDirecotryIndex != -1
+ && directory.charAt(regixDirecotryIndex) == '.') {
+ if (regixDirecotryIndex != directory.length() - 1) {
+ char c = directory.charAt(regixDirecotryIndex + 1);
+ if (StringUtils.indexOf(range, c) == -1) {
+ regixDirecotryIndex = StringUtils.indexOfAny(directory,
+ sign.substring(0, sign.length() - 1));
+ }
+ }
+ }
+ if (regixDirecotryIndex < fileNameIndex) {
+ int regixDirecotryBeginIndex = directory.lastIndexOf('/',
+ regixDirecotryIndex);
+ if (regixDirecotryBeginIndex == -1) {
+ baseDirectory = directory.substring(0, fileNameIndex);
+ regixDirecotry = "";
+ } else {
+
+ regixDirecotry = directory.substring(
+ regixDirecotryBeginIndex + 1, fileNameIndex);
+ if (regixDirecotryBeginIndex == 0) {
+ baseDirectory = "/";
+ } else {
+ baseDirectory = directory.substring(0,
+ regixDirecotryBeginIndex);
+ }
+ }
+ } else {
+ baseDirectory = directory.substring(0, fileNameIndex);
+ regixDirecotry = "";
+ }
+
+ ArrayList<String> ret = new ArrayList<String>();
+ ret.add(baseDirectory);
+ ret.add(regixDirecotry);
+ ret.add(fileName);
+ return ret;
+ }
+
+ public static ArrayList<String> getDirectoryLayers(String directory) {
+ String baseDirectory;
+ String regixDirectory;
+ String fileName;
+
+ File file = new File(directory);
+ fileName = file.getName();
+
+ int fileNameIndex = directory.length() - fileName.length() - 1;
+ String sign = "\\^$*+?{(|[.";
+
+ String range = "+?*{";
+
+ int regixDirecotryIndex = StringUtils.indexOfAny(directory, sign);
+ if (regixDirecotryIndex != -1
+ && directory.charAt(regixDirecotryIndex) == '.') {
+ if (regixDirecotryIndex != directory.length() - 1) {
+ char c = directory.charAt(regixDirecotryIndex + 1);
+ if (StringUtils.indexOf(range, c) == -1) {
+ regixDirecotryIndex = StringUtils.indexOfAny(directory,
+ sign.substring(0, sign.length() - 1));
+ }
+ }
+ }
+ if (regixDirecotryIndex < fileNameIndex) {
+ int regixDirecotryBeginIndex = directory.lastIndexOf('/',
+ regixDirecotryIndex);
+ if (regixDirecotryBeginIndex == -1) {
+ baseDirectory = directory.substring(0, fileNameIndex);
+ regixDirectory = "";
+ } else {
+ regixDirectory = directory.substring(
+ regixDirecotryBeginIndex + 1, fileNameIndex);
+ if (regixDirecotryBeginIndex == 0) {
+ baseDirectory = "/";
+ } else {
+ baseDirectory = directory.substring(0,
+ regixDirecotryBeginIndex);
+ }
+ }
+ } else {
+ baseDirectory = directory.substring(0, fileNameIndex);
+ regixDirectory = "";
+ }
+
+ int[] indexes = new int[]{
+ (baseDirectory.contains(YEAR) ? baseDirectory.indexOf(YEAR) : Integer.MAX_VALUE),
+ (baseDirectory.contains(MONTH) ? baseDirectory.indexOf(MONTH) : Integer.MAX_VALUE),
+ (baseDirectory.contains(DAY) ? baseDirectory.indexOf(DAY) : Integer.MAX_VALUE),
+ (baseDirectory.contains(HOUR) ? baseDirectory.indexOf(HOUR) : Integer.MAX_VALUE)};
+
+ int minIndex = Integer.MAX_VALUE;
+ for (int i = 0; i < indexes.length; i++) {
+ if (minIndex > indexes[i]) {
+ minIndex = indexes[i];
+ }
+ }
+
+ if (minIndex != Integer.MAX_VALUE) {
+ int lastIndex = baseDirectory.lastIndexOf('/', minIndex);
+ if (regixDirectory.length() > 0) {
+ regixDirectory = baseDirectory.substring(lastIndex + 1, baseDirectory.length())
+ + File.separator + regixDirectory;
+ } else {
+ regixDirectory = baseDirectory.substring(lastIndex + 1, baseDirectory.length());
+ }
+ baseDirectory = baseDirectory.substring(0, lastIndex);
+ }
+
+ ArrayList<String> ret = new ArrayList<String>();
+ ret.add(baseDirectory);
+ ret.add(regixDirectory);
+ ret.add(fileName);
+ return ret;
+ }
+
+ public static boolean isSameDir(String fileName1, String fileName2) {
+ ArrayList<String> ret1 = FilePathUtil.cutDirectory(fileName1);
+ ArrayList<String> ret2 = FilePathUtil.cutDirectory(fileName2);
+ if (ret1.get(0).equals(ret2.get(0))) {
Review Comment:
ok
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@inlong.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] [INLONG-9134][Agent] Add file related utils [inlong]
Posted by "gosonzhang (via GitHub)" <gi...@apache.org>.
gosonzhang merged PR #9135:
URL: https://github.com/apache/inlong/pull/9135
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@inlong.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] [INLONG-9134][Agent] Add file related utils [inlong]
Posted by "justinwwhuang (via GitHub)" <gi...@apache.org>.
justinwwhuang commented on code in PR #9135:
URL: https://github.com/apache/inlong/pull/9135#discussion_r1374056506
##########
inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/utils/file/NewDateUtils.java:
##########
@@ -0,0 +1,955 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.agent.plugin.utils.file;
+
+import hirondelle.date4j.DateTime;
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Calendar;
+import java.util.Date;
+import java.util.List;
+import java.util.StringTokenizer;
+import java.util.TimeZone;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class NewDateUtils {
+
+ public static final String FULL_FORMAT = "yyyyMMddHHmmss";
+ public static final String NULL_DATA_TIME = "000000000000";
+ public static final String DEFAULT_FORMAT = "yyyyMMddHHmm";
+ public static final String DEFAULT_TIME_ZONE = "Asia/Shanghai";
+ private static final Logger logger = LoggerFactory.getLogger(NewDateUtils.class);
+ private static final String TIME_REGEX = "YYYY(?:.MM|MM)?(?:.DD|DD)?(?:.hh|hh)?(?:.mm|mm)?(?:"
+ + ".ss|ss)?";
+ private static final String LIMIT_SEP = "(?<=[a-zA-Z])";
+ private static final String LETTER_STR = "\\D+";
+ private static final String DIGIT_STR = "[0-9]+";
+ private static final Pattern pattern = Pattern.compile(TIME_REGEX,
+ Pattern.CASE_INSENSITIVE | Pattern.DOTALL | Pattern.MULTILINE);
+ private static final Pattern bracePatt = Pattern.compile("\\{(.*?)\\}");
+ private static final int DEFAULT_LENGTH = "yyyyMMddHHmm".length();
+ public static long DAY_TIMEOUT_INTERVAL = 2 * 24 * 3600 * 1000;
+ public static long HOUR_TIMEOUT_INTERVAL = 2 * 3600 * 1000;
+ // 数据源配置异常 */
+ public static final String DATA_SOURCE_CONFIG_ERROR = "ERROR-0-TDAgent|10001|ERROR"
+ + "|ERROR_DATA_SOURCE_CONFIG|";
+
+ /* Return the time in milliseconds for a data time. */
+ /*
+ * public static long getTimeInMillis(String dataTime) { if (dataTime == null) { return 0; }
+ *
+ * try { SimpleDateFormat dateFormat = new SimpleDateFormat(DEFAULT_FORMAT); Date date = dateFormat.parse(dataTime);
+ *
+ * return date.getTime(); } catch (ParseException e) { return 0; } }
+ */
+ /* Return the format data time string from milliseconds. */
+ /*
+ * public static String getDataTimeFromTimeMillis(long dataTime) { SimpleDateFormat dateFormat = new
+ * SimpleDateFormat(DEFAULT_FORMAT); return dateFormat.format(new Date(dataTime)); }
+ */
+ /* Return the should start time for a data time log file. */
+ public static String getShouldStartTime(String dataTime, String cycleUnit,
+ String offset) {
+ if (dataTime == null || dataTime.length() > 12) {
+ return null;
+ }
+
+ SimpleDateFormat dateFormat = new SimpleDateFormat(DEFAULT_FORMAT);
+ TimeZone timeZone = TimeZone.getTimeZone(NewDateUtils.DEFAULT_TIME_ZONE);
+ dateFormat.setTimeZone(timeZone);
+
+ if (dataTime.length() < DEFAULT_LENGTH) {
+ StringBuffer sb = new StringBuffer();
+ for (int i = 0; i < DEFAULT_LENGTH - dataTime.length(); i++) {
+ sb.append("0");
+ }
+ dataTime = dataTime + sb.toString();
+ }
+
+ Calendar calendar = Calendar.getInstance();
+ try {
+ calendar.setTimeInMillis(dateFormat.parse(dataTime).getTime());
+ } catch (ParseException e) {
+ return null;
+ }
+
+ /*
+ * The delay should be added to the data time, so remove the - from offset.
+ */
+ if (offset.startsWith("-")) {
+ offset = offset.substring(1, offset.length());
+ } else { // 为正，配置提前读取文件
+ offset = "-" + offset;
+ }
+
+ return dateFormat
+ .format(new Date(getDateTime(calendar, cycleUnit, offset).getTimeInMillis()));
+ }
+
+ private static Calendar getCurDate(String cycleUnit, String offset) {
+ if (cycleUnit == null || cycleUnit.length() == 0) {
+ return null;
+ }
+
+ Calendar calendar = Calendar.getInstance();
+ calendar.setTimeInMillis(System.currentTimeMillis());
+
+ return getDateTime(calendar, cycleUnit, offset);
+ }
+
+ public static String getDateTime(String dataTime, String cycleUnit, String offset) {
+ String retTime = NewDateUtils.millSecConvertToTimeStr(
+ System.currentTimeMillis(), cycleUnit);
+ try {
+ long time = NewDateUtils.timeStrConvertTomillSec(dataTime, cycleUnit);
+
+ Calendar calendar = Calendar.getInstance();
+ calendar.setTimeInMillis(time);
+ Calendar retCalendar = getDateTime(calendar, cycleUnit, offset);
+ if (retCalendar == null) {
+ return dataTime;
+ }
+
+ retTime = NewDateUtils.millSecConvertToTimeStr(retCalendar.getTime().getTime(),
+ cycleUnit);
+ } catch (Exception e) {
+ logger.error("getDateTime error: ", e);
+ }
+ return retTime;
+ }
+
+ public static String getDateTime(long time, String cycleUnit, String offset) {
+ // String retTime = NewDateUtils.millSecConvertToTimeStr(System.currentTimeMillis(),
+ // cycleUnit);
+
+ Calendar calendar = Calendar.getInstance();
+ calendar.setTimeInMillis(time);
+ Calendar retCalendar = getDateTime(calendar, cycleUnit, offset);
+ return NewDateUtils.millSecConvertToTimeStr(retCalendar.getTime().getTime(), cycleUnit);
+
+ // return retTime;
+ }
+
+ private static Calendar getDateTime(Calendar calendar, String cycleUnit, String offset) {
+ int cycleNumber = (cycleUnit.length() <= 1
+ ? 1
+ : Integer.parseInt(cycleUnit.substring(0, cycleUnit.length() - 1)));
+
+ String offsetUnit = offset.substring(offset.length() - 1, offset.length());
+ int offsetNumber = Integer.parseInt(offset.substring(0, offset.length() - 1));
+
+ /*
+ * For day task, the offset cycle unit can only be day; for hourly task, the offset can't be minute; for
+ * minutely task, the offset cycle unit can be day, hour and minute, but if the offset cycle unit is minute, the
+ * offset must be divided by cycle number.
+ */
+ // if (("D".equalsIgnoreCase(cycleUnit) && !"D".equalsIgnoreCase(offsetUnit))) {
Review Comment:
get
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@inlong.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] [INLONG-9134][Agent] Add file related utils [inlong]
Posted by "justinwwhuang (via GitHub)" <gi...@apache.org>.
justinwwhuang commented on code in PR #9135:
URL: https://github.com/apache/inlong/pull/9135#discussion_r1374042164
##########
inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/utils/file/NewDateUtils.java:
##########
@@ -0,0 +1,955 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.agent.plugin.utils.file;
+
+import hirondelle.date4j.DateTime;
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Calendar;
+import java.util.Date;
+import java.util.List;
+import java.util.StringTokenizer;
+import java.util.TimeZone;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class NewDateUtils {
+
+ public static final String FULL_FORMAT = "yyyyMMddHHmmss";
+ public static final String NULL_DATA_TIME = "000000000000";
+ public static final String DEFAULT_FORMAT = "yyyyMMddHHmm";
+ public static final String DEFAULT_TIME_ZONE = "Asia/Shanghai";
+ private static final Logger logger = LoggerFactory.getLogger(NewDateUtils.class);
+ private static final String TIME_REGEX = "YYYY(?:.MM|MM)?(?:.DD|DD)?(?:.hh|hh)?(?:.mm|mm)?(?:"
+ + ".ss|ss)?";
+ private static final String LIMIT_SEP = "(?<=[a-zA-Z])";
+ private static final String LETTER_STR = "\\D+";
+ private static final String DIGIT_STR = "[0-9]+";
+ private static final Pattern pattern = Pattern.compile(TIME_REGEX,
+ Pattern.CASE_INSENSITIVE | Pattern.DOTALL | Pattern.MULTILINE);
+ private static final Pattern bracePatt = Pattern.compile("\\{(.*?)\\}");
+ private static final int DEFAULT_LENGTH = "yyyyMMddHHmm".length();
+ public static long DAY_TIMEOUT_INTERVAL = 2 * 24 * 3600 * 1000;
+ public static long HOUR_TIMEOUT_INTERVAL = 2 * 3600 * 1000;
+ // 数据源配置异常 */
+ public static final String DATA_SOURCE_CONFIG_ERROR = "ERROR-0-TDAgent|10001|ERROR"
+ + "|ERROR_DATA_SOURCE_CONFIG|";
+
+ /* Return the time in milliseconds for a data time. */
+ /*
+ * public static long getTimeInMillis(String dataTime) { if (dataTime == null) { return 0; }
+ *
+ * try { SimpleDateFormat dateFormat = new SimpleDateFormat(DEFAULT_FORMAT); Date date = dateFormat.parse(dataTime);
+ *
+ * return date.getTime(); } catch (ParseException e) { return 0; } }
+ */
+ /* Return the format data time string from milliseconds. */
+ /*
+ * public static String getDataTimeFromTimeMillis(long dataTime) { SimpleDateFormat dateFormat = new
+ * SimpleDateFormat(DEFAULT_FORMAT); return dateFormat.format(new Date(dataTime)); }
+ */
+ /* Return the should start time for a data time log file. */
+ public static String getShouldStartTime(String dataTime, String cycleUnit,
+ String offset) {
+ if (dataTime == null || dataTime.length() > 12) {
+ return null;
+ }
+
+ SimpleDateFormat dateFormat = new SimpleDateFormat(DEFAULT_FORMAT);
+ TimeZone timeZone = TimeZone.getTimeZone(NewDateUtils.DEFAULT_TIME_ZONE);
+ dateFormat.setTimeZone(timeZone);
+
+ if (dataTime.length() < DEFAULT_LENGTH) {
+ StringBuffer sb = new StringBuffer();
+ for (int i = 0; i < DEFAULT_LENGTH - dataTime.length(); i++) {
+ sb.append("0");
+ }
+ dataTime = dataTime + sb.toString();
+ }
+
+ Calendar calendar = Calendar.getInstance();
+ try {
+ calendar.setTimeInMillis(dateFormat.parse(dataTime).getTime());
+ } catch (ParseException e) {
+ return null;
+ }
+
+ /*
+ * The delay should be added to the data time, so remove the - from offset.
+ */
+ if (offset.startsWith("-")) {
+ offset = offset.substring(1, offset.length());
+ } else { // 为正，配置提前读取文件
Review Comment:
get
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@inlong.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] [INLONG-9134][Agent] Add file related utils [inlong]
Posted by "justinwwhuang (via GitHub)" <gi...@apache.org>.
justinwwhuang commented on code in PR #9135:
URL: https://github.com/apache/inlong/pull/9135#discussion_r1374041518
##########
inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/utils/file/Files.java:
##########
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.agent.plugin.utils.file;
+
+import org.apache.inlong.agent.utils.file.FileFinder;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Paths;
+import java.nio.file.attribute.BasicFileAttributes;
+
+public class Files {
+
+ /**
+ * Finds files or sub directories within a given base directory.
+ *
+ * @param baseDirectory A path string representing a directory to search within.
+ * @return A {@link org.apache.inlong.agent.plugin.utils.FileFinder}-Object to specify the search
+ * parameters using a builder pattern.
+ */
+ public static FileFinder find(String baseDirectory) {
+ return find(new File(baseDirectory));
+ }
+
+ /**
+ * Finds files or sub directories within a given base directory.
+ *
+ * @param baseDirectory A path as {@link File} object to search within.
+ * @return A {@link org.apache.inlong.agent.plugin.utils.FileFinder} object to specify the search
+ * parameters using a builder pattern.
+ */
+ public static FileFinder find(File baseDirectory) {
+ return new FileFinder(baseDirectory);
+ }
+
+ public static long getFileCreationTime(String fileName) {
+ long createTime = 0L;
+ try {
+ createTime = java.nio.file.Files.readAttributes(Paths.get(fileName),
+ BasicFileAttributes.class).creationTime().toMillis();
+ } catch (IOException ignore) {
+
+ }
+ return createTime;
+ }
+
+ public static long getFileLastModifyTime(String fileName) {
+ long lastModify = 0L;
+ try {
+ lastModify = java.nio.file.Files.getLastModifiedTime(Paths.get(fileName)).toMillis();
+ } catch (IOException ioe) {
Review Comment:
never use, delete it
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@inlong.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] [INLONG-9134][Agent] Add file related utils [inlong]
Posted by "justinwwhuang (via GitHub)" <gi...@apache.org>.
justinwwhuang commented on code in PR #9135:
URL: https://github.com/apache/inlong/pull/9135#discussion_r1374056080
##########
inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/utils/file/NewDateUtils.java:
##########
@@ -0,0 +1,955 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.agent.plugin.utils.file;
+
+import hirondelle.date4j.DateTime;
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Calendar;
+import java.util.Date;
+import java.util.List;
+import java.util.StringTokenizer;
+import java.util.TimeZone;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class NewDateUtils {
+
+ public static final String FULL_FORMAT = "yyyyMMddHHmmss";
+ public static final String NULL_DATA_TIME = "000000000000";
+ public static final String DEFAULT_FORMAT = "yyyyMMddHHmm";
+ public static final String DEFAULT_TIME_ZONE = "Asia/Shanghai";
+ private static final Logger logger = LoggerFactory.getLogger(NewDateUtils.class);
+ private static final String TIME_REGEX = "YYYY(?:.MM|MM)?(?:.DD|DD)?(?:.hh|hh)?(?:.mm|mm)?(?:"
+ + ".ss|ss)?";
+ private static final String LIMIT_SEP = "(?<=[a-zA-Z])";
+ private static final String LETTER_STR = "\\D+";
+ private static final String DIGIT_STR = "[0-9]+";
+ private static final Pattern pattern = Pattern.compile(TIME_REGEX,
+ Pattern.CASE_INSENSITIVE | Pattern.DOTALL | Pattern.MULTILINE);
+ private static final Pattern bracePatt = Pattern.compile("\\{(.*?)\\}");
+ private static final int DEFAULT_LENGTH = "yyyyMMddHHmm".length();
+ public static long DAY_TIMEOUT_INTERVAL = 2 * 24 * 3600 * 1000;
+ public static long HOUR_TIMEOUT_INTERVAL = 2 * 3600 * 1000;
+ // 数据源配置异常 */
+ public static final String DATA_SOURCE_CONFIG_ERROR = "ERROR-0-TDAgent|10001|ERROR"
+ + "|ERROR_DATA_SOURCE_CONFIG|";
+
+ /* Return the time in milliseconds for a data time. */
+ /*
+ * public static long getTimeInMillis(String dataTime) { if (dataTime == null) { return 0; }
+ *
+ * try { SimpleDateFormat dateFormat = new SimpleDateFormat(DEFAULT_FORMAT); Date date = dateFormat.parse(dataTime);
+ *
+ * return date.getTime(); } catch (ParseException e) { return 0; } }
+ */
+ /* Return the format data time string from milliseconds. */
+ /*
+ * public static String getDataTimeFromTimeMillis(long dataTime) { SimpleDateFormat dateFormat = new
+ * SimpleDateFormat(DEFAULT_FORMAT); return dateFormat.format(new Date(dataTime)); }
+ */
+ /* Return the should start time for a data time log file. */
+ public static String getShouldStartTime(String dataTime, String cycleUnit,
+ String offset) {
+ if (dataTime == null || dataTime.length() > 12) {
+ return null;
+ }
+
+ SimpleDateFormat dateFormat = new SimpleDateFormat(DEFAULT_FORMAT);
+ TimeZone timeZone = TimeZone.getTimeZone(NewDateUtils.DEFAULT_TIME_ZONE);
+ dateFormat.setTimeZone(timeZone);
+
+ if (dataTime.length() < DEFAULT_LENGTH) {
+ StringBuffer sb = new StringBuffer();
+ for (int i = 0; i < DEFAULT_LENGTH - dataTime.length(); i++) {
+ sb.append("0");
+ }
+ dataTime = dataTime + sb.toString();
+ }
+
+ Calendar calendar = Calendar.getInstance();
+ try {
+ calendar.setTimeInMillis(dateFormat.parse(dataTime).getTime());
+ } catch (ParseException e) {
+ return null;
+ }
+
+ /*
+ * The delay should be added to the data time, so remove the - from offset.
+ */
+ if (offset.startsWith("-")) {
+ offset = offset.substring(1, offset.length());
+ } else { // 为正，配置提前读取文件
+ offset = "-" + offset;
+ }
+
+ return dateFormat
+ .format(new Date(getDateTime(calendar, cycleUnit, offset).getTimeInMillis()));
+ }
+
+ private static Calendar getCurDate(String cycleUnit, String offset) {
+ if (cycleUnit == null || cycleUnit.length() == 0) {
+ return null;
+ }
+
+ Calendar calendar = Calendar.getInstance();
+ calendar.setTimeInMillis(System.currentTimeMillis());
+
+ return getDateTime(calendar, cycleUnit, offset);
+ }
+
+ public static String getDateTime(String dataTime, String cycleUnit, String offset) {
+ String retTime = NewDateUtils.millSecConvertToTimeStr(
+ System.currentTimeMillis(), cycleUnit);
+ try {
+ long time = NewDateUtils.timeStrConvertTomillSec(dataTime, cycleUnit);
+
+ Calendar calendar = Calendar.getInstance();
+ calendar.setTimeInMillis(time);
+ Calendar retCalendar = getDateTime(calendar, cycleUnit, offset);
+ if (retCalendar == null) {
+ return dataTime;
+ }
+
+ retTime = NewDateUtils.millSecConvertToTimeStr(retCalendar.getTime().getTime(),
+ cycleUnit);
+ } catch (Exception e) {
+ logger.error("getDateTime error: ", e);
+ }
+ return retTime;
+ }
+
+ public static String getDateTime(long time, String cycleUnit, String offset) {
+ // String retTime = NewDateUtils.millSecConvertToTimeStr(System.currentTimeMillis(),
Review Comment:
get
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@inlong.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] [INLONG-9134][Agent] Add file related utils [inlong]
Posted by "justinwwhuang (via GitHub)" <gi...@apache.org>.
justinwwhuang commented on code in PR #9135:
URL: https://github.com/apache/inlong/pull/9135#discussion_r1374033149
##########
inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/utils/file/MatchPoint.java:
##########
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.agent.plugin.utils.file;
+
+/**
+ * Created by lamberliu on 2015/11/13.
Review Comment:
ok
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@inlong.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] [INLONG-9134][Agent] Add file related utils [inlong]
Posted by "justinwwhuang (via GitHub)" <gi...@apache.org>.
justinwwhuang commented on code in PR #9135:
URL: https://github.com/apache/inlong/pull/9135#discussion_r1374117968
##########
inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/utils/file/NewDateUtils.java:
##########
@@ -0,0 +1,937 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.agent.plugin.utils.file;
+
+import hirondelle.date4j.DateTime;
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Calendar;
+import java.util.Date;
+import java.util.List;
+import java.util.StringTokenizer;
+import java.util.TimeZone;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class NewDateUtils {
+
+ public static final String FULL_FORMAT = "yyyyMMddHHmmss";
+ public static final String NULL_DATA_TIME = "000000000000";
+ public static final String DEFAULT_FORMAT = "yyyyMMddHHmm";
+ public static final String DEFAULT_TIME_ZONE = "Asia/Shanghai";
+ private static final Logger logger = LoggerFactory.getLogger(NewDateUtils.class);
+ private static final String TIME_REGEX = "YYYY(?:.MM|MM)?(?:.DD|DD)?(?:.hh|hh)?(?:.mm|mm)?(?:"
+ + ".ss|ss)?";
+ private static final String LIMIT_SEP = "(?<=[a-zA-Z])";
+ private static final String LETTER_STR = "\\D+";
+ private static final String DIGIT_STR = "[0-9]+";
+ private static final Pattern pattern = Pattern.compile(TIME_REGEX,
+ Pattern.CASE_INSENSITIVE | Pattern.DOTALL | Pattern.MULTILINE);
+ private static final Pattern bracePatt = Pattern.compile("\\{(.*?)\\}");
+ private static final int DEFAULT_LENGTH = "yyyyMMddHHmm".length();
+ public static long DAY_TIMEOUT_INTERVAL = 2 * 24 * 3600 * 1000;
+ public static long HOUR_TIMEOUT_INTERVAL = 2 * 3600 * 1000;
+ // data source config error */
+ public static final String DATA_SOURCE_CONFIG_ERROR = "ERROR-0-TDAgent|10001|ERROR"
+ + "|ERROR_DATA_SOURCE_CONFIG|";
+
+ /* Return the time in milliseconds for a data time. */
+ /*
+ * public static long getTimeInMillis(String dataTime) { if (dataTime == null) { return 0; }
+ *
+ * try { SimpleDateFormat dateFormat = new SimpleDateFormat(DEFAULT_FORMAT); Date date = dateFormat.parse(dataTime);
+ *
+ * return date.getTime(); } catch (ParseException e) { return 0; } }
+ */
+ /* Return the format data time string from milliseconds. */
+ /*
+ * public static String getDataTimeFromTimeMillis(long dataTime) { SimpleDateFormat dateFormat = new
+ * SimpleDateFormat(DEFAULT_FORMAT); return dateFormat.format(new Date(dataTime)); }
+ */
+ /* Return the should start time for a data time log file. */
+ public static String getShouldStartTime(String dataTime, String cycleUnit,
+ String offset) {
+ if (dataTime == null || dataTime.length() > 12) {
+ return null;
+ }
+
+ SimpleDateFormat dateFormat = new SimpleDateFormat(DEFAULT_FORMAT);
+ TimeZone timeZone = TimeZone.getTimeZone(NewDateUtils.DEFAULT_TIME_ZONE);
+ dateFormat.setTimeZone(timeZone);
+
+ if (dataTime.length() < DEFAULT_LENGTH) {
+ StringBuffer sb = new StringBuffer();
+ for (int i = 0; i < DEFAULT_LENGTH - dataTime.length(); i++) {
+ sb.append("0");
+ }
+ dataTime = dataTime + sb.toString();
+ }
+
+ Calendar calendar = Calendar.getInstance();
+ try {
+ calendar.setTimeInMillis(dateFormat.parse(dataTime).getTime());
+ } catch (ParseException e) {
+ return null;
+ }
+
+ /*
+ * The delay should be added to the data time, so remove the - from offset.
+ */
+ if (offset.startsWith("-")) {
+ offset = offset.substring(1, offset.length());
+ } else { // positive，read file earlier
+ offset = "-" + offset;
+ }
+
+ return dateFormat
+ .format(new Date(getDateTime(calendar, cycleUnit, offset).getTimeInMillis()));
+ }
+
+ private static Calendar getCurDate(String cycleUnit, String offset) {
+ if (cycleUnit == null || cycleUnit.length() == 0) {
+ return null;
+ }
+
+ Calendar calendar = Calendar.getInstance();
+ calendar.setTimeInMillis(System.currentTimeMillis());
+
+ return getDateTime(calendar, cycleUnit, offset);
+ }
+
+ public static String getDateTime(String dataTime, String cycleUnit, String offset) {
+ String retTime = NewDateUtils.millSecConvertToTimeStr(
+ System.currentTimeMillis(), cycleUnit);
+ try {
+ long time = NewDateUtils.timeStrConvertTomillSec(dataTime, cycleUnit);
+
+ Calendar calendar = Calendar.getInstance();
+ calendar.setTimeInMillis(time);
+ Calendar retCalendar = getDateTime(calendar, cycleUnit, offset);
+ if (retCalendar == null) {
+ return dataTime;
+ }
+
+ retTime = NewDateUtils.millSecConvertToTimeStr(retCalendar.getTime().getTime(),
+ cycleUnit);
+ } catch (Exception e) {
+ logger.error("getDateTime error: ", e);
+ }
+ return retTime;
+ }
+
+ public static String getDateTime(long time, String cycleUnit, String offset) {
+ Calendar calendar = Calendar.getInstance();
+ calendar.setTimeInMillis(time);
+ Calendar retCalendar = getDateTime(calendar, cycleUnit, offset);
+ return NewDateUtils.millSecConvertToTimeStr(retCalendar.getTime().getTime(), cycleUnit);
+ }
+
+ private static Calendar getDateTime(Calendar calendar, String cycleUnit, String offset) {
+ int cycleNumber = (cycleUnit.length() <= 1
+ ? 1
+ : Integer.parseInt(cycleUnit.substring(0, cycleUnit.length() - 1)));
+
+ String offsetUnit = offset.substring(offset.length() - 1, offset.length());
+ int offsetNumber = Integer.parseInt(offset.substring(0, offset.length() - 1));
+
+ /*
+ * For day task, the offset cycle unit can only be day; for hourly task, the offset can't be minute; for
+ * minutely task, the offset cycle unit can be day, hour and minute, but if the offset cycle unit is minute, the
+ * offset must be divided by cycle number.
+ */
+ if (cycleUnit.length() > 1 && (StringUtils.endsWithIgnoreCase(cycleUnit, "M"))) {
+ calendar.set(Calendar.SECOND, 0);
+ int minTime = calendar.get(Calendar.MINUTE);
+
+ int leftMin = minTime % cycleNumber;
+ minTime = minTime - leftMin;
+ calendar.set(Calendar.MINUTE, minTime);
+
+ /* Calculate the offset. */
+ if ("D".equalsIgnoreCase(offsetUnit)) {
+ calendar.add(Calendar.DAY_OF_YEAR, offsetNumber);
+ }
+
+ if ("H".equalsIgnoreCase(offsetUnit)) {
+ calendar.add(Calendar.HOUR_OF_DAY, offsetNumber);
+ }
+ } else if (cycleUnit.length() == 1) {
+ if ("D".equalsIgnoreCase(cycleUnit)) {
+ calendar.set(Calendar.HOUR_OF_DAY, 0);
+ calendar.set(Calendar.MINUTE, 0);
+ calendar.set(Calendar.SECOND, 0);
+ } else if ("h".equalsIgnoreCase(cycleUnit)) {
+ calendar.set(Calendar.MINUTE, 0);
+ calendar.set(Calendar.SECOND, 0);
+ }
+ }
+
+ /* Calculate the offset. */
+ if ("D".equalsIgnoreCase(offsetUnit)) {
+ calendar.add(Calendar.DAY_OF_YEAR, offsetNumber);
+ }
+
+ if ("h".equalsIgnoreCase(offsetUnit)) {
+ calendar.add(Calendar.HOUR_OF_DAY, offsetNumber);
+ }
+
+ if ("m".equals(offsetUnit)) {
+ calendar.add(Calendar.MINUTE, offsetNumber);
+ }
+
+ return calendar;
+ }
+
+ public static boolean isValidCreationTime(String dataTime, String cycleUnit,
+ String timeOffset) {
+ long timeInterval = 0;
+ if ("Y".equalsIgnoreCase(cycleUnit)) {
+ timeInterval = DAY_TIMEOUT_INTERVAL;
+ } else if ("M".equals(cycleUnit)) {
+ timeInterval = HOUR_TIMEOUT_INTERVAL;
+ } else if ("D".equalsIgnoreCase(cycleUnit)) {
+ timeInterval = DAY_TIMEOUT_INTERVAL;
+ } else if ("h".equalsIgnoreCase(cycleUnit)) {
+ timeInterval = HOUR_TIMEOUT_INTERVAL;
+ } else if (cycleUnit.contains("m")) {
+ timeInterval = HOUR_TIMEOUT_INTERVAL;
+ } else {
+ logger.error("cycleUnit {} can't parse!", cycleUnit);
+ timeInterval = DAY_TIMEOUT_INTERVAL;
+ }
+
+ // 处理偏移量，超时周期要加上时间偏移偏移量
+ if (timeOffset.startsWith("-")) {
+ timeInterval += caclOffset(timeOffset);
+ } else { // 处理向后偏移
+ timeInterval -= caclOffset(timeOffset);
+ }
+
+ return isValidCreationTime(dataTime, timeInterval);
+ }
+
+ /**
+ * 根据偏移量计算偏移时间
Review Comment:
fix it in next issue
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@inlong.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] [INLONG-9134][Agent] Add file related utils [inlong]
Posted by "justinwwhuang (via GitHub)" <gi...@apache.org>.
justinwwhuang commented on code in PR #9135:
URL: https://github.com/apache/inlong/pull/9135#discussion_r1374041612
##########
inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/utils/file/NewDateUtils.java:
##########
@@ -0,0 +1,955 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.agent.plugin.utils.file;
+
+import hirondelle.date4j.DateTime;
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Calendar;
+import java.util.Date;
+import java.util.List;
+import java.util.StringTokenizer;
+import java.util.TimeZone;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class NewDateUtils {
+
+ public static final String FULL_FORMAT = "yyyyMMddHHmmss";
+ public static final String NULL_DATA_TIME = "000000000000";
+ public static final String DEFAULT_FORMAT = "yyyyMMddHHmm";
+ public static final String DEFAULT_TIME_ZONE = "Asia/Shanghai";
+ private static final Logger logger = LoggerFactory.getLogger(NewDateUtils.class);
+ private static final String TIME_REGEX = "YYYY(?:.MM|MM)?(?:.DD|DD)?(?:.hh|hh)?(?:.mm|mm)?(?:"
+ + ".ss|ss)?";
+ private static final String LIMIT_SEP = "(?<=[a-zA-Z])";
+ private static final String LETTER_STR = "\\D+";
+ private static final String DIGIT_STR = "[0-9]+";
+ private static final Pattern pattern = Pattern.compile(TIME_REGEX,
+ Pattern.CASE_INSENSITIVE | Pattern.DOTALL | Pattern.MULTILINE);
+ private static final Pattern bracePatt = Pattern.compile("\\{(.*?)\\}");
+ private static final int DEFAULT_LENGTH = "yyyyMMddHHmm".length();
+ public static long DAY_TIMEOUT_INTERVAL = 2 * 24 * 3600 * 1000;
+ public static long HOUR_TIMEOUT_INTERVAL = 2 * 3600 * 1000;
+ // 数据源配置异常 */
Review Comment:
get
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@inlong.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] [INLONG-9134][Agent] Add file related utils [inlong]
Posted by "EMsnap (via GitHub)" <gi...@apache.org>.
EMsnap commented on code in PR #9135:
URL: https://github.com/apache/inlong/pull/9135#discussion_r1374035999
##########
inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/utils/file/MatchPoint.java:
##########
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.agent.plugin.utils.file;
+
+public class MatchPoint {
+
+ String str;
+ int start;
+ int end;
+
+ MatchPoint(String str1, int start1, int end1) {
Review Comment:
please don't use number in parameter name
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@inlong.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] [INLONG-9134][Agent] Add file related utils [inlong]
Posted by "EMsnap (via GitHub)" <gi...@apache.org>.
EMsnap commented on code in PR #9135:
URL: https://github.com/apache/inlong/pull/9135#discussion_r1374035999
##########
inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/utils/file/MatchPoint.java:
##########
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.agent.plugin.utils.file;
+
+public class MatchPoint {
+
+ String str;
+ int start;
+ int end;
+
+ MatchPoint(String str1, int start1, int end1) {
Review Comment:
please don't use number in parameter name
##########
inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/utils/file/FilePathUtil.java:
##########
@@ -0,0 +1,170 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.agent.plugin.utils.file;
+
+import org.apache.commons.lang.StringUtils;
+
+import java.io.File;
+import java.util.ArrayList;
+
+public class FilePathUtil {
+
+ private static final String YEAR = "YYYY";
+ private static final String MONTH = "MM";
+ private static final String DAY = "DD";
+ private static final String HOUR = "hh";
+
+ public static ArrayList<String> cutDirectory(String directory) {
+ String baseDirectory;
+ String regixDirecotry;
+ String fileName;
+
+ File file = new File(directory);
+ fileName = file.getName();
+
+ int fileNameIndex = directory.length() - fileName.length() - 1;
+ String sign = "\\^$*+?{(|[.";
+ String range = "+?*{";
+
+ int regixDirecotryIndex = StringUtils.indexOfAny(directory, sign);
+ if (regixDirecotryIndex != -1
+ && directory.charAt(regixDirecotryIndex) == '.') {
+ if (regixDirecotryIndex != directory.length() - 1) {
+ char c = directory.charAt(regixDirecotryIndex + 1);
+ if (StringUtils.indexOf(range, c) == -1) {
+ regixDirecotryIndex = StringUtils.indexOfAny(directory,
+ sign.substring(0, sign.length() - 1));
+ }
+ }
+ }
+ if (regixDirecotryIndex < fileNameIndex) {
+ int regixDirecotryBeginIndex = directory.lastIndexOf('/',
+ regixDirecotryIndex);
+ if (regixDirecotryBeginIndex == -1) {
+ baseDirectory = directory.substring(0, fileNameIndex);
+ regixDirecotry = "";
+ } else {
+
+ regixDirecotry = directory.substring(
+ regixDirecotryBeginIndex + 1, fileNameIndex);
+ if (regixDirecotryBeginIndex == 0) {
+ baseDirectory = "/";
+ } else {
+ baseDirectory = directory.substring(0,
+ regixDirecotryBeginIndex);
+ }
+ }
+ } else {
+ baseDirectory = directory.substring(0, fileNameIndex);
+ regixDirecotry = "";
+ }
+
+ ArrayList<String> ret = new ArrayList<String>();
+ ret.add(baseDirectory);
+ ret.add(regixDirecotry);
+ ret.add(fileName);
+ return ret;
+ }
+
+ public static ArrayList<String> getDirectoryLayers(String directory) {
+ String baseDirectory;
+ String regixDirectory;
+ String fileName;
+
+ File file = new File(directory);
+ fileName = file.getName();
+
+ int fileNameIndex = directory.length() - fileName.length() - 1;
+ String sign = "\\^$*+?{(|[.";
+
+ String range = "+?*{";
+
+ int regixDirecotryIndex = StringUtils.indexOfAny(directory, sign);
+ if (regixDirecotryIndex != -1
+ && directory.charAt(regixDirecotryIndex) == '.') {
+ if (regixDirecotryIndex != directory.length() - 1) {
+ char c = directory.charAt(regixDirecotryIndex + 1);
+ if (StringUtils.indexOf(range, c) == -1) {
+ regixDirecotryIndex = StringUtils.indexOfAny(directory,
+ sign.substring(0, sign.length() - 1));
+ }
+ }
+ }
+ if (regixDirecotryIndex < fileNameIndex) {
+ int regixDirecotryBeginIndex = directory.lastIndexOf('/',
+ regixDirecotryIndex);
+ if (regixDirecotryBeginIndex == -1) {
+ baseDirectory = directory.substring(0, fileNameIndex);
+ regixDirectory = "";
+ } else {
+ regixDirectory = directory.substring(
+ regixDirecotryBeginIndex + 1, fileNameIndex);
+ if (regixDirecotryBeginIndex == 0) {
+ baseDirectory = "/";
+ } else {
+ baseDirectory = directory.substring(0,
+ regixDirecotryBeginIndex);
+ }
+ }
+ } else {
+ baseDirectory = directory.substring(0, fileNameIndex);
+ regixDirectory = "";
+ }
+
+ int[] indexes = new int[]{
+ (baseDirectory.contains(YEAR) ? baseDirectory.indexOf(YEAR) : Integer.MAX_VALUE),
+ (baseDirectory.contains(MONTH) ? baseDirectory.indexOf(MONTH) : Integer.MAX_VALUE),
+ (baseDirectory.contains(DAY) ? baseDirectory.indexOf(DAY) : Integer.MAX_VALUE),
+ (baseDirectory.contains(HOUR) ? baseDirectory.indexOf(HOUR) : Integer.MAX_VALUE)};
+
+ int minIndex = Integer.MAX_VALUE;
+ for (int i = 0; i < indexes.length; i++) {
+ if (minIndex > indexes[i]) {
+ minIndex = indexes[i];
+ }
+ }
+
+ if (minIndex != Integer.MAX_VALUE) {
+ int lastIndex = baseDirectory.lastIndexOf('/', minIndex);
+ if (regixDirectory.length() > 0) {
+ regixDirectory = baseDirectory.substring(lastIndex + 1, baseDirectory.length())
+ + File.separator + regixDirectory;
+ } else {
+ regixDirectory = baseDirectory.substring(lastIndex + 1, baseDirectory.length());
+ }
+ baseDirectory = baseDirectory.substring(0, lastIndex);
+ }
+
+ ArrayList<String> ret = new ArrayList<String>();
+ ret.add(baseDirectory);
+ ret.add(regixDirectory);
+ ret.add(fileName);
+ return ret;
+ }
+
+ public static boolean isSameDir(String fileName1, String fileName2) {
+ ArrayList<String> ret1 = FilePathUtil.cutDirectory(fileName1);
+ ArrayList<String> ret2 = FilePathUtil.cutDirectory(fileName2);
+ if (ret1.get(0).equals(ret2.get(0))) {
Review Comment:
return ret1.get(0).equals(ret2.get(0))
##########
inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/utils/file/Files.java:
##########
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.agent.plugin.utils.file;
+
+import org.apache.inlong.agent.utils.file.FileFinder;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Paths;
+import java.nio.file.attribute.BasicFileAttributes;
+
+public class Files {
+
+ /**
+ * Finds files or sub directories within a given base directory.
+ *
+ * @param baseDirectory A path string representing a directory to search within.
+ * @return A {@link org.apache.inlong.agent.plugin.utils.FileFinder}-Object to specify the search
+ * parameters using a builder pattern.
+ */
+ public static FileFinder find(String baseDirectory) {
+ return find(new File(baseDirectory));
+ }
+
+ /**
+ * Finds files or sub directories within a given base directory.
+ *
+ * @param baseDirectory A path as {@link File} object to search within.
+ * @return A {@link org.apache.inlong.agent.plugin.utils.FileFinder} object to specify the search
+ * parameters using a builder pattern.
+ */
+ public static FileFinder find(File baseDirectory) {
+ return new FileFinder(baseDirectory);
+ }
+
+ public static long getFileCreationTime(String fileName) {
+ long createTime = 0L;
+ try {
+ createTime = java.nio.file.Files.readAttributes(Paths.get(fileName),
+ BasicFileAttributes.class).creationTime().toMillis();
+ } catch (IOException ignore) {
+
+ }
+ return createTime;
+ }
+
+ public static long getFileLastModifyTime(String fileName) {
+ long lastModify = 0L;
+ try {
+ lastModify = java.nio.file.Files.getLastModifiedTime(Paths.get(fileName)).toMillis();
+ } catch (IOException ioe) {
Review Comment:
catch without handle
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@inlong.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org