You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@accumulo.apache.org by GitBox <gi...@apache.org> on 2018/12/14 21:09:34 UTC
[GitHub] keith-turner commented on a change in pull request #830: #820 - Fix
broken ITs due to client issues
keith-turner commented on a change in pull request #830: #820 - Fix broken ITs due to client issues
URL: https://github.com/apache/accumulo/pull/830#discussion_r241887294
##########
File path: hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapreduce/AbstractInputFormat.java
##########
@@ -450,149 +466,154 @@ public V getCurrentValue() {
public static Map<String,Map<KeyExtent,List<Range>>> binOfflineTable(JobContext context,
Table.ID tableId, List<Range> ranges)
throws TableNotFoundException, AccumuloException, AccumuloSecurityException {
- ClientContext clientContext = new ClientContext(getClientInfo(context));
- return InputConfigurator.binOffline(tableId, ranges, clientContext);
+ try (AccumuloClient client = createClient(context)) {
+ ClientContext clientContext = new ClientContext(client);
+ return InputConfigurator.binOffline(tableId, ranges, clientContext);
+ }
}
public static List<InputSplit> getSplits(JobContext context) throws IOException {
validateOptions(context);
Random random = new SecureRandom();
LinkedList<InputSplit> splits = new LinkedList<>();
- Map<String,InputTableConfig> tableConfigs = getInputTableConfigs(context);
- for (Map.Entry<String,InputTableConfig> tableConfigEntry : tableConfigs.entrySet()) {
-
- String tableName = tableConfigEntry.getKey();
- InputTableConfig tableConfig = tableConfigEntry.getValue();
-
- ClientContext clientContext = new ClientContext(getClientInfo(context));
- Table.ID tableId;
- // resolve table name to id once, and use id from this point forward
- try {
- tableId = Tables.getTableId(clientContext, tableName);
- } catch (TableNotFoundException e) {
- throw new IOException(e);
- }
+ try (AccumuloClient client = createClient(context)) {
+ Map<String,InputTableConfig> tableConfigs = getInputTableConfigs(context);
+ for (Map.Entry<String,InputTableConfig> tableConfigEntry : tableConfigs.entrySet()) {
- boolean batchScan = InputConfigurator.isBatchScan(CLASS, context.getConfiguration());
- boolean supportBatchScan = !(tableConfig.isOfflineScan()
- || tableConfig.shouldUseIsolatedScanners() || tableConfig.shouldUseLocalIterators());
- if (batchScan && !supportBatchScan)
- throw new IllegalArgumentException("BatchScanner optimization not available for offline"
- + " scan, isolated, or local iterators");
-
- boolean autoAdjust = tableConfig.shouldAutoAdjustRanges();
- if (batchScan && !autoAdjust)
- throw new IllegalArgumentException(
- "AutoAdjustRanges must be enabled when using BatchScanner optimization");
-
- List<Range> ranges = autoAdjust ? Range.mergeOverlapping(tableConfig.getRanges())
- : tableConfig.getRanges();
- if (ranges.isEmpty()) {
- ranges = new ArrayList<>(1);
- ranges.add(new Range());
- }
+ String tableName = tableConfigEntry.getKey();
+ InputTableConfig tableConfig = tableConfigEntry.getValue();
+
+ ClientContext clientContext = new ClientContext(client);
+ Table.ID tableId;
+ // resolve table name to id once, and use id from this point forward
+ try {
+ tableId = Tables.getTableId(clientContext, tableName);
+ } catch (TableNotFoundException e) {
+ throw new IOException(e);
+ }
+
+ boolean batchScan = InputConfigurator.isBatchScan(CLASS, context.getConfiguration());
+ boolean supportBatchScan = !(tableConfig.isOfflineScan()
+ || tableConfig.shouldUseIsolatedScanners() || tableConfig.shouldUseLocalIterators());
+ if (batchScan && !supportBatchScan)
+ throw new IllegalArgumentException("BatchScanner optimization not available for offline"
+ + " scan, isolated, or local iterators");
+
+ boolean autoAdjust = tableConfig.shouldAutoAdjustRanges();
+ if (batchScan && !autoAdjust)
+ throw new IllegalArgumentException(
+ "AutoAdjustRanges must be enabled when using BatchScanner optimization");
+
+ List<Range> ranges = autoAdjust ? Range.mergeOverlapping(tableConfig.getRanges())
+ : tableConfig.getRanges();
+ if (ranges.isEmpty()) {
+ ranges = new ArrayList<>(1);
+ ranges.add(new Range());
+ }
- // get the metadata information for these ranges
- Map<String,Map<KeyExtent,List<Range>>> binnedRanges = new HashMap<>();
- TabletLocator tl;
- try {
- if (tableConfig.isOfflineScan()) {
- binnedRanges = binOfflineTable(context, tableId, ranges);
- while (binnedRanges == null) {
- // Some tablets were still online, try again
- // sleep randomly between 100 and 200 ms
- sleepUninterruptibly(100 + random.nextInt(100), TimeUnit.MILLISECONDS);
+ // get the metadata information for these ranges
+ Map<String,Map<KeyExtent,List<Range>>> binnedRanges = new HashMap<>();
+ TabletLocator tl;
+ try {
+ if (tableConfig.isOfflineScan()) {
binnedRanges = binOfflineTable(context, tableId, ranges);
+ while (binnedRanges == null) {
+ // Some tablets were still online, try again
+ // sleep randomly between 100 and 200 ms
+ sleepUninterruptibly(100 + random.nextInt(100), TimeUnit.MILLISECONDS);
+ binnedRanges = binOfflineTable(context, tableId, ranges);
- }
- } else {
- tl = InputConfigurator.getTabletLocator(CLASS, context.getConfiguration(), tableId);
- // its possible that the cache could contain complete, but old information about a tables
- // tablets... so clear it
- tl.invalidateCache();
-
- while (!tl.binRanges(clientContext, ranges, binnedRanges).isEmpty()) {
- String tableIdStr = tableId.canonicalID();
- if (!Tables.exists(clientContext, tableId))
- throw new TableDeletedException(tableIdStr);
- if (Tables.getTableState(clientContext, tableId) == TableState.OFFLINE)
- throw new TableOfflineException(Tables.getTableOfflineMsg(clientContext, tableId));
- binnedRanges.clear();
- log.warn("Unable to locate bins for specified ranges. Retrying.");
- // sleep randomly between 100 and 200 ms
- sleepUninterruptibly(100 + random.nextInt(100), TimeUnit.MILLISECONDS);
+ }
+ } else {
+ tl = InputConfigurator.getTabletLocator(CLASS, context.getConfiguration(), tableId);
+ // its possible that the cache could contain complete, but old information about a
+ // tables
Review comment:
would be nice to make this comment two lines instead of three
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services