You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@druid.apache.org by ab...@apache.org on 2024/09/30 17:25:44 UTC

(druid) branch 31.0.0 updated: Web console: Fixed sampling for delta source in classic data loader and MSQ (#17160) (#17199)

This is an automated email from the ASF dual-hosted git repository.

abhishekrb pushed a commit to branch 31.0.0
in repository https://gitbox.apache.org/repos/asf/druid.git


The following commit(s) were added to refs/heads/31.0.0 by this push:
     new c6f41dcd22d Web console: Fixed sampling for delta source in classic data loader and MSQ (#17160) (#17199)
c6f41dcd22d is described below

commit c6f41dcd22de1e1290569d11511192ee5d3b1cf6
Author: Abhishek Radhakrishnan <ab...@gmail.com>
AuthorDate: Mon Sep 30 10:25:38 2024 -0700

    Web console: Fixed sampling for delta source in classic data loader and MSQ (#17160) (#17199)
---
 web-console/script/druid                           |   4 +-
 .../src/components/json-input/json-input.tsx       |   5 +-
 .../druid-models/ingestion-spec/ingestion-spec.tsx |  10 +-
 .../src/druid-models/input-source/input-source.tsx |   3 +-
 web-console/src/utils/sampler.ts                   |  13 ++-
 .../src/views/load-data-view/load-data-view.tsx    |  77 +++++++++------
 .../sql-data-loader-view/sql-data-loader-view.tsx  |   1 -
 .../connect-external-data-dialog.tsx               |   1 -
 .../input-format-step/input-format-step.tsx        |  63 +++++++-----
 .../input-source-step/input-source-step.tsx        | 110 +++++++++------------
 10 files changed, 151 insertions(+), 136 deletions(-)

diff --git a/web-console/script/druid b/web-console/script/druid
index 7b3163bf9d6..122febaf049 100755
--- a/web-console/script/druid
+++ b/web-console/script/druid
@@ -56,7 +56,7 @@ function _build_distribution() {
   (
     # Add HEAD as an allowed HTTP method since this is how we check when the Druid service is ready.
     cd "$(_get_code_root)" \
-    && mvn -Pdist,skip-static-checks,skip-tests -Dmaven.javadoc.skip=true -q -T1C install \
+    && mvn -Pdist,bundle-contrib-exts,skip-static-checks,skip-tests -Dforbiddenapis.skip=true -Dcheckstyle.skip=true -Dpmd.skip=true -Dmaven.javadoc.skip=true -Danimal.sniffer.skip=true -Denforcer.skip=true -Dcyclonedx.skip=true -q -T1C install \
     && cd distribution/target \
     && tar xzf "apache-druid-$(_get_druid_version)-bin.tar.gz" \
     && cd apache-druid-$(_get_druid_version) \
@@ -64,7 +64,7 @@ function _build_distribution() {
     && cp "$(_get_code_root)/extensions-core/testing-tools/target/druid-testing-tools-$(_get_druid_version).jar" extensions/druid-testing-tools/ \
     && mkdir -p extensions/druid-compressed-bigdecimal \
     && cp "$(_get_code_root)/extensions-contrib/compressed-bigdecimal/target/druid-compressed-bigdecimal-$(_get_druid_version).jar" extensions/druid-compressed-bigdecimal/ \
-    && echo -e "\n\ndruid.extensions.loadList=[\"druid-hdfs-storage\", \"druid-kafka-indexing-service\", \"druid-multi-stage-query\", \"druid-testing-tools\", \"druid-bloom-filter\", \"druid-datasketches\", \"druid-histogram\", \"druid-stats\", \"druid-compressed-bigdecimal\"]" >> conf/druid/auto/_common/common.runtime.properties \
+    && echo -e "\n\ndruid.extensions.loadList=[\"druid-hdfs-storage\", \"druid-kafka-indexing-service\", \"druid-multi-stage-query\", \"druid-testing-tools\", \"druid-bloom-filter\", \"druid-datasketches\", \"druid-histogram\", \"druid-stats\", \"druid-compressed-bigdecimal\", \"druid-parquet-extensions\", \"druid-deltalake-extensions\"]" >> conf/druid/auto/_common/common.runtime.properties \
     && echo -e "\n\ndruid.server.http.allowedHttpMethods=[\"HEAD\"]" >> conf/druid/auto/_common/common.runtime.properties \
     && echo -e "\n\ndruid.export.storage.baseDir=/" >> conf/druid/auto/_common/common.runtime.properties \
   )
diff --git a/web-console/src/components/json-input/json-input.tsx b/web-console/src/components/json-input/json-input.tsx
index f4ffddc1114..578cab3ebcc 100644
--- a/web-console/src/components/json-input/json-input.tsx
+++ b/web-console/src/components/json-input/json-input.tsx
@@ -25,9 +25,8 @@ import AceEditor from 'react-ace';
 
 import './json-input.scss';
 
-function parseHjson(str: string) {
-  // Throwing on empty input is more consistent with how JSON.parse works
-  if (str.trim() === '') throw new Error('empty hjson');
+function parseHjson(str: string): any {
+  if (str.trim() === '') return;
   return Hjson.parse(str);
 }
 
diff --git a/web-console/src/druid-models/ingestion-spec/ingestion-spec.tsx b/web-console/src/druid-models/ingestion-spec/ingestion-spec.tsx
index 65a7ed3b6c4..46d6a3a4cbb 100644
--- a/web-console/src/druid-models/ingestion-spec/ingestion-spec.tsx
+++ b/web-console/src/druid-models/ingestion-spec/ingestion-spec.tsx
@@ -395,6 +395,10 @@ export function isDruidSource(spec: Partial<IngestionSpec>): boolean {
   return deepGet(spec, 'spec.ioConfig.inputSource.type') === 'druid';
 }
 
+export function isFixedFormatSource(spec: Partial<IngestionSpec>): boolean {
+  return oneOf(deepGet(spec, 'spec.ioConfig.inputSource.type'), 'druid', 'delta');
+}
+
 export function getPossibleSystemFieldsForSpec(spec: Partial<IngestionSpec>): string[] {
   const inputSource = deepGet(spec, 'spec.ioConfig.inputSource');
   if (!inputSource) return [];
@@ -1061,7 +1065,6 @@ export function getIoConfigFormFields(ingestionComboType: IngestionComboType): F
           label: 'Delta filter',
           type: 'json',
           placeholder: '{"type": "=", "column": "name", "value": "foo"}',
-          defaultValue: {},
           info: (
             <>
               <ExternalLink
@@ -1078,7 +1081,7 @@ export function getIoConfigFormFields(ingestionComboType: IngestionComboType): F
           label: 'Delta snapshot version',
           type: 'number',
           placeholder: '(latest)',
-          defaultValue: {},
+          zeroMeansUndefined: true,
           info: (
             <>
               The snapshot version to read from the Delta table. By default, the latest snapshot is
@@ -1613,6 +1616,9 @@ export function guessDataSourceNameFromInputSource(inputSource: InputSource): st
       return actualPath ? actualPath.path : uriPath ? filenameFromPath(uriPath) : undefined;
     }
 
+    case 'delta':
+      return inputSource.tablePath ? filenameFromPath(inputSource.tablePath) : undefined;
+
     case 'http':
       return Array.isArray(inputSource.uris) ? filenameFromPath(inputSource.uris[0]) : undefined;
 
diff --git a/web-console/src/druid-models/input-source/input-source.tsx b/web-console/src/druid-models/input-source/input-source.tsx
index eb19e4575db..6ad3b8e0a1a 100644
--- a/web-console/src/druid-models/input-source/input-source.tsx
+++ b/web-console/src/druid-models/input-source/input-source.tsx
@@ -653,7 +653,6 @@ export const INPUT_SOURCE_FIELDS: Field<InputSource>[] = [
     type: 'json',
     placeholder: '{"type": "=", "column": "name", "value": "foo"}',
     defined: typeIsKnown(KNOWN_TYPES, 'delta'),
-    required: false,
     info: (
       <>
         <ExternalLink href={`${getLink('DOCS')}/ingestion/input-sources/#delta-filter-object`}>
@@ -668,8 +667,8 @@ export const INPUT_SOURCE_FIELDS: Field<InputSource>[] = [
     label: 'Delta snapshot version',
     type: 'number',
     placeholder: '(latest)',
+    zeroMeansUndefined: true,
     defined: typeIsKnown(KNOWN_TYPES, 'delta'),
-    required: false,
     info: (
       <>
         The snapshot version to read from the Delta table. By default, the latest snapshot is read.
diff --git a/web-console/src/utils/sampler.ts b/web-console/src/utils/sampler.ts
index f74acf024c9..7fd3d67364c 100644
--- a/web-console/src/utils/sampler.ts
+++ b/web-console/src/utils/sampler.ts
@@ -17,6 +17,7 @@
  */
 
 import { dedupe, F, SqlExpression, SqlFunction } from '@druid-toolkit/query';
+import type { CancelToken } from 'axios';
 import * as JSONBig from 'json-bigint-native';
 
 import type {
@@ -40,6 +41,7 @@ import {
   getSpecType,
   getTimestampSchema,
   isDruidSource,
+  isFixedFormatSource,
   PLACEHOLDER_TIMESTAMP_SPEC,
   REINDEX_TIMESTAMP_SPEC,
   TIME_COLUMN,
@@ -187,12 +189,15 @@ export async function getProxyOverlordModules(): Promise<string[]> {
 export async function postToSampler(
   sampleSpec: SampleSpec,
   forStr: string,
+  cancelToken?: CancelToken,
 ): Promise<SampleResponse> {
   sampleSpec = fixSamplerLookups(fixSamplerTypes(sampleSpec));
 
   let sampleResp: any;
   try {
-    sampleResp = await Api.instance.post(`/druid/indexer/v1/sampler?for=${forStr}`, sampleSpec);
+    sampleResp = await Api.instance.post(`/druid/indexer/v1/sampler?for=${forStr}`, sampleSpec, {
+      cancelToken,
+    });
   } catch (e) {
     throw new Error(getDruidErrorMessage(e));
   }
@@ -269,8 +274,7 @@ export async function sampleForConnect(
     sampleStrategy,
   );
 
-  const reingestMode = isDruidSource(spec);
-  if (!reingestMode) {
+  if (!isFixedFormatSource(spec)) {
     ioConfig = deepSet(
       ioConfig,
       'inputFormat',
@@ -282,6 +286,7 @@ export async function sampleForConnect(
     );
   }
 
+  const reingestMode = isDruidSource(spec);
   const sampleSpec: SampleSpec = {
     type: samplerType,
     spec: {
@@ -290,7 +295,7 @@ export async function sampleForConnect(
       dataSchema: {
         dataSource: 'sample',
         timestampSpec: reingestMode ? REINDEX_TIMESTAMP_SPEC : PLACEHOLDER_TIMESTAMP_SPEC,
-        dimensionsSpec: {},
+        dimensionsSpec: { useSchemaDiscovery: true },
         granularitySpec: {
           rollup: false,
         },
diff --git a/web-console/src/views/load-data-view/load-data-view.tsx b/web-console/src/views/load-data-view/load-data-view.tsx
index 42cfa25a7b7..bcb79d639f2 100644
--- a/web-console/src/views/load-data-view/load-data-view.tsx
+++ b/web-console/src/views/load-data-view/load-data-view.tsx
@@ -115,6 +115,7 @@ import {
   invalidPartitionConfig,
   isDruidSource,
   isEmptyIngestionSpec,
+  isFixedFormatSource,
   isKafkaOrKinesis,
   isStreamingSpec,
   issueWithIoConfig,
@@ -265,26 +266,27 @@ function showBlankLine(line: SampleEntry): string {
 
 function formatSampleEntries(
   sampleEntries: SampleEntry[],
-  specialSource: undefined | 'druid' | 'kafka' | 'kinesis',
-): string {
-  if (!sampleEntries.length) return 'No data returned from sampler';
+  specialSource: undefined | 'fixedFormat' | 'druid' | 'kafka' | 'kinesis',
+): string[] {
+  if (!sampleEntries.length) return ['No data returned from sampler'];
 
   switch (specialSource) {
+    case 'fixedFormat':
+      return sampleEntries.map(l => JSONBig.stringify(l.parsed));
+
     case 'druid':
-      return sampleEntries.map(showDruidLine).join('\n');
+      return sampleEntries.map(showDruidLine);
 
     case 'kafka':
-      return sampleEntries.map(showKafkaLine).join('\n');
+      return sampleEntries.map(showKafkaLine);
 
     case 'kinesis':
-      return sampleEntries.map(showKinesisLine).join('\n');
+      return sampleEntries.map(showKinesisLine);
 
     default:
-      return (
-        sampleEntries.every(l => !l.parsed)
-          ? sampleEntries.map(showBlankLine)
-          : sampleEntries.map(showRawLine)
-      ).join('\n');
+      return sampleEntries.every(l => !l.parsed)
+        ? sampleEntries.map(showBlankLine)
+        : sampleEntries.map(showRawLine);
   }
 }
 
@@ -551,7 +553,6 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
 
   isStepEnabled(step: Step): boolean {
     const { spec, cacheRows } = this.state;
-    const druidSource = isDruidSource(spec);
     const ioConfig: IoConfig = deepGet(spec, 'spec.ioConfig') || EMPTY_OBJECT;
 
     switch (step) {
@@ -559,10 +560,12 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
         return Boolean(spec.type);
 
       case 'parser':
-        return Boolean(!druidSource && spec.type && !issueWithIoConfig(ioConfig));
+        return Boolean(!isFixedFormatSource(spec) && spec.type && !issueWithIoConfig(ioConfig));
 
       case 'timestamp':
-        return Boolean(!druidSource && cacheRows && deepGet(spec, 'spec.dataSchema.timestampSpec'));
+        return Boolean(
+          !isDruidSource(spec) && cacheRows && deepGet(spec, 'spec.dataSchema.timestampSpec'),
+        );
 
       case 'transform':
       case 'filter':
@@ -1256,7 +1259,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
     const deltaState: Partial<LoadDataViewState> = {
       inputQueryState: new QueryState({ data: sampleResponse }),
     };
-    if (isDruidSource(spec)) {
+    if (isFixedFormatSource(spec)) {
       deltaState.cacheRows = getCacheRowsFromSampleResponse(sampleResponse);
     }
     this.setState(deltaState as LoadDataViewState);
@@ -1268,8 +1271,15 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
     const specType = getSpecType(spec);
     const ioConfig: IoConfig = deepGet(spec, 'spec.ioConfig') || EMPTY_OBJECT;
     const inlineMode = deepGet(spec, 'spec.ioConfig.inputSource.type') === 'inline';
+    const fixedFormatSource = isFixedFormatSource(spec);
     const druidSource = isDruidSource(spec);
-    const specialSource = druidSource ? 'druid' : isKafkaOrKinesis(specType) ? specType : undefined;
+    const specialSource = druidSource
+      ? 'druid'
+      : fixedFormatSource
+      ? 'fixedFormat'
+      : isKafkaOrKinesis(specType)
+      ? specType
+      : undefined;
 
     let mainFill: JSX.Element | string;
     if (inlineMode) {
@@ -1301,7 +1311,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
             <TextArea
               className="raw-lines"
               readOnly
-              value={formatSampleEntries(inputData, specialSource)}
+              value={formatSampleEntries(inputData, specialSource).join('\n')}
             />
           )}
           {inputQueryState.isLoading() && <Loader />}
@@ -1373,7 +1383,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
         </div>
         {this.renderNextBar({
           disabled: !inputQueryState.data,
-          nextStep: druidSource ? 'transform' : 'parser',
+          nextStep: druidSource ? 'transform' : fixedFormatSource ? 'timestamp' : 'parser',
           onNextStep: () => {
             if (!inputQueryState.data) return false;
             const inputData = inputQueryState.data;
@@ -1421,6 +1431,15 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
                 }
               }
 
+              this.updateSpec(fillDataSourceNameIfNeeded(newSpec));
+            }
+            if (fixedFormatSource) {
+              const newSpec = deepSet(
+                spec,
+                'spec.dataSchema.timestampSpec',
+                getTimestampSpec(inputQueryState.data),
+              );
+
               this.updateSpec(fillDataSourceNameIfNeeded(newSpec));
             } else {
               const issue = issueWithSampleData(
@@ -1673,21 +1692,15 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
           disabled: !parserQueryState.data,
           onNextStep: () => {
             if (!parserQueryState.data) return false;
-            let possibleTimestampSpec: TimestampSpec;
-            if (isDruidSource(spec)) {
-              possibleTimestampSpec = {
-                column: TIME_COLUMN,
-                format: 'auto',
-              };
-            } else {
-              possibleTimestampSpec = getTimestampSpec(parserQueryState.data);
-            }
-
-            if (possibleTimestampSpec) {
-              const newSpec = deepSet(spec, 'spec.dataSchema.timestampSpec', possibleTimestampSpec);
-              this.updateSpec(newSpec);
-            }
+            const possibleTimestampSpec = isDruidSource(spec)
+              ? {
+                  column: TIME_COLUMN,
+                  format: 'auto',
+                }
+              : getTimestampSpec(parserQueryState.data);
 
+            const newSpec = deepSet(spec, 'spec.dataSchema.timestampSpec', possibleTimestampSpec);
+            this.updateSpec(newSpec);
             return true;
           },
         })}
diff --git a/web-console/src/views/sql-data-loader-view/sql-data-loader-view.tsx b/web-console/src/views/sql-data-loader-view/sql-data-loader-view.tsx
index a7d6adac71e..8238384d9ae 100644
--- a/web-console/src/views/sql-data-loader-view/sql-data-loader-view.tsx
+++ b/web-console/src/views/sql-data-loader-view/sql-data-loader-view.tsx
@@ -256,7 +256,6 @@ export const SqlDataLoaderView = React.memo(function SqlDataLoaderView(
         <TitleFrame title="Load data" subtitle="Select input type">
           <InputSourceStep
             initInputSource={inputSource}
-            mode="sampler"
             onSet={(inputSource, inputFormat) => {
               setExternalConfigStep({ inputSource, inputFormat });
             }}
diff --git a/web-console/src/views/workbench-view/connect-external-data-dialog/connect-external-data-dialog.tsx b/web-console/src/views/workbench-view/connect-external-data-dialog/connect-external-data-dialog.tsx
index 4302c3783b2..124b4441194 100644
--- a/web-console/src/views/workbench-view/connect-external-data-dialog/connect-external-data-dialog.tsx
+++ b/web-console/src/views/workbench-view/connect-external-data-dialog/connect-external-data-dialog.tsx
@@ -83,7 +83,6 @@ export const ConnectExternalDataDialog = React.memo(function ConnectExternalData
         ) : (
           <InputSourceStep
             initInputSource={inputSource}
-            mode="sampler"
             onSet={(inputSource, inputFormat, partitionedByHint) => {
               setExternalConfigStep({ inputSource, inputFormat, partitionedByHint });
             }}
diff --git a/web-console/src/views/workbench-view/input-format-step/input-format-step.tsx b/web-console/src/views/workbench-view/input-format-step/input-format-step.tsx
index 3586170d63d..2f054a1543e 100644
--- a/web-console/src/views/workbench-view/input-format-step/input-format-step.tsx
+++ b/web-console/src/views/workbench-view/input-format-step/input-format-step.tsx
@@ -98,7 +98,9 @@ export const InputFormatStep = React.memo(function InputFormatStep(props: InputF
   const [previewState] = useQueryManager<InputSourceAndFormat, SampleResponse>({
     query: inputSourceAndFormatToSample,
     processQuery: async ({ inputSource, inputFormat }) => {
-      if (!isValidInputFormat(inputFormat)) throw new Error('invalid input format');
+      const fixedFormatSource = inputSource.type === 'delta';
+      if (!fixedFormatSource && !isValidInputFormat(inputFormat))
+        throw new Error('invalid input format');
 
       const sampleSpec: SampleSpec = {
         type: 'index_parallel',
@@ -106,7 +108,9 @@ export const InputFormatStep = React.memo(function InputFormatStep(props: InputF
           ioConfig: {
             type: 'index_parallel',
             inputSource,
-            inputFormat: deepSet(inputFormat, 'keepNullColumns', true),
+            inputFormat: fixedFormatSource
+              ? undefined
+              : (deepSet(inputFormat, 'keepNullColumns', true) as InputFormat),
           },
           dataSchema: {
             dataSource: 'sample',
@@ -196,6 +200,8 @@ export const InputFormatStep = React.memo(function InputFormatStep(props: InputF
   const needsResample = inputSourceAndFormatToSample !== inputSourceAndFormat;
   const nextDisabled = !inputSourceFormatAndMore || needsResample;
 
+  const fixedFormatSource = inputSourceFormatAndMore?.inputSource.type === 'delta';
+
   return (
     <div className="input-format-step">
       <div className="preview">
@@ -227,27 +233,38 @@ export const InputFormatStep = React.memo(function InputFormatStep(props: InputF
               <LearnMore href={`${getLink('DOCS')}/ingestion/data-formats`} />
             </Callout>
           </FormGroup>
-          <AutoForm
-            fields={BATCH_INPUT_FORMAT_FIELDS}
-            model={inputSourceAndFormat.inputFormat}
-            onChange={inputFormat =>
-              setInputSourceAndFormat({ ...inputSourceAndFormat, inputFormat })
-            }
-          />
-          {possibleSystemFields.length > 0 && (
-            <AutoForm
-              fields={[
-                {
-                  name: 'inputSource.systemFields',
-                  label: 'System fields',
-                  type: 'string-array',
-                  suggestions: possibleSystemFields,
-                  info: 'JSON array of system fields to return as part of input rows.',
-                },
-              ]}
-              model={inputSourceAndFormat}
-              onChange={setInputSourceAndFormat as any}
-            />
+          {fixedFormatSource ? (
+            <FormGroup>
+              <Callout>
+                The <Tag minimal>{inputSourceFormatAndMore?.inputSource.type}</Tag> input source has
+                a fixed format that can not be configured.
+              </Callout>
+            </FormGroup>
+          ) : (
+            <>
+              <AutoForm
+                fields={BATCH_INPUT_FORMAT_FIELDS}
+                model={inputSourceAndFormat.inputFormat}
+                onChange={inputFormat =>
+                  setInputSourceAndFormat({ ...inputSourceAndFormat, inputFormat })
+                }
+              />
+              {possibleSystemFields.length > 0 && (
+                <AutoForm
+                  fields={[
+                    {
+                      name: 'inputSource.systemFields',
+                      label: 'System fields',
+                      type: 'string-array',
+                      suggestions: possibleSystemFields,
+                      info: 'JSON array of system fields to return as part of input rows.',
+                    },
+                  ]}
+                  model={inputSourceAndFormat}
+                  onChange={setInputSourceAndFormat as any}
+                />
+              )}
+            </>
           )}
           {needsResample && (
             <FormGroup className="control-buttons">
diff --git a/web-console/src/views/workbench-view/input-source-step/input-source-step.tsx b/web-console/src/views/workbench-view/input-source-step/input-source-step.tsx
index 6d483cf4410..8beed74fa27 100644
--- a/web-console/src/views/workbench-view/input-source-step/input-source-step.tsx
+++ b/web-console/src/views/workbench-view/input-source-step/input-source-step.tsx
@@ -28,7 +28,6 @@ import {
 } from '@blueprintjs/core';
 import { IconNames } from '@blueprintjs/icons';
 import type { QueryResult } from '@druid-toolkit/query';
-import { SqlColumnDeclaration } from '@druid-toolkit/query';
 import classNames from 'classnames';
 import type { JSX } from 'react';
 import React, { useEffect, useState } from 'react';
@@ -37,7 +36,6 @@ import { AutoForm, ExternalLink } from '../../../components';
 import { ShowValueDialog } from '../../../dialogs/show-value-dialog/show-value-dialog';
 import type { Execution, ExecutionError, InputFormat, InputSource } from '../../../druid-models';
 import {
-  externalConfigToTableExpression,
   getIngestionImage,
   getIngestionTitle,
   guessSimpleInputFormat,
@@ -45,11 +43,7 @@ import {
   issueWithSampleData,
   PLACEHOLDER_TIMESTAMP_SPEC,
 } from '../../../druid-models';
-import {
-  executionBackgroundResultStatusCheck,
-  extractResult,
-  submitTaskQuery,
-} from '../../../helpers';
+import { executionBackgroundResultStatusCheck } from '../../../helpers';
 import { useQueryManager } from '../../../hooks';
 import { AppToaster, UrlBaser } from '../../../singletons';
 import { filterMap, IntermediateQueryState } from '../../../utils';
@@ -61,17 +55,20 @@ import { InputSourceInfo } from './input-source-info';
 
 import './input-source-step.scss';
 
+const BOGUS_LIST_DELIMITER = '56616469-6de2-9da4-efb8-8f416e6e6965'; // Just a UUID to disable the list delimiter, let's hope we do not see this UUID in the data
+const ROWS_TO_SAMPLE = 50;
+
+const FIXED_FORMAT_FOR_SOURCE: Record<string, InputFormat> = {
+  delta: { type: 'parquet' },
+};
+
 function resultToInputFormat(result: QueryResult): InputFormat {
   if (!result.rows.length) throw new Error('No data returned from sample query');
   return guessSimpleInputFormat(result.rows.map((r: any) => r[0]));
 }
 
-const BOGUS_LIST_DELIMITER = '56616469-6de2-9da4-efb8-8f416e6e6965'; // Just a UUID to disable the list delimiter, let's hope we do not see this UUID in the data
-const ROWS_TO_SAMPLE = 50;
-
 export interface InputSourceStepProps {
   initInputSource: Partial<InputSource> | undefined;
-  mode: 'sampler' | 'msq';
   onSet(
     inputSource: InputSource,
     inputFormat: InputFormat,
@@ -80,7 +77,7 @@ export interface InputSourceStepProps {
 }
 
 export const InputSourceStep = React.memo(function InputSourceStep(props: InputSourceStepProps) {
-  const { initInputSource, mode, onSet } = props;
+  const { initInputSource, onSet } = props;
 
   const [stackToShow, setStackToShow] = useState<string | undefined>();
   const [inputSource, setInputSource] = useState<Partial<InputSource> | string | undefined>(
@@ -94,66 +91,46 @@ export const InputSourceStep = React.memo(function InputSourceStep(props: InputS
     Execution
   >({
     processQuery: async ({ inputSource, suggestedInputFormat }, cancelToken) => {
-      let sampleLines: string[];
-      if (mode === 'sampler') {
-        const sampleSpec: SampleSpec = {
-          type: 'index_parallel',
-          spec: {
-            ioConfig: {
-              type: 'index_parallel',
-              inputSource,
-              inputFormat: {
-                type: 'regex',
-                pattern: '([\\s\\S]*)', // Match the entire line, every single character
-                listDelimiter: BOGUS_LIST_DELIMITER,
-                columns: ['raw'],
-              },
-            },
-            dataSchema: {
-              dataSource: 'sample',
-              timestampSpec: PLACEHOLDER_TIMESTAMP_SPEC,
-              dimensionsSpec: {},
-              granularitySpec: {
-                rollup: false,
-              },
-            },
-          },
-          samplerConfig: {
-            numRows: ROWS_TO_SAMPLE,
-            timeoutMs: 15000,
-          },
-        };
-
-        const sampleResponse = await postToSampler(sampleSpec, 'input-source-step');
+      const fixedFormat = FIXED_FORMAT_FOR_SOURCE['delta'];
 
-        sampleLines = filterMap(sampleResponse.data, l => (l.input ? l.input.raw : undefined));
-      } else {
-        const tableExpression = externalConfigToTableExpression({
-          inputSource,
-          inputFormat: {
-            type: 'regex',
-            pattern: '([\\s\\S]*)',
-            listDelimiter: BOGUS_LIST_DELIMITER,
-            columns: ['raw'],
+      const sampleSpec: SampleSpec = {
+        type: 'index_parallel',
+        spec: {
+          ioConfig: {
+            type: 'index_parallel',
+            inputSource,
+            inputFormat: fixedFormat
+              ? undefined
+              : {
+                  type: 'regex',
+                  pattern: '([\\s\\S]*)', // Match the entire line, every single character
+                  listDelimiter: BOGUS_LIST_DELIMITER,
+                  columns: ['raw'],
+                },
           },
-          signature: [SqlColumnDeclaration.create('raw', 'VARCHAR')],
-        });
-
-        const result = extractResult(
-          await submitTaskQuery({
-            query: `SELECT REPLACE(raw, U&'\\0000', '') AS "raw" FROM ${tableExpression}`, // Make sure to remove possible \u0000 chars as they are not allowed and will produce an InvalidNullByte error message
-            context: {
-              sqlOuterLimit: ROWS_TO_SAMPLE,
+          dataSchema: {
+            dataSource: 'sample',
+            timestampSpec: PLACEHOLDER_TIMESTAMP_SPEC,
+            dimensionsSpec: {},
+            granularitySpec: {
+              rollup: false,
             },
-            cancelToken,
-          }),
-        );
+          },
+        },
+        samplerConfig: {
+          numRows: ROWS_TO_SAMPLE,
+          timeoutMs: 15000,
+        },
+      };
 
-        if (result instanceof IntermediateQueryState) return result;
-        sampleLines = result.rows.map((r: string[]) => r[0]);
-      }
+      const sampleResponse = await postToSampler(sampleSpec, 'input-source-step', cancelToken);
+      const sampleLines = filterMap(
+        sampleResponse.data,
+        fixedFormat ? l => l.input : l => (l.input ? l.input.raw : undefined),
+      );
 
       if (!sampleLines.length) throw new Error('No data returned from sampler');
+      if (fixedFormat) return fixedFormat;
 
       const issue = issueWithSampleData(sampleLines, false);
       if (issue) {
@@ -226,6 +203,7 @@ export const InputSourceStep = React.memo(function InputSourceStep(props: InputS
         {renderIngestionCard('s3')}
         {renderIngestionCard('azureStorage')}
         {renderIngestionCard('google')}
+        {renderIngestionCard('delta')}
         {renderIngestionCard('hdfs')}
         {renderIngestionCard('http')}
         {renderIngestionCard('local')}


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@druid.apache.org
For additional commands, e-mail: commits-help@druid.apache.org