Merge pull request #4224 from JLUpengjiaji/V2.0

Flatten JSON data
2026-05-04 19:38:47 -04:00
parent 0e586dc2f9 f6703d3d6d
commit 2ffa655097
2 changed files with 269 additions and 32 deletions
@@ -0,0 +1,246 @@
+import { DATA_TYPE } from '../constants';
+import {
+  flattenJSON,
+  createKeyValue,
+  isValidDateValue,
+  getDataType,
+  getStringValue,
+  objectToArray,
+  type KeyValueData,
+} from '../data';
+
+describe('isValidDateValue', () => {
+  test.each([
+    ['2024-01-15T10:30:00Z', true],
+    ['2024-01-15T10:30:00.123Z', true],
+    ['2024-01-15T10:30:00+02:00', true],
+    ['not-a-date', false],
+    ['2024/01/15', false],
+    ['', false],
+  ])('validates datetime strings correctly (%s → %s)', (input, expected) => {
+    expect(isValidDateValue(input)).toBe(expected);
+  });
+
+  test('returns false for non-string values', () => {
+    expect(isValidDateValue(123 as any)).toBe(false);
+    expect(isValidDateValue(null as any)).toBe(false);
+    expect(isValidDateValue(undefined as any)).toBe(false);
+  });
+});
+
+describe('getDataType', () => {
+  test.each([
+    ['string', 'string'],
+    [123, 'number'],
+    [true, 'boolean'],
+    [null, 'object'],
+    [[], 'object'],
+    [{}, 'object'],
+    ['2024-01-15T10:30:00Z', 'date'],
+  ])('detects type correctly (%s → %s)', (input, expected) => {
+    expect(getDataType(input)).toBe(expected);
+  });
+});
+
+describe('createKeyValue', () => {
+  test('handles string values', () => {
+    const result = createKeyValue('name', 'test');
+    expect(result).toEqual({
+      key: 'name',
+      value: 'test',
+      dataType: DATA_TYPE.string,
+    });
+  });
+
+  test('handles number values', () => {
+    const result = createKeyValue('count', 42);
+    expect(result).toEqual({
+      key: 'count',
+      value: 42,
+      dataType: DATA_TYPE.number,
+    });
+  });
+
+  test('handles boolean values and converts to string', () => {
+    expect(createKeyValue('active', true)).toEqual({
+      key: 'active',
+      value: 'true',
+      dataType: DATA_TYPE.boolean,
+    });
+    expect(createKeyValue('active', false)).toEqual({
+      key: 'active',
+      value: 'false',
+      dataType: DATA_TYPE.boolean,
+    });
+  });
+
+  test('handles date strings', () => {
+    const dateStr = '2024-01-15T10:30:00Z';
+    const result = createKeyValue('timestamp', dateStr);
+    expect(result).toEqual({
+      key: 'timestamp',
+      value: dateStr,
+      dataType: DATA_TYPE.date,
+    });
+  });
+
+  test('handles arrays and converts to JSON string', () => {
+    const arr = [1, 2, 3];
+    const result = createKeyValue('items', arr);
+    expect(result).toEqual({
+      key: 'items',
+      value: '[1,2,3]',
+      dataType: DATA_TYPE.array,
+    });
+  });
+
+  test('handles null values', () => {
+    const result = createKeyValue('nullable', null);
+    expect(result.dataType).toBe(DATA_TYPE.array);
+    expect(result.value).toBe('null');
+  });
+});
+
+describe('getStringValue', () => {
+  test('formats number values with 4 decimal places', () => {
+    expect(getStringValue('42', DATA_TYPE.number)).toBe('42.0000');
+    expect(getStringValue('3.14159', DATA_TYPE.number)).toBe('3.1416');
+  });
+
+  test('converts date values to ISO string', () => {
+    const result = getStringValue('2024-01-15T10:30:00', DATA_TYPE.date);
+    expect(result).toMatch(/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}/);
+  });
+
+  test('returns string values as-is for other types', () => {
+    expect(getStringValue('test', DATA_TYPE.string)).toBe('test');
+    expect(getStringValue('true', DATA_TYPE.boolean)).toBe('true');
+  });
+});
+
+describe('objectToArray', () => {
+  test('converts object values to array', () => {
+    const obj = { a: 1, b: 2, c: 3 };
+    const result = objectToArray(obj);
+    expect(result).toEqual([1, 2, 3]);
+  });
+
+  test('returns empty array for empty object', () => {
+    expect(objectToArray({})).toEqual([]);
+  });
+});
+
+describe('flattenJSON', () => {
+  test('flattens simple object', () => {
+    const input = {
+      name: 'test',
+      count: 42,
+    };
+
+    const result = flattenJSON(input);
+
+    expect(result).toHaveLength(2);
+    expect(result).toContainEqual(expect.objectContaining({ key: 'name', value: 'test' }));
+    expect(result).toContainEqual(expect.objectContaining({ key: 'count', value: 42 }));
+  });
+
+  test('flattens nested object with dot notation', () => {
+    const input = {
+      user: {
+        name: 'John',
+        age: 30,
+      },
+    };
+
+    const result = flattenJSON(input);
+
+    expect(result).toHaveLength(2);
+    expect(result).toContainEqual(expect.objectContaining({ key: 'user.name', value: 'John' }));
+    expect(result).toContainEqual(expect.objectContaining({ key: 'user.age', value: 30 }));
+  });
+
+  test('flattens deeply nested object', () => {
+    const input = {
+      level1: {
+        level2: {
+          level3: {
+            value: 'deep',
+          },
+        },
+      },
+    };
+
+    const result = flattenJSON(input);
+
+    expect(result).toHaveLength(1);
+    expect(result[0].key).toBe('level1.level2.level3.value');
+    expect(result[0].value).toBe('deep');
+  });
+
+  test('treats arrays as leaf values (converts to JSON string)', () => {
+    const input = {
+      tags: ['a', 'b', 'c'],
+    };
+
+    const result = flattenJSON(input);
+
+    expect(result).toHaveLength(1);
+    expect(result[0].key).toBe('tags');
+    expect(result[0].value).toBe('["a","b","c"]');
+    expect(result[0].dataType).toBe(DATA_TYPE.array);
+  });
+
+  test('treats date strings as leaf values', () => {
+    const input = {
+      createdAt: '2024-01-15T10:30:00Z',
+    };
+
+    const result = flattenJSON(input);
+
+    expect(result).toHaveLength(1);
+    expect(result[0].key).toBe('createdAt');
+    expect(result[0].dataType).toBe(DATA_TYPE.date);
+  });
+
+  test('handles mixed nested and flat structure', () => {
+    const input = {
+      id: '123',
+      metadata: {
+        timestamp: '2024-01-15T10:30:00Z',
+        source: 'web',
+        details: {
+          referrer: 'google',
+        },
+      },
+    };
+
+    const result = flattenJSON(input);
+    const keys = result.map(r => r.key);
+
+    expect(result).toHaveLength(4);
+    expect(keys).toContain('id');
+    expect(keys).toContain('metadata.timestamp');
+    expect(keys).toContain('metadata.source');
+    expect(keys).toContain('metadata.details.referrer');
+  });
+
+  test('returns empty array for empty object', () => {
+    expect(flattenJSON({})).toEqual([]);
+  });
+
+  test('converts boolean values to strings', () => {
+    const input = {
+      isActive: true,
+      isAdmin: false,
+    };
+
+    const result = flattenJSON(input);
+
+    expect(result).toContainEqual(
+      expect.objectContaining({ key: 'isActive', value: 'true', dataType: DATA_TYPE.boolean }),
+    );
+    expect(result).toContainEqual(
+      expect.objectContaining({ key: 'isAdmin', value: 'false', dataType: DATA_TYPE.boolean }),
+    );
+  });
+});
@@ -1,27 +1,26 @@
 import { DATA_TYPE, DATETIME_REGEX } from './constants';
 import type { DynamicDataType } from './types';

-export function flattenJSON(
-  eventData: Record<string, any>,
-  keyValues: { key: string; value: any; dataType: DynamicDataType }[] = [],
-  parentKey = '',
-): { key: string; value: any; dataType: DynamicDataType }[] {
-  return Object.keys(eventData).reduce(
-    (acc, key) => {
-      const value = eventData[key];
-      const type = typeof eventData[key];
+export interface KeyValueData {
+  key: string;
+  value: any;
+  dataType: DynamicDataType;
+}

-      // nested object
-      if (value && type === 'object' && !Array.isArray(value) && !isValidDateValue(value)) {
-        flattenJSON(value, acc.keyValues, getKeyName(key, parentKey));
-      } else {
-        createKey(getKeyName(key, parentKey), value, acc);
+export function flattenJSON(eventData: Record<string, any>): KeyValueData[] {
+  function flatten(obj: Record<string, any>, parentKey: string): KeyValueData[] {
+    return Object.entries(obj).flatMap(([key, value]) => {
+      const fullKey = parentKey ? `${parentKey}.${key}` : key;
+
+      if (value && typeof value === 'object' && !Array.isArray(value) && !isValidDateValue(value)) {
+        return flatten(value, fullKey);
      }

-      return acc;
-    },
-    { keyValues, parentKey },
-  ).keyValues;
+      return [createKeyValue(fullKey, value)];
+    });
+  }
+
+  return flatten(eventData, '');
 }

 export function isValidDateValue(value: string) {
@@ -50,10 +49,10 @@ export function getStringValue(value: string, dataType: number) {
  return value;
 }

-function createKey(key: string, value: string, acc: { keyValues: any[]; parentKey: string }) {
+export function createKeyValue(key: string, value: any): KeyValueData {
  const type = getDataType(value);
-
-  let dataType = null;
+  let dataType: DynamicDataType;
+  let processedValue = value;

  switch (type) {
    case 'number':
@@ -64,29 +63,21 @@ function createKey(key: string, value: string, acc: { keyValues: any[]; parentKe
      break;
    case 'boolean':
      dataType = DATA_TYPE.boolean;
-      value = value ? 'true' : 'false';
+      processedValue = value ? 'true' : 'false';
      break;
    case 'date':
      dataType = DATA_TYPE.date;
      break;
    case 'object':
      dataType = DATA_TYPE.array;
-      value = JSON.stringify(value);
+      processedValue = JSON.stringify(value);
      break;
    default:
      dataType = DATA_TYPE.string;
      break;
  }

-  acc.keyValues.push({ key, value, dataType });
-}
-
-function getKeyName(key: string, parentKey: string) {
-  if (!parentKey) {
-    return key;
-  }
-
-  return `${parentKey}.${key}`;
+  return { key, value: processedValue, dataType };
 }

 export function objectToArray(obj: object) {