diff --git a/README.md b/README.md index bed7858..c3fb44e 100644 --- a/README.md +++ b/README.md @@ -76,6 +76,30 @@ dataxl -i examples/people.yaml -o people.tsv dataxl -from tsv -to yaml < people.tsv ``` +## データのマージ + +主入力の各レコードを保持したまま、同じキーを持つ第2入力のフィールドを追加できます。 +次の例は `people.csv` の `id` と `details.json` の `id` を照合し、JSON側の +フィールドを追加してYAMLへ出力します。 + +```sh +dataxl -i people.csv -merge details.json -on id -to yaml +``` + +マージはleft joinです。主入力にだけ存在するレコードはそのまま残り、第2入力にだけ +存在するレコードは出力しません。`-on` には `user.id` や +`items[0].sku` のような列パスも指定できます。 + +データを暗黙に上書きしないため、次の場合はエラーになります。 + +- いずれかのレコードにマージキーがない、またはキーがscalarでない +- 第2入力に同じキーが複数ある +- 同名フィールドの値が異なる + +入力形式は各ファイル名から推定します。第2入力がstdinや拡張子なしの場合は +`-merge-from` を指定してください。主入力と第2入力を同時にstdinから読むことは +できません。 + ## Excelとの連携 Excelへ貼り付ける場合はTSVが便利です。 @@ -174,6 +198,9 @@ CSV、TSV、XLSXから構造化形式へ戻す際は、セル文字列を次の - `-to`: 出力形式。`json`, `yaml`, `toml`, `csv`, `tsv`, `xlsx`, `markdown`。 - `-sheet`: XLSXの読み書きに使うシート名。既定値は `Sheet1`。 - `-pretty`: JSONなどの構造化出力を整形するか。既定値は `true`。 +- `-merge`: 主入力へフィールドを追加する第2入力ファイル。 +- `-merge-from`: 第2入力の形式。通常は `-merge` の拡張子から推定。 +- `-on`: マージ時に両入力を照合する列パス。 - `-version`: バージョンを表示して終了します。 ## 現在の制約 diff --git a/cmd/dataxl/main.go b/cmd/dataxl/main.go index 7a50d48..738a70b 100644 --- a/cmd/dataxl/main.go +++ b/cmd/dataxl/main.go @@ -14,13 +14,16 @@ var buildVersion = "dev" // options contains CLI concerns only. Conversion functions receive explicit // arguments so they can be reused and tested without constructing a FlagSet. type options struct { - inFile string - outFile string - from string - to string - sheet string - pretty bool - version bool + inFile string + outFile string + from string + to string + sheet string + pretty bool + version bool + mergeFile string + mergeFrom string + mergeOn string } func main() { @@ -47,7 +50,22 @@ func run(args []string, stdin io.Reader, stdout, stderr io.Writer) error { if err != nil { return err } - output, err := convert(input, opt.from, opt.to, opt.sheet, opt.pretty) + var output []byte + if opt.mergeFile != "" { + if isStdinPath(opt.inFile) && isStdinPath(opt.mergeFile) { + return fmt.Errorf("primary and merge inputs cannot both use stdin") + } + mergeInput, readErr := readInput(opt.mergeFile, stdin) + if readErr != nil { + return readErr + } + output, err = mergeAndConvert( + input, opt.from, mergeInput, opt.mergeFrom, opt.mergeOn, + opt.to, opt.sheet, opt.pretty, + ) + } else { + output, err = convert(input, opt.from, opt.to, opt.sheet, opt.pretty) + } if err != nil { return err } @@ -65,11 +83,15 @@ func parseOptions(args []string, stderr io.Writer) (options, error) { fs.StringVar(&opt.sheet, "sheet", "Sheet1", "worksheet name for xlsx input/output") fs.BoolVar(&opt.pretty, "pretty", true, "pretty-print structured output") fs.BoolVar(&opt.version, "version", false, "print version and exit") + fs.StringVar(&opt.mergeFile, "merge", "", "second input file used to enrich primary records") + fs.StringVar(&opt.mergeFrom, "merge-from", "", "second input format; inferred from -merge when omitted") + fs.StringVar(&opt.mergeOn, "on", "", "column path used to match records when -merge is set") fs.Usage = func() { _, _ = fmt.Fprintln(stderr, `Usage: dataxl -from yaml -to tsv -i input.yaml -o output.tsv dataxl -from tsv -to json < paste-from-excel.tsv dataxl -from json -to xlsx -i data.json -o data.xlsx + dataxl -i people.csv -merge details.json -on id -to yaml Formats: json, yaml/yml, toml, csv, tsv, xlsx, markdown/md @@ -102,6 +124,7 @@ func resolvedVersion() string { func (opt *options) resolveFormats() error { opt.from = normalizeFormat(opt.from) opt.to = normalizeFormat(opt.to) + opt.mergeFrom = normalizeFormat(opt.mergeFrom) if opt.from == "" { opt.from = inferFormat(opt.inFile) } @@ -114,7 +137,32 @@ func (opt *options) resolveFormats() error { if err := validateFormat(opt.from); err != nil { return err } - return validateFormat(opt.to) + if err := validateFormat(opt.to); err != nil { + return err + } + if opt.mergeFile == "" { + if opt.mergeFrom != "" || opt.mergeOn != "" { + return fmt.Errorf("-merge-from and -on require -merge") + } + return nil + } + if opt.mergeFrom == "" { + opt.mergeFrom = inferFormat(opt.mergeFile) + } + if opt.mergeFrom == "" { + return fmt.Errorf("-merge-from is required when the second input format cannot be inferred") + } + if err := validateFormat(opt.mergeFrom); err != nil { + return fmt.Errorf("merge input: %w", err) + } + if strings.TrimSpace(opt.mergeOn) == "" { + return fmt.Errorf("-on is required when -merge is set") + } + return nil +} + +func isStdinPath(path string) bool { + return path == "" || path == "-" } func readInput(path string, stdin io.Reader) ([]byte, error) { diff --git a/cmd/dataxl/merge.go b/cmd/dataxl/merge.go new file mode 100644 index 0000000..a32fb95 --- /dev/null +++ b/cmd/dataxl/merge.go @@ -0,0 +1,255 @@ +package main + +import ( + "encoding/json" + "fmt" + "math" + "math/big" + "reflect" + "strconv" +) + +// mergeAndConvert parses two record sets, enriches primary records with fields +// from matching secondary records, and encodes the result in the requested +// output format. Primary order and unmatched primary records are preserved. +func mergeAndConvert( + primary []byte, + primaryFormat string, + secondary []byte, + secondaryFormat string, + on string, + to string, + sheet string, + pretty bool, +) ([]byte, error) { + primaryRecords, err := parseMergeRecords(primary, primaryFormat, sheet) + if err != nil { + return nil, fmt.Errorf("primary input: %w", err) + } + secondaryRecords, err := parseMergeRecords(secondary, secondaryFormat, sheet) + if err != nil { + return nil, fmt.Errorf("merge input: %w", err) + } + merged, err := mergeRecords(primaryRecords, secondaryRecords, on) + if err != nil { + return nil, err + } + return encodeMergedRecords(merged, to, sheet, pretty) +} + +func parseMergeRecords(input []byte, format, sheet string) ([]map[string]any, error) { + if isTabular(format) { + t, err := parseTable(input, format, sheet) + if err != nil { + return nil, err + } + return tableToRecords(t) + } + value, err := parseStructured(input, format) + if err != nil { + return nil, err + } + values := recordsFromValue(value) + records := make([]map[string]any, 0, len(values)) + for i, value := range values { + record, ok := value.(map[string]any) + if !ok { + return nil, fmt.Errorf("record %d is %T; merge inputs require object records", i+1, value) + } + records = append(records, record) + } + return records, nil +} + +func encodeMergedRecords(records []map[string]any, format, sheet string, pretty bool) ([]byte, error) { + values := make([]any, len(records)) + for i, record := range records { + values[i] = record + } + if !isTabular(format) { + return encodeStructured(values, format, pretty) + } + t := valueToTable(values) + if len(t.Header) == 0 && len(t.Rows) > 0 { + return nil, fmt.Errorf("merged records have no scalar fields to represent as a table") + } + return encodeTable(t, format, sheet) +} + +func mergeRecords(primary, secondary []map[string]any, on string) ([]map[string]any, error) { + tokens, err := parsePath(on) + if err != nil { + return nil, fmt.Errorf("invalid merge path %q: %w", on, err) + } + lookup := make(map[string]map[string]any, len(secondary)) + for i, record := range secondary { + value, err := valueAtPath(record, tokens) + if err != nil { + return nil, fmt.Errorf("merge record %d path %q: %w", i+1, on, err) + } + key, err := mergeKey(value) + if err != nil { + return nil, fmt.Errorf("merge record %d path %q: %w", i+1, on, err) + } + if _, exists := lookup[key]; exists { + return nil, fmt.Errorf("merge input has duplicate key %v at record %d", value, i+1) + } + lookup[key] = record + } + + merged := make([]map[string]any, 0, len(primary)) + for i, record := range primary { + value, err := valueAtPath(record, tokens) + if err != nil { + return nil, fmt.Errorf("primary record %d path %q: %w", i+1, on, err) + } + key, err := mergeKey(value) + if err != nil { + return nil, fmt.Errorf("primary record %d path %q: %w", i+1, on, err) + } + result := cloneMap(record) + if addition, ok := lookup[key]; ok { + if err := mergeMap(result, addition, ""); err != nil { + return nil, fmt.Errorf("primary record %d: %w", i+1, err) + } + } + merged = append(merged, result) + } + return merged, nil +} + +func valueAtPath(root map[string]any, tokens []pathToken) (any, error) { + var current any = root + for _, token := range tokens { + if token.isIndex { + values, ok := current.([]any) + if !ok { + return nil, fmt.Errorf("expected an array at index [%d], found %T", token.index, current) + } + if token.index >= len(values) { + return nil, fmt.Errorf("array index [%d] does not exist", token.index) + } + current = values[token.index] + continue + } + object, ok := current.(map[string]any) + if !ok { + return nil, fmt.Errorf("expected an object before key %q, found %T", token.key, current) + } + value, exists := object[token.key] + if !exists { + return nil, fmt.Errorf("key %q does not exist", token.key) + } + current = value + } + return current, nil +} + +func mergeKey(value any) (string, error) { + switch typed := value.(type) { + case string: + return "string:" + typed, nil + case bool: + return "bool:" + strconv.FormatBool(typed), nil + case json.Number: + return rationalNumberKey(typed.String()) + case int: + return rationalNumberKey(strconv.FormatInt(int64(typed), 10)) + case int8: + return rationalNumberKey(strconv.FormatInt(int64(typed), 10)) + case int16: + return rationalNumberKey(strconv.FormatInt(int64(typed), 10)) + case int32: + return rationalNumberKey(strconv.FormatInt(int64(typed), 10)) + case int64: + return rationalNumberKey(strconv.FormatInt(typed, 10)) + case uint: + return rationalNumberKey(strconv.FormatUint(uint64(typed), 10)) + case uint8: + return rationalNumberKey(strconv.FormatUint(uint64(typed), 10)) + case uint16: + return rationalNumberKey(strconv.FormatUint(uint64(typed), 10)) + case uint32: + return rationalNumberKey(strconv.FormatUint(uint64(typed), 10)) + case uint64: + return rationalNumberKey(strconv.FormatUint(typed, 10)) + case float32: + return finiteFloatKey(float64(typed)) + case float64: + return finiteFloatKey(typed) + case nil: + return "", fmt.Errorf("merge key is null") + default: + return "", fmt.Errorf("merge key must be a string, number, or boolean, found %T", value) + } +} + +func finiteFloatKey(value float64) (string, error) { + if math.IsNaN(value) || math.IsInf(value, 0) { + return "", fmt.Errorf("merge key must be a finite number") + } + return rationalNumberKey(strconv.FormatFloat(value, 'g', -1, 64)) +} + +func rationalNumberKey(value string) (string, error) { + rational, ok := new(big.Rat).SetString(value) + if !ok { + return "", fmt.Errorf("value %q is not a valid number", value) + } + return "number:" + rational.RatString(), nil +} + +func mergeMap(destination, source map[string]any, prefix string) error { + for key, incoming := range source { + path := appendPathKey(prefix, key) + existing, exists := destination[key] + if !exists { + destination[key] = cloneValue(incoming) + continue + } + existingMap, existingIsMap := existing.(map[string]any) + incomingMap, incomingIsMap := incoming.(map[string]any) + if existingIsMap && incomingIsMap { + if err := mergeMap(existingMap, incomingMap, path); err != nil { + return err + } + continue + } + if !mergeValuesEqual(existing, incoming) { + return fmt.Errorf("merge field %q conflicts: primary=%v merge=%v", path, existing, incoming) + } + } + return nil +} + +func mergeValuesEqual(first, second any) bool { + if reflect.DeepEqual(first, second) { + return true + } + firstKey, firstErr := mergeKey(first) + secondKey, secondErr := mergeKey(second) + return firstErr == nil && secondErr == nil && firstKey == secondKey +} + +func cloneMap(value map[string]any) map[string]any { + cloned := make(map[string]any, len(value)) + for key, child := range value { + cloned[key] = cloneValue(child) + } + return cloned +} + +func cloneValue(value any) any { + switch typed := value.(type) { + case map[string]any: + return cloneMap(typed) + case []any: + cloned := make([]any, len(typed)) + for i, child := range typed { + cloned[i] = cloneValue(child) + } + return cloned + default: + return typed + } +} diff --git a/cmd/dataxl/merge_test.go b/cmd/dataxl/merge_test.go new file mode 100644 index 0000000..a227687 --- /dev/null +++ b/cmd/dataxl/merge_test.go @@ -0,0 +1,134 @@ +package main + +import ( + "bytes" + "encoding/json" + "strings" + "testing" +) + +func TestMergeCLIEnrichesPrimaryRecords(t *testing.T) { + t.Parallel() + primary := "id,name\n1,Alice\n2,Bob\n3,Carol\n" + details := "[{\"id\":1,\"contact\":{\"email\":\"alice@example.test\"},\"active\":true},{\"id\":2,\"contact\":{\"email\":\"bob@example.test\"},\"active\":false},{\"id\":99,\"ignored\":\"no primary match\"}]" + detailsPath := t.TempDir() + "/details.json" + if err := writeOutput(detailsPath, ioDiscard{}, []byte(details)); err != nil { + t.Fatal(err) + } + + var out bytes.Buffer + err := run( + []string{"-from", "csv", "-to", "json", "-merge", detailsPath, "-on", "id", "-pretty=false"}, + strings.NewReader(primary), &out, ioDiscard{}, + ) + if err != nil { + t.Fatal(err) + } + var got []map[string]any + if err := json.Unmarshal(out.Bytes(), &got); err != nil { + t.Fatal(err) + } + if len(got) != 3 { + t.Fatalf("record count = %d, want 3", len(got)) + } + if got[0]["active"] != true || got[1]["active"] != false { + t.Fatalf("merged active fields = %#v, %#v", got[0]["active"], got[1]["active"]) + } + if _, exists := got[2]["active"]; exists { + t.Fatalf("unmatched primary record was modified: %#v", got[2]) + } + contact := got[0]["contact"].(map[string]any) + if contact["email"] != "alice@example.test" { + t.Fatalf("nested merged field = %#v", contact) + } +} + +func TestMergeRecordsSupportsNestedKeyAndFields(t *testing.T) { + t.Parallel() + primary := []map[string]any{{"user": map[string]any{"id": "001", "name": "Alice"}}} + secondary := []map[string]any{{"user": map[string]any{"id": "001", "role": "admin"}}} + got, err := mergeRecords(primary, secondary, "user.id") + if err != nil { + t.Fatal(err) + } + user := got[0]["user"].(map[string]any) + if user["name"] != "Alice" || user["role"] != "admin" { + t.Fatalf("merged user = %#v", user) + } + if _, exists := primary[0]["user"].(map[string]any)["role"]; exists { + t.Fatal("merge mutated primary input") + } +} + +func TestMergeRecordsRejectsDuplicateLookupKey(t *testing.T) { + t.Parallel() + _, err := mergeRecords( + []map[string]any{{"id": int64(1)}}, + []map[string]any{{"id": int64(1)}, {"id": int64(1)}}, + "id", + ) + if err == nil || !strings.Contains(err.Error(), "duplicate key") { + t.Fatalf("error = %v, want duplicate-key error", err) + } +} + +func TestMergeRecordsRejectsConflictingField(t *testing.T) { + t.Parallel() + _, err := mergeRecords( + []map[string]any{{"id": int64(1), "name": "Alice"}}, + []map[string]any{{"id": int64(1), "name": "Alicia"}}, + "id", + ) + if err == nil || !strings.Contains(err.Error(), "merge field \"name\" conflicts") { + t.Fatalf("error = %v, want field-conflict error", err) + } +} + +func TestMergeRecordsMatchesNumericKeysAcrossInputTypes(t *testing.T) { + t.Parallel() + got, err := mergeRecords( + []map[string]any{{"id": int64(1)}}, + []map[string]any{{"id": json.Number("1.0"), "label": "one"}}, + "id", + ) + if err != nil { + t.Fatal(err) + } + if got[0]["label"] != "one" { + t.Fatalf("merged record = %#v", got[0]) + } +} + +func TestMergeRecordsPreservesLargeNumericKeyPrecision(t *testing.T) { + t.Parallel() + got, err := mergeRecords( + []map[string]any{{"id": uint64(18_446_744_073_709_551_615)}}, + []map[string]any{{"id": json.Number("18446744073709551615"), "label": "max"}}, + "id", + ) + if err != nil { + t.Fatal(err) + } + if got[0]["label"] != "max" { + t.Fatalf("merged record = %#v", got[0]) + } +} + +func TestMergeOptionsRequireJoinPathAndFormat(t *testing.T) { + t.Parallel() + opt, err := parseOptions([]string{"-from", "csv", "-to", "json", "-merge", "-"}, ioDiscard{}) + if err != nil { + t.Fatal(err) + } + if err := opt.resolveFormats(); err == nil || !strings.Contains(err.Error(), "-merge-from is required") { + t.Fatalf("error = %v, want merge-format error", err) + } + + opt, err = parseOptions([]string{"-from", "csv", "-to", "json", "-merge", "lookup.json"}, ioDiscard{}) + if err != nil { + t.Fatal(err) + } + if err := opt.resolveFormats(); err == nil || !strings.Contains(err.Error(), "-on is required") { + t.Fatalf("error = %v, want join-path error", err) + } +} diff --git a/docs/architecture.md b/docs/architecture.md index d311a7f..5f489dd 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -21,6 +21,7 @@ - `structured.go`: JSON/YAML/TOML adapter - `table.go`: CSV/TSV/XLSX/Markdown adapter、table model、flatten - `path.go`: セル値の型推定、列パスのparse、unflatten +- `merge.go`: record集合のparse、left join、競合検出、マージ結果のencode `convert` はファイル入出力から独立しているため、CLIを経由せず変換matrixをテストできます。 形式固有処理は `parseStructured` / `encodeStructured` または @@ -121,6 +122,20 @@ array-index = "[", digit, { digit }, "]"; - recordが存在するのにscalar fieldが1つもないstructured valueは、表へ変換すると record数を失うため拒否します。空のrecord listは空の表として扱います。 +## Record merge + +`-merge` は主入力をleft side、第2入力をlookup sideとするleft joinです。両入力を +object recordの配列へ正規化し、`-on` の列パスからscalar keyを読み取ります。 +主入力の順序と未一致recordを保持し、第2入力にしかないrecordは出力しません。 + +第2入力のkeyは一意でなければなりません。追加フィールドはmapを再帰的に統合しますが、 +既存値と異なる同名フィールドは上書きせずエラーにします。数値keyおよび同名の数値値は、 +CSV由来のintegerとJSON由来のnumberのように具体的なGo型が異なっても同じ数値なら一致と +みなします。文字列 `"1"` と数値 `1` は別keyです。 + +両入力を同じstdinから取得することはできません。片方だけstdinにする場合、もう片方は +ファイルで指定します。 + ## TOML Output TOMLはトップレベル配列を直接表せないため、表からTOMLへ出力する場合など、 diff --git a/docs/development.md b/docs/development.md index 4c4316f..a5eb1a0 100644 --- a/docs/development.md +++ b/docs/development.md @@ -69,6 +69,7 @@ The current tests cover: - wrapper arrays with sibling metadata - duplicate, blank, malformed, and conflicting headers - nested arrays and JSON-quoted path keys +- cross-format left join, nested merge paths, unmatched rows, duplicate keys, and field conflicts When adding a new format or path rule, add tests around both directions where possible.