Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -15,12 +15,15 @@ Read-only: `.proto`, `.env`

## Transcoding Between Formats

Any input format can be transcoded to any output format in a single command. Use `-i`/`--input` and `-o`/`--output` to specify formats explicitly, or let `qq` detect them from the file extension.
Any input format can be transcoded to any output format in a single command. Use `-i`/`--input` and `-o`/`--output` to specify formats explicitly, or let `qq` detect them from the file extension. When reading from stdin and you don't know the format up front, pass `-i auto` to detect it from the content (json, yaml, toml and xml/html).

```sh
# YAML → JSON
qq '.' config.yaml -o json

# Detect the input format from content (handy for stdin)
kubectl get pod my-pod -o yaml | qq -i auto '.metadata.name'

# TOML → YAML
qq '.' pyproject.toml -o yaml

Expand Down
13 changes: 11 additions & 2 deletions cli/qq.go
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ func CreateRootCmd() *cobra.Command {
handleCommand(cmd, args, inputType, outputType, rawOutput, help, interactive, monochrome, stream, slurp, exitStatus)
},
}
cmd.Flags().StringVarP(&inputType, "input", "i", "json", "specify input file type, only required on parsing stdin.")
cmd.Flags().StringVarP(&inputType, "input", "i", "json", "specify input file type, only required on parsing stdin. use \"auto\" to detect the format from the content.")
cmd.Flags().StringVarP(&outputType, "output", "o", "json", "specify output file type by extension name. This is inferred from extension if passing file position argument.")
cmd.Flags().BoolVarP(&rawOutput, "raw-output", "r", false, "output strings without escapes and quotes.")
cmd.Flags().BoolVarP(&help, "help", "h", false, "help for qq")
Expand Down Expand Up @@ -161,7 +161,16 @@ func handleCommand(cmd *cobra.Command, args []string, inputtype string, outputty
// Check if -i flag was explicitly set by user
inputFlagSet := cmd.Flags().Changed("input")

if inputFlagSet {
if strings.EqualFold(inputtype, "auto") {
// Content based detection, opt-in via -i auto so it never changes the
// default behaviour. Detection needs the whole input buffered, which is
// at odds with streaming, so ask the user to be explicit there.
if stream {
fmt.Println("Error: input type \"auto\" cannot be used with --stream; specify the input type explicitly")
os.Exit(1)
}
inputCodec, err = codec.Detect(input)
} else if inputFlagSet {
// -i flag takes precedence over file extension
inputCodec, err = codec.GetEncodingType(inputtype)
} else if filename != "" {
Expand Down
77 changes: 77 additions & 0 deletions codec/detect.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
package codec

import (
"bytes"
"fmt"
"reflect"

"github.com/goccy/go-json"
)

// Detect infers the encoding of input from its content. It backs the "auto"
// input type and is intentionally conservative: it only reports a format when a
// strict parse confirms it, and returns an error otherwise so callers can ask
// the user to be explicit rather than guess wrong.
//
// Detection covers the common structured text formats (json, xml/html, toml,
// yaml). Binary formats and ambiguous line based formats (csv, ini, env,
// properties, ...) are deliberately left out because their inputs overlap too
// much to tell apart reliably.
func Detect(input []byte) (EncodingType, error) {
trimmed := bytes.TrimSpace(input)
if len(trimmed) == 0 {
return JSON, fmt.Errorf("cannot detect input format: empty input")
}

switch trimmed[0] {
case '{', '[':
if json.Valid(input) {
return JSON, nil
}
case '<':
lower := bytes.ToLower(trimmed)
if bytes.HasPrefix(lower, []byte("<!doctype html")) || bytes.Contains(lower, []byte("<html")) {
return HTML, nil
}
if _, err := tryDecode(input, XML); err == nil {
return XML, nil
}
if _, err := tryDecode(input, HTML); err == nil {
return HTML, nil
}
}

// TOML is strict enough that arbitrary text will not parse, so try it before
// falling back to YAML which accepts almost anything.
if v, err := tryDecode(input, TOML); err == nil && isStructured(v) {
return TOML, nil
}

// YAML is the catch all for structured config. Require a map or sequence so
// that plain scalars and prose are not claimed as YAML.
if v, err := tryDecode(input, YAML); err == nil && isStructured(v) {
return YAML, nil
}

return JSON, fmt.Errorf("could not detect input format, please specify it with -i/--input")
}

func tryDecode(input []byte, t EncodingType) (any, error) {
var v any
if err := Codecs[t].Unmarshal(input, &v); err != nil {
return nil, err
}
return v, nil
}

func isStructured(v any) bool {
if v == nil {
return false
}
switch reflect.ValueOf(v).Kind() {
case reflect.Map, reflect.Slice, reflect.Array:
return true
default:
return false
}
}
56 changes: 56 additions & 0 deletions codec/detect_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
package codec

import "testing"

func TestDetect(t *testing.T) {
tests := []struct {
name string
input string
expected EncodingType
}{
{"json object", `{"name":"Alice","age":30}`, JSON},
{"json array", `[1,2,3]`, JSON},
{"json with leading space", " \n{\"a\":1}", JSON},
{"yaml map", "name: Alice\nage: 30\n", YAML},
{"yaml doc marker", "---\nname: Bob\n", YAML},
{"yaml list", "- one\n- two\n", YAML},
{"toml", "title = \"hi\"\n[owner]\nname = \"x\"\n", TOML},
{"xml", "<root><a>1</a></root>", XML},
{"xml with declaration", "<?xml version=\"1.0\"?><note><to>x</to></note>", XML},
{"html doctype", "<!DOCTYPE html><html><body><p>hi</p></body></html>", HTML},
{"html tag", "<html><head></head></html>", HTML},
}

for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := Detect([]byte(tt.input))
if err != nil {
t.Fatalf("Detect(%q) returned error: %v", tt.input, err)
}
if got != tt.expected {
t.Errorf("Detect(%q) = %v, want %v", tt.input, got, tt.expected)
}
})
}
}

func TestDetectUndetectable(t *testing.T) {
// Inputs that Detect should refuse rather than guess at.
cases := []struct {
name string
input string
}{
{"empty", ""},
{"whitespace only", " \n\t"},
{"plain scalar", "just some words"},
{"bare number", "42"},
}

for _, tt := range cases {
t.Run(tt.name, func(t *testing.T) {
if got, err := Detect([]byte(tt.input)); err == nil {
t.Errorf("Detect(%q) = %v, expected an error", tt.input, got)
}
})
}
}
Loading