Support for Python3.9 type hint syntax

.github/workflows/test-pr.yaml

@@ -144,16 +144,16 @@ jobs:
                   if ! command -v stack > /dev/null 2>&1; then
                     curl -sL "https://get.haskellstack.org/" | sh
                   fi
-            - name: Install Python 3.7
+            - name: Install Python 3.12
               if: ${{ contains(matrix.fixture, 'python') }}
               uses: actions/setup-python@v4
               with:
-                  python-version: 3.7
+                  python-version: "3.12"
 
             - name: Install Python Dependencies
               if: ${{ contains(matrix.fixture, 'python') }}
               run: |
-                  pip3.7 install mypy python-dateutil types-python-dateutil
+                  pip3.12 install mypy python-dateutil types-python-dateutil
             - name: Install flow
               if: ${{ contains(matrix.fixture, 'flow') }}
               run: |

packages/quicktype-core/src/language/Python/JSONPythonRenderer.ts

@@ -370,7 +370,15 @@ export class JSONPythonRenderer extends PythonRenderer {
                 ", ",
                 this.typingDecl("x", "Any"),
                 ")",
-                this.typeHint(" -> ", this.withTyping("List"), "[", tvar, "]"),
+                this.typeHint(
+                    " -> ",
+                    this.pyOptions.features.builtinGenerics
+                        ? "list"
+                        : this.withTyping("List"),
+                    "[",
+                    tvar,
+                    "]",
+                ),
                 ":",
             ],
             () => {
@@ -422,7 +430,9 @@ export class JSONPythonRenderer extends PythonRenderer {
                 ")",
                 this.typeHint(
                     " -> ",
-                    this.withTyping("Dict"),
+                    this.pyOptions.features.builtinGenerics
+                        ? "dict"
+                        : this.withTyping("Dict"),
                     "[str, ",
                     tvar,
                     "]",
@@ -611,7 +621,8 @@ export class JSONPythonRenderer extends PythonRenderer {
             (_integerType) => "int",
             (_doubleType) => "float",
             (_stringType) => "str",
-            (_arrayType) => "List",
+            (_arrayType) =>
+                this.pyOptions.features.builtinGenerics ? "list" : "List",
             (classType) => this.nameForNamedType(classType),
             (_mapType) => "dict",
             (enumType) => this.nameForNamedType(enumType),

packages/quicktype-core/src/language/Python/PythonRenderer.ts

@@ -1,6 +1,7 @@
 import {
     arrayIntercalate,
     iterableFirst,
+    iterableSome,
     mapSortBy,
     mapUpdateInto,
     setUnionInto,
@@ -19,17 +20,15 @@ import { defined, panic } from "../../support/Support.js";
 import type { TargetLanguage } from "../../TargetLanguage.js";
 import { followTargetType } from "../../Transformers.js";
 import {
+    ArrayType,
     type ClassProperty,
     ClassType,
     EnumType,
+    MapType,
     type Type,
     UnionType,
 } from "../../Type/index.js";
-import {
-    matchType,
-    nullableFromUnion,
-    removeNullFromUnion,
-} from "../../Type/TypeUtils.js";
+import { matchType, removeNullFromUnion } from "../../Type/TypeUtils.js";
 
 import { forbiddenPropertyNames, forbiddenTypeNames } from "./constants.js";
 import type { pythonOptions } from "./language.js";
@@ -137,56 +136,160 @@ export class PythonRenderer extends ConvenienceRenderer {
         return this.withImport("typing", name);
     }
 
-    protected namedType(t: Type): Sourcelike {
+    protected namedType(t: Type, suppressQuotes = false): Sourcelike {
         const name = this.nameForNamedType(t);
-        if (this.declaredTypes.has(t)) return name;
+        if (suppressQuotes || this.declaredTypes.has(t)) return name;
         return ["'", name, "'"];
     }
 
-    protected pythonType(t: Type, _isRootTypeDef = false): Sourcelike {
+    // Would rendering `t` as a type annotation right now require a forward
+    // reference, i.e. does it mention a named type that hasn't been declared
+    // yet?
+    private typeContainsForwardRef(t: Type): boolean {
+        const actualType = followTargetType(t);
+        if (actualType instanceof ClassType || actualType instanceof EnumType) {
+            return !this.declaredTypes.has(actualType);
+        }
+
+        if (actualType instanceof ArrayType) {
+            return this.typeContainsForwardRef(actualType.items);
+        }
+
+        if (actualType instanceof MapType) {
+            return this.typeContainsForwardRef(actualType.values);
+        }
+
+        if (actualType instanceof UnionType) {
+            return iterableSome(actualType.members, (m) =>
+                this.typeContainsForwardRef(m),
+            );
+        }
+
+        return false;
+    }
+
+    // Renders a union with PEP 604 syntax: `A | B | None`.  A quoted forward
+    // reference is not allowed as an operand of `|` (`'A' | None` is a runtime
+    // `TypeError`), so if any member needs a forward reference we quote the
+    // whole union expression instead, suppressing the quotes on the individual
+    // names.  A `" = None"` default, if required, must go outside the closing
+    // quote: `foo: 'Foo | None' = None`.
+    private pep604UnionType(
+        unionType: UnionType,
+        isRootTypeDef: boolean,
+        suppressQuotes: boolean,
+    ): Sourcelike {
+        const [hasNull, nonNulls] = removeNullFromUnion(unionType);
+        const needsQuotes =
+            !suppressQuotes &&
+            iterableSome(nonNulls, (m) => this.typeContainsForwardRef(m));
+        const quote = needsQuotes ? "'" : "";
+        const memberTypes = Array.from(nonNulls).map((m) =>
+            this.pythonType(m, false, true),
+        );
+
+        const union: Sourcelike[] = [
+            quote,
+            arrayIntercalate(" | ", memberTypes),
+        ];
+        if (hasNull !== null) {
+            union.push(" | None");
+        }
+
+        union.push(quote);
+
+        if (hasNull !== null) {
+            union.push(...this.noneDefault(isRootTypeDef));
+        }
+
+        return union;
+    }
+
+    // A `" = None"` default for a class property whose value can be `None`.
+    // Only emitted for root level type defs, otherwise we may get type defs
+    // like `List[Optional[int] = None]`, which are invalid.  Every property
+    // that gets a default must sort after all properties that don't — see
+    // `sortClassProperties`.
+    private noneDefault(isRootTypeDef: boolean): string[] {
+        if (
+            isRootTypeDef &&
+            !this.getAlphabetizeProperties() &&
+            (this.pyOptions.features.dataClasses ||
+                this.pyOptions.pydanticBaseModel)
+        ) {
+            return [" = None"];
+        }
+
+        return [];
+    }
+
+    // Does `pythonType(p.type, true)` end in a `" = None"` default?  This
+    // must mirror the `noneDefault` calls in `pythonType` exactly: nullable
+    // unions, plus `Any` and `None` typed properties — an optional `Any`
+    // stays `Any` (`null` is absorbed by it), and an optional `null`
+    // collapses to just `null`, so those also default to `None`.
+    private classPropertyHasNoneDefault(p: ClassProperty): boolean {
+        const actualType = followTargetType(p.type);
+        if (actualType instanceof UnionType) {
+            const [hasNull] = removeNullFromUnion(actualType);
+            return hasNull !== null;
+        }
+
+        return actualType.kind === "any" || actualType.kind === "null";
+    }
+
+    protected pythonType(
+        t: Type,
+        _isRootTypeDef = false,
+        suppressQuotes = false,
+    ): Sourcelike {
         const actualType = followTargetType(t);
 
         return matchType<Sourcelike>(
             actualType,
-            (_anyType) => this.withTyping("Any"),
-            (_nullType) => "None",
+            (_anyType) => [
+                this.withTyping("Any"),
+                ...this.noneDefault(_isRootTypeDef),
+            ],
+            (_nullType) => ["None", ...this.noneDefault(_isRootTypeDef)],
             (_boolType) => "bool",
             (_integerType) => "int",
             (_doubletype) => "float",
             (_stringType) => "str",
             (arrayType) => [
-                this.withTyping("List"),
+                this.pyOptions.features.builtinGenerics
+                    ? "list"
+                    : this.withTyping("List"),
                 "[",
-                this.pythonType(arrayType.items),
+                this.pythonType(arrayType.items, false, suppressQuotes),
                 "]",
             ],
-            (classType) => this.namedType(classType),
+            (classType) => this.namedType(classType, suppressQuotes),
             (mapType) => [
-                this.withTyping("Dict"),
+                this.pyOptions.features.builtinGenerics
+                    ? "dict"
+                    : this.withTyping("Dict"),
                 "[str, ",
-                this.pythonType(mapType.values),
+                this.pythonType(mapType.values, false, suppressQuotes),
                 "]",
             ],
-            (enumType) => this.namedType(enumType),
+            (enumType) => this.namedType(enumType, suppressQuotes),
             (unionType) => {
+                if (this.pyOptions.features.unionOperators) {
+                    return this.pep604UnionType(
+                        unionType,
+                        _isRootTypeDef,
+                        suppressQuotes,
+                    );
+                }
+
                 const [hasNull, nonNulls] = removeNullFromUnion(unionType);
                 const memberTypes = Array.from(nonNulls).map((m) =>
-                    this.pythonType(m),
+                    this.pythonType(m, false, suppressQuotes),
                 );
 
                 if (hasNull !== null) {
-                    const rest: string[] = [];
-                    if (
-                        !this.getAlphabetizeProperties() &&
-                        (this.pyOptions.features.dataClasses ||
-                            this.pyOptions.pydanticBaseModel) &&
-                        _isRootTypeDef
-                    ) {
-                        // Only push "= None" if this is a root level type def
-                        //   otherwise we may get type defs like List[Optional[int] = None]
-                        //   which are invalid
-                        rest.push(" = None");
-                    }
+                    const rest = this.noneDefault(_isRootTypeDef);
 
                     if (nonNulls.size > 1) {
                         this.withImport("typing", "Union");
@@ -321,13 +424,11 @@ export class PythonRenderer extends ConvenienceRenderer {
             this.pyOptions.features.dataClasses ||
             this.pyOptions.pydanticBaseModel
         ) {
-            return mapSortBy(properties, (p: ClassProperty) => {
-                return (p.type instanceof UnionType &&
-                    nullableFromUnion(p.type) !== null) ||
-                    p.isOptional
-                    ? 1
-                    : 0;
-            });
+            // Properties that get a `" = None"` default must come after all
+            // properties that don't, or the generated dataclass is invalid.
+            return mapSortBy(properties, (p: ClassProperty) =>
+                this.classPropertyHasNoneDefault(p) ? 1 : 0,
+            );
         }
 
         return super.sortClassProperties(properties, propertyNames);

packages/quicktype-core/src/language/Python/constants.ts

@@ -5,7 +5,9 @@ export const forbiddenTypeNames = [
     "None",
     "Enum",
     "List",
+    "list",
     "Dict",
+    "dict",
     "Optional",
     "Union",
     "Iterable",

packages/quicktype-core/src/language/Python/language.ts

@@ -21,20 +21,51 @@ import { JSONPythonRenderer } from "./JSONPythonRenderer.js";
 import { PythonRenderer } from "./PythonRenderer.js";
 
 export interface PythonFeatures {
+    /** PEP 585 builtin generics (`list[str]`, `dict[str, int]`), Python 3.9+ */
+    builtinGenerics: boolean;
     dataClasses: boolean;
     typeHints: boolean;
+    /** PEP 604 union operators (`str | None`), Python 3.10+ */
+    unionOperators: boolean;
 }
 
 export const pythonOptions = {
     features: new EnumOption(
         "python-version",
         "Python version",
         {
-            "3.5": { typeHints: false, dataClasses: false },
-            "3.6": { typeHints: true, dataClasses: false },
-            "3.7": { typeHints: true, dataClasses: true },
-        },
-        "3.6",
+            "3.5": {
+                typeHints: false,
+                dataClasses: false,
+                builtinGenerics: false,
+                unionOperators: false,
+            },
+            "3.6": {
+                typeHints: true,
+                dataClasses: false,
+                builtinGenerics: false,
+                unionOperators: false,
+            },
+            "3.7": {
+                typeHints: true,
+                dataClasses: true,
+                builtinGenerics: false,
+                unionOperators: false,
+            },
+            "3.9": {
+                typeHints: true,
+                dataClasses: true,
+                builtinGenerics: true,
+                unionOperators: false,
+            },
+            "3.10": {
+                typeHints: true,
+                dataClasses: true,
+                builtinGenerics: true,
+                unionOperators: true,
+            },
+        } satisfies Record<string, PythonFeatures>,
+        "3.10",
     ),
     justTypes: new BooleanOption("just-types", "Classes only", false),
     nicePropertyNames: new BooleanOption(

test/fixtures/python/run.sh

@@ -3,7 +3,7 @@
 if [ "x$QUICKTYPE_PYTHON_VERSION" = "x2.7" ] ; then
     PYTHON="python2.7"
 else
-    PYTHON="python3.7"
+    PYTHON="python3.12"
 fi
 
 "$PYTHON" "$@"

test/languages.ts

@@ -263,7 +263,13 @@ export const PythonLanguage: Language = {
         "keyword-unions.schema", // Requires more than 255 arguments
     ],
     rendererOptions: {},
-    quickTestRendererOptions: [{ "python-version": "3.5" }],
+    quickTestRendererOptions: [
+        // The default is "3.10"; keep the older feature sets covered.
+        { "python-version": "3.5" },
+        { "python-version": "3.6" },
+        { "python-version": "3.7" },
+        { "python-version": "3.9" },
+    ],
     sourceFiles: ["src/language/Python/index.ts"],
 };