From 5ef1d72f16608248aa38fd1fb3dec0144486c4e5 Mon Sep 17 00:00:00 2001 From: Matt Shin Date: Mon, 21 Jul 2025 16:52:38 +0100 Subject: [PATCH 1/2] Dump with no aliases and remove root underscore Dump without aliases, i.e., expand out references. Option to dump without root level sub-object underscore. --- .github/workflows/pythonapp.yml | 1 + docs/cli.rst | 4 ++ docs/conf.py | 7 --- docs/data-process.rst | 16 ++++++ environment.yml | 2 +- pyproject.toml | 4 +- src/yamlprocessor/dataprocess.py | 19 +++++- src/yamlprocessor/tests/test_dataprocess.py | 64 +++++++++++++++------ 8 files changed, 91 insertions(+), 26 deletions(-) diff --git a/.github/workflows/pythonapp.yml b/.github/workflows/pythonapp.yml index 6aeb375..7b37531 100644 --- a/.github/workflows/pythonapp.yml +++ b/.github/workflows/pythonapp.yml @@ -1,3 +1,4 @@ +--- name: Python application on: diff --git a/docs/cli.rst b/docs/cli.rst index 4a67d6f..7dc0215 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -55,6 +55,10 @@ See :doc:`data-process` for detail. Do not process variable substitutions. +.. option:: --no-remove-root-underscore + + Turn off removal of root level sub-object with an underscore key. + .. option:: --schema-prefix=PREFIX Prefix for relative path schemas. See also :envvar:`YP_SCHEMA_PREFIX`. diff --git a/docs/conf.py b/docs/conf.py index 20ca9e9..6dbe1a1 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -14,8 +14,6 @@ # import sys # sys.path.insert(0, os.path.abspath('.')) -import sphinx_rtd_theme - import yamlprocessor @@ -64,11 +62,6 @@ # html_theme = 'sphinx_rtd_theme' -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". -html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] - intersphinx_mapping = { 'python': ('http://python.readthedocs.org/en/latest/', None), } diff --git a/docs/data-process.rst b/docs/data-process.rst index 2246c4b..9b8a179 100644 --- a/docs/data-process.rst +++ b/docs/data-process.rst @@ -634,6 +634,22 @@ you can do: :py:class:`yamlprocessor.dataprocess.DataProcessor` instance to ``False``. +Turn Off Removal of Root Level Sub-object with Underscore Key +------------------------------------------------------------- + +By default, if the document is an object at root and if the root object +contains a value with an underscore ``_`` key, the application will remove the +value with the underscore key before dumping the result To turn off this +behaviour, you can do: + + - On the command line, use the + :option:`--no-remove-root-underscore ` + option. + - In Python, set the :py:attr:`.is_remove_root_underscore` attribute of the + relevant :py:class:`yamlprocessor.dataprocess.DataProcessor` instance to + ``True``. + + Validation with JSON Schema --------------------------- diff --git a/environment.yml b/environment.yml index a688058..dae9da2 100644 --- a/environment.yml +++ b/environment.yml @@ -2,7 +2,7 @@ channels: - conda-forge - defaults dependencies: - - python>=3.7 + - python>=3.9 - doc8 - flake8 - jmespath diff --git a/pyproject.toml b/pyproject.toml index 64ffbef..334baec 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ description = "Process values in YAML files" readme = "README.md" -requires-python = ">=3.11" +requires-python = ">=3.9" license = {file = "LICENSE"} @@ -22,6 +22,8 @@ classifiers = [ "License :: OSI Approved :: BSD License", "Operating System :: OS Independent", "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", diff --git a/src/yamlprocessor/dataprocess.py b/src/yamlprocessor/dataprocess.py index 8d35813..fff788e 100755 --- a/src/yamlprocessor/dataprocess.py +++ b/src/yamlprocessor/dataprocess.py @@ -187,7 +187,10 @@ class DataProcessor: .. py:attribute:: .is_process_variable :type: bool - Turn on/off variable substitution. + .. py:attribute:: .is_remove_root_underscore + :type: bool + + Turn on/off removal of root level sub-object with an underscore key. .. py:attribute:: .include_dict :type: dict @@ -305,6 +308,7 @@ class DataProcessor: def __init__(self): self.is_process_include = True self.is_process_variable = True + self.is_remove_root_underscore = True self.include_paths = list( item for item in os.getenv('YP_INCLUDE_PATH', '').split(os.pathsep) @@ -447,6 +451,7 @@ def process_data( if isinstance(p_item, dict) or isinstance(p_item, list): stack.append( [p_item, parent_filenames_x, variable_map_x]) + # Finally dump data if out_filename == '-': out_file = sys.stdout else: @@ -454,9 +459,13 @@ def process_data( yaml = YAML(typ='safe', pure=True) yaml.default_flow_style = False yaml.sort_base_mapping_type_on_output = False + yaml.representer.ignore_aliases = lambda data: True yaml.representer.add_representer( datetime, get_represent_datetime(self.time_formats[''])) + if self.is_remove_root_underscore: + with suppress(KeyError, TypeError): + del root['_'] yaml.dump(root, out_file) self.validate_data(root, out_filename, schema_location) @@ -882,6 +891,12 @@ def main(argv=None): action='store_false', default=True, help='Do not process variable substitutions') + parser.add_argument( + '--no-remove-root-underscore', + dest='is_remove_root_underscore', + action='store_false', + default=True, + help='Do not remove root level sub-object with an underscore key') parser.add_argument( '--quiet', '-q', dest='is_quiet_mode', @@ -930,6 +945,8 @@ def main(argv=None): # Set up processor processor = DataProcessor() + # Remove root level sub-object with underscore key option + processor.is_remove_root_underscore = args.is_remove_root_underscore # Include options processor.is_process_include = args.is_process_include for item in args.include_paths: diff --git a/src/yamlprocessor/tests/test_dataprocess.py b/src/yamlprocessor/tests/test_dataprocess.py index a085c6e..a31cb1e 100644 --- a/src/yamlprocessor/tests/test_dataprocess.py +++ b/src/yamlprocessor/tests/test_dataprocess.py @@ -205,7 +205,7 @@ def test_main_0(tmp_path, yaml): with infilename.open('w') as infile: yaml.dump(data, infile) outfilename = tmp_path / 'b.yaml' - main([str(infilename), str(outfilename)]) + main([str(infilename), '-o', str(outfilename)]) assert yaml.load(outfilename.open()) == data @@ -220,7 +220,7 @@ def test_main_1(capsys, tmp_path, yaml): with (infilename_1).open('w') as infile_1: yaml.dump(1, infile_1) outfilename = tmp_path / 'b.yaml' - main([str(infilename), str(outfilename)]) + main([str(infilename), '-o', str(outfilename)]) assert yaml.load(outfilename.open()) == data captured = capsys.readouterr() assert f'[INFO] < {infilename}' in captured.err.splitlines() @@ -241,7 +241,7 @@ def test_main_3(tmp_path, yaml): with (tmp_path / '3x.yaml').open('w') as infile_3x: yaml.dump([3.1, 3.14], infile_3x) outfilename = tmp_path / 'b.yaml' - main([str(infilename), str(outfilename)]) + main([str(infilename), '-o', str(outfilename)]) assert yaml.load(outfilename.open()) == data @@ -252,7 +252,7 @@ def test_main_4(tmp_path, yaml): with infilename.open('w') as infile: yaml.dump(data, infile) outfilename = tmp_path / 'b.yaml' - main(['--no-process-include', str(infilename), str(outfilename)]) + main(['--no-process-include', str(infilename), '-o', str(outfilename)]) assert yaml.load(outfilename.open()) == data @@ -269,6 +269,7 @@ def test_main_5(tmp_path, yaml): '--define=PERSON=Jo', '--unbound-placeholder=unknown', str(infilename), + '-o', str(outfilename), ]) assert yaml.load(outfilename.open()) == ['Hello Jo', 'Hello unknown'] @@ -278,6 +279,7 @@ def test_main_5(tmp_path, yaml): '--define=PERSON=Jo', '--unbound-placeholder=' + DataProcessor.UNBOUND_ORIGINAL, str(infilename), + '-o', str(outfilename), ]) assert yaml.load(outfilename.open()) == ['Hello Jo', 'Hello ${ALIEN}'] @@ -290,7 +292,7 @@ def test_main_6(tmp_path, yaml): with infilename.open('w') as infile: yaml.dump(data, infile) outfilename = tmp_path / 'b.yaml' - main(['--no-process-variable', str(infilename), str(outfilename)]) + main(['--no-process-variable', str(infilename), '-o', str(outfilename)]) assert yaml.load(outfilename.open()) == data @@ -313,7 +315,7 @@ def test_main_7(capsys, tmp_path, yaml): with (include_3x).open('w') as infile_3x: yaml.dump([3.1, 3.14], infile_3x) outfilename = tmp_path / 'b.yaml' - main(['-I', str(include_d), str(infilename), str(outfilename)]) + main(['-I', str(include_d), str(infilename), '-o', str(outfilename)]) assert yaml.load(outfilename.open()) == data captured = capsys.readouterr() assert f'[INFO] YP_INCLUDE_PATH={include_d}' in captured.err.splitlines() @@ -330,7 +332,7 @@ def test_main_8(tmp_path, yaml): with infilename.open('w') as infile: infile.write(incontent) outfilename = tmp_path / 'b.yaml' - main([str(infilename), str(outfilename)]) + main([str(infilename), '-o', str(outfilename)]) with outfilename.open() as outfile: outcontent = outfile.read() assert incontent == outcontent @@ -357,7 +359,7 @@ def test_main_9(tmp_path, yaml): with (tmp_path / '1.yaml').open('w') as infile_1: yaml.dump(data_1, infile_1) outfilename = tmp_path / 'b.yaml' - main([str(infilename), str(outfilename)]) + main([str(infilename), '-o', str(outfilename)]) assert yaml.load(outfilename.open()) == data @@ -372,7 +374,7 @@ def test_main_10(tmp_path, yaml): with infilename.open('w') as infile: yaml.dump(data, infile) outfilename = tmp_path / 'b.yaml' - main([str(infilename), str(outfilename)]) + main([str(infilename), '-o', str(outfilename)]) assert yaml.load(outfilename.open()) == { 'you-time': '2030-04-05T06:07:08Z', 'me-time': '2030-04-05T06:07:08+09:00', @@ -413,6 +415,7 @@ def test_main_11(tmp_path, yaml): '--define=NAME=earth', '--define=PEOPLE=human', str(infilename), + '-o', str(outfilename)]) assert yaml.load(outfilename.open()) == { 'hello': [ @@ -449,7 +452,7 @@ def test_main_12(tmp_path, yaml): with include_infilename.open('w') as infile: yaml.dump(more_data_2, infile) outfilename = tmp_path / 'b.yaml' - main([str(infilename), str(outfilename)]) + main([str(infilename), '-o', str(outfilename)]) assert yaml.load(outfilename.open()) == [ {'name': 'cat', 'speak': ['meow', 'miaow']}, {'name': 'dog', 'speak': ['woof', 'bark']}, @@ -478,7 +481,7 @@ def test_main_12_empty_list_1(tmp_path, yaml): with include_infilename.open('w') as infile: yaml.dump(void_data, infile) outfilename = tmp_path / 'b.yaml' - main(['--define=MATTER=stuff', str(infilename), str(outfilename)]) + main(['--define=MATTER=stuff', str(infilename), '-o', str(outfilename)]) assert yaml.load(outfilename.open()) == [ {'name': 'stuff'}, {'name': 'stuff'}, @@ -504,7 +507,7 @@ def test_main_12_empty_list_2(tmp_path, yaml): with infilename.open('w') as infile: yaml.dump(data, infile) outfilename = tmp_path / 'b.yaml' - main(['--define=MATTER=stuff', str(infilename), str(outfilename)]) + main(['--define=MATTER=stuff', str(infilename), '-o', str(outfilename)]) assert yaml.load(outfilename.open()) == [ {'name': 'stuff'}, {'name': 'stuff'}, @@ -529,7 +532,7 @@ def test_main_12_empty_dict_1(tmp_path, yaml): with include_infilename.open('w') as infile: yaml.dump(void_data, infile) outfilename = tmp_path / 'b.yaml' - main(['--define=MATTER=stuff', str(infilename), str(outfilename)]) + main(['--define=MATTER=stuff', str(infilename), '-o', str(outfilename)]) assert yaml.load(outfilename.open()) == { 'name1': 'stuff', 'name2': 'stuff', @@ -555,7 +558,7 @@ def test_main_12_empty_dict_2(tmp_path, yaml): with infilename.open('w') as infile: yaml.dump(data, infile) outfilename = tmp_path / 'b.yaml' - main(['--define=MATTER=stuff', str(infilename), str(outfilename)]) + main(['--define=MATTER=stuff', str(infilename), '-o', str(outfilename)]) assert yaml.load(outfilename.open()) == { 'name1': 'stuff', 'name2': 'stuff', @@ -586,6 +589,7 @@ def test_main_13(tmp_path, yaml): outfilename = tmp_path / 'b.yaml' main([ str(infilename), + '-o', str(outfilename), '-D', 'CAT_THINK=humans are cats', ]) @@ -625,6 +629,7 @@ def test_main_14(tmp_path, yaml): '--define=WORLD=Mars', '--define=PEOPLE=Martians', str(infilename), + '-o', str(outfilename), ]) assert yaml.load(outfilename.open()) == { @@ -656,7 +661,7 @@ def test_main_15(tmp_path, yaml): with (tmp_path / 'in_1.yaml').open('w') as infile: infile.write(yaml_1) outfilename = tmp_path / 'out.yaml' - main([str(infilename), str(outfilename)]) + main([str(infilename), '-o', str(outfilename)]) assert yaml.load(outfilename.open()) == { 'hello': { 'earth': 'sapiens', @@ -733,6 +738,7 @@ def test_main_19(tmp_path, yaml): '-DCOLOUR_A=red', '-DCOLOUR_B=yellow', str(infilename), + '-o', str(outfilename), ]) assert yaml.load(outfilename.open()) == { @@ -747,6 +753,32 @@ def test_main_19(tmp_path, yaml): 'flowers': {'tulip': 'U+1F337'}} +def test_main_20(tmp_path, yaml): + """Test main, remove underscore sub-object at root level.""" + infilename = tmp_path / 'a.yaml' + with infilename.open('w') as infile: + infile.write('_:\n') + infile.write('- &breakfast\n') + infile.write(' - egg\n') + infile.write(' - bread\n') + infile.write('food: *breakfast\n') + outfilename = tmp_path / 'b.yaml' + # Default + main([str(infilename), '-o', str(outfilename)]) + assert yaml.load(outfilename.open()) == {'food': ['egg', 'bread']} + # Don't remove root underscore + main([ + '--no-remove-root-underscore', + str(infilename), + '-o', + str(outfilename), + ]) + assert yaml.load(outfilename.open()) == { + '_': [['egg', 'bread']], + 'food': ['egg', 'bread'], + } + + def test_main_validate_1(tmp_path, capsys, yaml): """Test main, YAML with JSON schema validation.""" schema = { @@ -766,7 +798,7 @@ def test_main_validate_1(tmp_path, capsys, yaml): with infilename.open('w') as infile: infile.write(f'{prefix}{schemafilename}\n') yaml.dump({'hello': 'earth'}, infile) - main([str(infilename), str(outfilename)]) + main([str(infilename), '-o', str(outfilename)]) captured = capsys.readouterr() assert f'[INFO] ok {outfilename}' in captured.err.splitlines() # Schema specified as a file:// URL From 7c8077ee850ba9def9a043b269a6abaa6654fb13 Mon Sep 17 00:00:00 2001 From: Matt Shin Date: Tue, 22 Jul 2025 16:46:09 +0100 Subject: [PATCH 2/2] Update docs/data-process.rst Co-authored-by: Stephen Oxley --- docs/data-process.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/data-process.rst b/docs/data-process.rst index 9b8a179..f344e2c 100644 --- a/docs/data-process.rst +++ b/docs/data-process.rst @@ -639,7 +639,7 @@ Turn Off Removal of Root Level Sub-object with Underscore Key By default, if the document is an object at root and if the root object contains a value with an underscore ``_`` key, the application will remove the -value with the underscore key before dumping the result To turn off this +value with the underscore key before dumping the result. To turn off this behaviour, you can do: - On the command line, use the