Dump with no aliases and remove root underscore

.github/workflows/pythonapp.yml

@@ -1,3 +1,4 @@
+---
 name: Python application
 
 on:

docs/cli.rst

@@ -55,6 +55,10 @@ See :doc:`data-process` for detail.
 
    Do not process variable substitutions.
 
+.. option:: --no-remove-root-underscore
+
+   Turn off removal of root level sub-object with an underscore key.
+
 .. option:: --schema-prefix=PREFIX
 
    Prefix for relative path schemas. See also :envvar:`YP_SCHEMA_PREFIX`.

docs/conf.py

@@ -14,8 +14,6 @@
 # import sys
 # sys.path.insert(0, os.path.abspath('.'))
 
-import sphinx_rtd_theme
-
 import yamlprocessor
 
 
@@ -64,11 +62,6 @@
 #
 html_theme = 'sphinx_rtd_theme'
 
-# Add any paths that contain custom static files (such as style sheets) here,
-# relative to this directory. They are copied after the builtin static files,
-# so a file named "default.css" will overwrite the builtin "default.css".
-html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
-
 intersphinx_mapping = {
     'python': ('http://python.readthedocs.org/en/latest/', None),
 }

docs/data-process.rst

@@ -634,6 +634,22 @@ you can do:
    :py:class:`yamlprocessor.dataprocess.DataProcessor` instance to ``False``.
 
 
+Turn Off Removal of Root Level Sub-object with Underscore Key
+-------------------------------------------------------------
+
+By default, if the document is an object at root and if the root object
+contains a value with an underscore ``_`` key, the application will remove the
+value with the underscore key before dumping the result. To turn off this
+behaviour, you can do:
+
+ - On the command line, use the
+   :option:`--no-remove-root-underscore <yp-data --no-remove-root-underscore>`
+   option.
+ - In Python, set the :py:attr:`.is_remove_root_underscore` attribute of the
+   relevant :py:class:`yamlprocessor.dataprocess.DataProcessor` instance to
+   ``True``.
+
+
 Validation with JSON Schema
 ---------------------------
 

environment.yml

@@ -2,7 +2,7 @@ channels:
   - conda-forge
   - defaults
 dependencies:
-  - python>=3.7
+  - python>=3.9
   - doc8
   - flake8
   - jmespath

pyproject.toml

@@ -7,7 +7,7 @@ description = "Process values in YAML files"
 
 readme = "README.md"
 
-requires-python = ">=3.11"
+requires-python = ">=3.9"
 
 license = {file = "LICENSE"}
 
@@ -22,6 +22,8 @@ classifiers = [
   "License :: OSI Approved :: BSD License",
   "Operating System :: OS Independent",
   "Programming Language :: Python :: 3",
+  "Programming Language :: Python :: 3.9",
+  "Programming Language :: Python :: 3.10",
   "Programming Language :: Python :: 3.11",
   "Programming Language :: Python :: 3.12",
   "Programming Language :: Python :: 3.13",

src/yamlprocessor/dataprocess.py

@@ -187,7 +187,10 @@ class DataProcessor:
     .. py:attribute:: .is_process_variable
        :type: bool
 
-       Turn on/off variable substitution.
+    .. py:attribute:: .is_remove_root_underscore
+       :type: bool
+
+       Turn on/off removal of root level sub-object with an underscore key.
 
     .. py:attribute:: .include_dict
        :type: dict
@@ -305,6 +308,7 @@ class DataProcessor:
     def __init__(self):
         self.is_process_include = True
         self.is_process_variable = True
+        self.is_remove_root_underscore = True
         self.include_paths = list(
             item
             for item in os.getenv('YP_INCLUDE_PATH', '').split(os.pathsep)
@@ -447,16 +451,21 @@ def process_data(
                     if isinstance(p_item, dict) or isinstance(p_item, list):
                         stack.append(
                             [p_item, parent_filenames_x, variable_map_x])
+        # Finally dump data
         if out_filename == '-':
             out_file = sys.stdout
         else:
             out_file = open(out_filename, 'w')
         yaml = YAML(typ='safe', pure=True)
         yaml.default_flow_style = False
         yaml.sort_base_mapping_type_on_output = False
+        yaml.representer.ignore_aliases = lambda data: True
         yaml.representer.add_representer(
             datetime,
             get_represent_datetime(self.time_formats['']))
+        if self.is_remove_root_underscore:
+            with suppress(KeyError, TypeError):
+                del root['_']
         yaml.dump(root, out_file)
         self.validate_data(root, out_filename, schema_location)
 
@@ -882,6 +891,12 @@ def main(argv=None):
         action='store_false',
         default=True,
         help='Do not process variable substitutions')
+    parser.add_argument(
+        '--no-remove-root-underscore',
+        dest='is_remove_root_underscore',
+        action='store_false',
+        default=True,
+        help='Do not remove root level sub-object with an underscore key')
     parser.add_argument(
         '--quiet', '-q',
         dest='is_quiet_mode',
@@ -930,6 +945,8 @@ def main(argv=None):
 
     # Set up processor
     processor = DataProcessor()
+    # Remove root level sub-object with underscore key option
+    processor.is_remove_root_underscore = args.is_remove_root_underscore
     # Include options
     processor.is_process_include = args.is_process_include
     for item in args.include_paths:

src/yamlprocessor/tests/test_dataprocess.py

@@ -205,7 +205,7 @@ def test_main_0(tmp_path, yaml):
     with infilename.open('w') as infile:
         yaml.dump(data, infile)
     outfilename = tmp_path / 'b.yaml'
-    main([str(infilename), str(outfilename)])
+    main([str(infilename), '-o', str(outfilename)])
     assert yaml.load(outfilename.open()) == data
 
 
@@ -220,7 +220,7 @@ def test_main_1(capsys, tmp_path, yaml):
     with (infilename_1).open('w') as infile_1:
         yaml.dump(1, infile_1)
     outfilename = tmp_path / 'b.yaml'
-    main([str(infilename), str(outfilename)])
+    main([str(infilename), '-o', str(outfilename)])
     assert yaml.load(outfilename.open()) == data
     captured = capsys.readouterr()
     assert f'[INFO] < {infilename}' in captured.err.splitlines()
@@ -241,7 +241,7 @@ def test_main_3(tmp_path, yaml):
     with (tmp_path / '3x.yaml').open('w') as infile_3x:
         yaml.dump([3.1, 3.14], infile_3x)
     outfilename = tmp_path / 'b.yaml'
-    main([str(infilename), str(outfilename)])
+    main([str(infilename), '-o', str(outfilename)])
     assert yaml.load(outfilename.open()) == data
 
 
@@ -252,7 +252,7 @@ def test_main_4(tmp_path, yaml):
     with infilename.open('w') as infile:
         yaml.dump(data, infile)
     outfilename = tmp_path / 'b.yaml'
-    main(['--no-process-include', str(infilename), str(outfilename)])
+    main(['--no-process-include', str(infilename), '-o', str(outfilename)])
     assert yaml.load(outfilename.open()) == data
 
 
@@ -269,6 +269,7 @@ def test_main_5(tmp_path, yaml):
         '--define=PERSON=Jo',
         '--unbound-placeholder=unknown',
         str(infilename),
+        '-o',
         str(outfilename),
     ])
     assert yaml.load(outfilename.open()) == ['Hello Jo', 'Hello unknown']
@@ -278,6 +279,7 @@ def test_main_5(tmp_path, yaml):
         '--define=PERSON=Jo',
         '--unbound-placeholder=' + DataProcessor.UNBOUND_ORIGINAL,
         str(infilename),
+        '-o',
         str(outfilename),
     ])
     assert yaml.load(outfilename.open()) == ['Hello Jo', 'Hello ${ALIEN}']
@@ -290,7 +292,7 @@ def test_main_6(tmp_path, yaml):
     with infilename.open('w') as infile:
         yaml.dump(data, infile)
     outfilename = tmp_path / 'b.yaml'
-    main(['--no-process-variable', str(infilename), str(outfilename)])
+    main(['--no-process-variable', str(infilename), '-o', str(outfilename)])
     assert yaml.load(outfilename.open()) == data
 
 
@@ -313,7 +315,7 @@ def test_main_7(capsys, tmp_path, yaml):
     with (include_3x).open('w') as infile_3x:
         yaml.dump([3.1, 3.14], infile_3x)
     outfilename = tmp_path / 'b.yaml'
-    main(['-I', str(include_d), str(infilename), str(outfilename)])
+    main(['-I', str(include_d), str(infilename), '-o', str(outfilename)])
     assert yaml.load(outfilename.open()) == data
     captured = capsys.readouterr()
     assert f'[INFO] YP_INCLUDE_PATH={include_d}' in captured.err.splitlines()
@@ -330,7 +332,7 @@ def test_main_8(tmp_path, yaml):
     with infilename.open('w') as infile:
         infile.write(incontent)
     outfilename = tmp_path / 'b.yaml'
-    main([str(infilename), str(outfilename)])
+    main([str(infilename), '-o', str(outfilename)])
     with outfilename.open() as outfile:
         outcontent = outfile.read()
     assert incontent == outcontent
@@ -357,7 +359,7 @@ def test_main_9(tmp_path, yaml):
     with (tmp_path / '1.yaml').open('w') as infile_1:
         yaml.dump(data_1, infile_1)
     outfilename = tmp_path / 'b.yaml'
-    main([str(infilename), str(outfilename)])
+    main([str(infilename), '-o', str(outfilename)])
     assert yaml.load(outfilename.open()) == data
 
 
@@ -372,7 +374,7 @@ def test_main_10(tmp_path, yaml):
     with infilename.open('w') as infile:
         yaml.dump(data, infile)
     outfilename = tmp_path / 'b.yaml'
-    main([str(infilename), str(outfilename)])
+    main([str(infilename), '-o', str(outfilename)])
     assert yaml.load(outfilename.open()) == {
         'you-time': '2030-04-05T06:07:08Z',
         'me-time': '2030-04-05T06:07:08+09:00',
@@ -413,6 +415,7 @@ def test_main_11(tmp_path, yaml):
         '--define=NAME=earth',
         '--define=PEOPLE=human',
         str(infilename),
+        '-o',
         str(outfilename)])
     assert yaml.load(outfilename.open()) == {
         'hello': [
@@ -449,7 +452,7 @@ def test_main_12(tmp_path, yaml):
     with include_infilename.open('w') as infile:
         yaml.dump(more_data_2, infile)
     outfilename = tmp_path / 'b.yaml'
-    main([str(infilename), str(outfilename)])
+    main([str(infilename), '-o', str(outfilename)])
     assert yaml.load(outfilename.open()) == [
         {'name': 'cat', 'speak': ['meow', 'miaow']},
         {'name': 'dog', 'speak': ['woof', 'bark']},
@@ -478,7 +481,7 @@ def test_main_12_empty_list_1(tmp_path, yaml):
     with include_infilename.open('w') as infile:
         yaml.dump(void_data, infile)
     outfilename = tmp_path / 'b.yaml'
-    main(['--define=MATTER=stuff', str(infilename), str(outfilename)])
+    main(['--define=MATTER=stuff', str(infilename), '-o', str(outfilename)])
     assert yaml.load(outfilename.open()) == [
         {'name': 'stuff'},
         {'name': 'stuff'},
@@ -504,7 +507,7 @@ def test_main_12_empty_list_2(tmp_path, yaml):
         with infilename.open('w') as infile:
             yaml.dump(data, infile)
     outfilename = tmp_path / 'b.yaml'
-    main(['--define=MATTER=stuff', str(infilename), str(outfilename)])
+    main(['--define=MATTER=stuff', str(infilename), '-o', str(outfilename)])
     assert yaml.load(outfilename.open()) == [
         {'name': 'stuff'},
         {'name': 'stuff'},
@@ -529,7 +532,7 @@ def test_main_12_empty_dict_1(tmp_path, yaml):
     with include_infilename.open('w') as infile:
         yaml.dump(void_data, infile)
     outfilename = tmp_path / 'b.yaml'
-    main(['--define=MATTER=stuff', str(infilename), str(outfilename)])
+    main(['--define=MATTER=stuff', str(infilename), '-o', str(outfilename)])
     assert yaml.load(outfilename.open()) == {
         'name1': 'stuff',
         'name2': 'stuff',
@@ -555,7 +558,7 @@ def test_main_12_empty_dict_2(tmp_path, yaml):
         with infilename.open('w') as infile:
             yaml.dump(data, infile)
     outfilename = tmp_path / 'b.yaml'
-    main(['--define=MATTER=stuff', str(infilename), str(outfilename)])
+    main(['--define=MATTER=stuff', str(infilename), '-o', str(outfilename)])
     assert yaml.load(outfilename.open()) == {
         'name1': 'stuff',
         'name2': 'stuff',
@@ -586,6 +589,7 @@ def test_main_13(tmp_path, yaml):
     outfilename = tmp_path / 'b.yaml'
     main([
         str(infilename),
+        '-o',
         str(outfilename),
         '-D', 'CAT_THINK=humans are cats',
     ])
@@ -625,6 +629,7 @@ def test_main_14(tmp_path, yaml):
         '--define=WORLD=Mars',
         '--define=PEOPLE=Martians',
         str(infilename),
+        '-o',
         str(outfilename),
     ])
     assert yaml.load(outfilename.open()) == {
@@ -656,7 +661,7 @@ def test_main_15(tmp_path, yaml):
     with (tmp_path / 'in_1.yaml').open('w') as infile:
         infile.write(yaml_1)
     outfilename = tmp_path / 'out.yaml'
-    main([str(infilename), str(outfilename)])
+    main([str(infilename), '-o', str(outfilename)])
     assert yaml.load(outfilename.open()) == {
         'hello': {
             'earth': 'sapiens',
@@ -733,6 +738,7 @@ def test_main_19(tmp_path, yaml):
         '-DCOLOUR_A=red',
         '-DCOLOUR_B=yellow',
         str(infilename),
+        '-o',
         str(outfilename),
     ])
     assert yaml.load(outfilename.open()) == {
@@ -747,6 +753,32 @@ def test_main_19(tmp_path, yaml):
         'flowers': {'tulip': 'U+1F337'}}
 
 
+def test_main_20(tmp_path, yaml):
+    """Test main, remove underscore sub-object at root level."""
+    infilename = tmp_path / 'a.yaml'
+    with infilename.open('w') as infile:
+        infile.write('_:\n')
+        infile.write('- &breakfast\n')
+        infile.write('  - egg\n')
+        infile.write('  - bread\n')
+        infile.write('food: *breakfast\n')
+    outfilename = tmp_path / 'b.yaml'
+    # Default
+    main([str(infilename), '-o', str(outfilename)])
+    assert yaml.load(outfilename.open()) == {'food': ['egg', 'bread']}
+    # Don't remove root underscore
+    main([
+        '--no-remove-root-underscore',
+        str(infilename),
+        '-o',
+        str(outfilename),
+    ])
+    assert yaml.load(outfilename.open()) == {
+        '_': [['egg', 'bread']],
+        'food': ['egg', 'bread'],
+    }
+
+
 def test_main_validate_1(tmp_path, capsys, yaml):
     """Test main, YAML with JSON schema validation."""
     schema = {
@@ -766,7 +798,7 @@ def test_main_validate_1(tmp_path, capsys, yaml):
         with infilename.open('w') as infile:
             infile.write(f'{prefix}{schemafilename}\n')
             yaml.dump({'hello': 'earth'}, infile)
-        main([str(infilename), str(outfilename)])
+        main([str(infilename), '-o', str(outfilename)])
         captured = capsys.readouterr()
         assert f'[INFO] ok {outfilename}' in captured.err.splitlines()
         # Schema specified as a file:// URL