extra-files: Write a metadata file enumerating extra files

Introduces a new metadata file to track arbitrary files added during the
extra-files phase. This file is placed in the root of each tree and is
called ``extra_files.json``. It is a JSON file containing a single
object, which contains a "header" key with an object describing the
metadata, and a "data" key, which is an array of objects, where each
object represents a file. Each object contains the "file", "checksums",
and "size" keys. "file" is the relative path from the tree root to the
extra file. "checksums" is an object containing one or more checksums,
where the key is the digest type and the value of that key is the hex
digest. Finally, the size is the size of the file in bytes.

For example:
{
  "header": {"version": "1.0},
  "data": [
    {
      "file": "GPL",
      "checksums": {
        "sha256": "8177f97513213526df2cf6184d8ff986c675afb514d4e68a404010521b880643"
      },
      "size": 18092
    },
    {
      "file": "release-notes/notes.html",
      "checksums": {
        "sha256": "82b1ba8db522aadf101dca6404235fba179e559b95ea24ff39ee1e5d9a53bdcb"
      },
      "size": 1120
    }
  ]
}

Signed-off-by: Jeremy Cline <jeremy@jcline.org>
Fixes: #295

tests/test_extra_files_phase.py

@@ -81,7 +81,6 @@ class TestCopyFiles(helpers.PungiTestCase):
         compose = helpers.DummyCompose(self.topdir, {})
         cfg = {'scm': 'file', 'dir': os.path.join(self.topdir, 'src'),
                'repo': None, 'target': 'subdir'}
-
         extra_files.copy_extra_files(compose, [cfg], 'x86_64',
                                      compose.variants['Server'], mock.Mock())
 
@@ -147,6 +146,7 @@ class TestCopyFiles(helpers.PungiTestCase):
     def fake_get_file(self, scm_dict, dest, logger):
         self.scm_dict = scm_dict
         helpers.touch(os.path.join(dest, scm_dict['file']))
+        return [scm_dict['file']]
 
 
 if __name__ == "__main__":

tests/test_metadata.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python2
 # -*- coding: utf-8 -*-
-
+import json
 import mock
 import unittest
 import os
@@ -159,5 +159,58 @@ class MediaRepoTestCase(helpers.PungiTestCase):
         self.assertFalse(os.path.isfile(self.path))
 
 
+class TestWriteExtraFiles(helpers.PungiTestCase):
+
+    def setUp(self):
+        super(TestWriteExtraFiles, self).setUp()
+        self.compose = helpers.DummyCompose(self.topdir, {})
+
+    def test_write_extra_files(self):
+        """Assert metadata is written to the proper location with valid data"""
+        mock_logger = mock.Mock()
+        files = ['file1', 'file2', 'subdir/file3']
+        expected_metadata = {
+            u'header': {u'version': u'1.0'},
+            u'data': [
+                {
+                    u'file': u'file1',
+                    u'checksums': {u'sha256': u'ecdc5536f73bdae8816f0ea40726ef5e9b810d914493075903bb90623d97b1d8'},
+                    u'size': 6,
+                },
+                {
+                    u'file': u'file2',
+                    u'checksums': {u'sha256': u'67ee5478eaadb034ba59944eb977797b49ca6aa8d3574587f36ebcbeeb65f70e'},
+                    u'size': 6,
+                },
+                {
+                    u'file': u'subdir/file3',
+                    u'checksums': {u'sha256': u'52f9f0e467e33da811330cad085fdb4eaa7abcb9ebfe6001e0f5910da678be51'},
+                    u'size': 13,
+                },
+            ]
+        }
+        tree_dir = os.path.join(self.topdir, 'compose', 'Server', 'x86_64', 'os')
+        for f in files:
+            helpers.touch(os.path.join(tree_dir, f), f + '\n')
+
+        metadata_file = metadata.write_extra_files(tree_dir, files, logger=mock_logger)
+        with open(metadata_file) as metadata_fd:
+            actual_metadata = json.load(metadata_fd)
+
+        self.assertEqual(expected_metadata['header'], actual_metadata['header'])
+        self.assertEqual(expected_metadata['data'], actual_metadata['data'])
+
+    def test_write_extra_files_missing_file(self):
+        """Assert metadata is written to the proper location with valid data"""
+        mock_logger = mock.Mock()
+        files = ['file1', 'file2', 'subdir/file3']
+        tree_dir = os.path.join(self.topdir, 'compose', 'Server', 'x86_64', 'os')
+        for f in files:
+            helpers.touch(os.path.join(tree_dir, f), f + '\n')
+        files.append('missing_file')
+
+        self.assertRaises(RuntimeError, metadata.write_extra_files, tree_dir, files, 'sha256', mock_logger)
+
+
 if __name__ == "__main__":
     unittest.main()

tests/test_util.py

@@ -379,5 +379,32 @@ class TestLevenshtein(unittest.TestCase):
         self.assertEqual(util.levenshtein('kitten', 'sitting'), 3)
 
 
+class TestRecursiveFileList(unittest.TestCase):
+
+    def setUp(self):
+        self.tmp_dir = tempfile.mkdtemp()
+
+    def tearDown(self):
+        shutil.rmtree(self.tmp_dir)
+
+    def test_flat_file_list(self):
+        """Build a directory containing files and assert they are listed."""
+        expected_files = sorted(['file1', 'file2', 'file3'])
+        for expected_file in [os.path.join(self.tmp_dir, f) for f in expected_files]:
+            touch(expected_file)
+
+        actual_files = sorted(util.recursive_file_list(self.tmp_dir))
+        self.assertEqual(expected_files, actual_files)
+
+    def test_nested_file_list(self):
+        """Build a directory containing files and assert they are listed."""
+        expected_files = sorted(['file1', 'subdir/file2', 'sub/subdir/file3'])
+        for expected_file in [os.path.join(self.tmp_dir, f) for f in expected_files]:
+            touch(expected_file)
+
+        actual_files = sorted(util.recursive_file_list(self.tmp_dir))
+        self.assertEqual(expected_files, actual_files)
+
+
 if __name__ == "__main__":
     unittest.main()