From f1eb346e336fee7c14ed289dc854da4556c29a44 Mon Sep 17 00:00:00 2001 From: AKASHI Takahiro Date: Sat, 3 Jul 2021 15:43:17 +0900 Subject: [PATCH 01/10] env: efi: fix a wrong address dereference Probably, a pointer to a variable in an inner block should not be exposed to an outer block. Fixes: c70f44817d46 ("efi_loader: simplify 'printenv -e'") Signed-off-by: AKASHI Takahiro [trini: Don't make guid const now] Signed-off-by: Tom Rini --- cmd/nvedit_efi.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cmd/nvedit_efi.c b/cmd/nvedit_efi.c index 094c0e8098..d5e93827e4 100644 --- a/cmd/nvedit_efi.c +++ b/cmd/nvedit_efi.c @@ -241,6 +241,7 @@ int do_env_print_efi(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]) { const efi_guid_t *guid_p = NULL; + efi_guid_t guid; bool verbose = true; efi_status_t ret; @@ -254,8 +255,6 @@ int do_env_print_efi(struct cmd_tbl *cmdtp, int flag, int argc, for (argc--, argv++; argc > 0 && argv[0][0] == '-'; argc--, argv++) { if (!strcmp(argv[0], "-guid")) { - efi_guid_t guid; - if (argc == 1) return CMD_RET_USAGE; argc--; From 8f5f5d3a4593f26f0a755f5ecfaa7299e42edeeb Mon Sep 17 00:00:00 2001 From: Alper Nebi Yasak Date: Fri, 4 Jun 2021 22:04:45 +0300 Subject: [PATCH 02/10] test/py: Use loop mounts if guestmount fails in filesystem tests If guestmount isn't available on the system, filesystem test setup falls back to using loop mounts to prepare its disk images. If guestmount is available but fails to work, the tests are immediately skipped. Instead of giving up on a guestmount failure, try using loop mounts as an attempt to keep tests running. Also stop checking if guestmount is in PATH, as trying to run a missing guestmount can now follow the same failure codepath and fall back to loop mounts anyway. Signed-off-by: Alper Nebi Yasak --- test/py/tests/test_fs/conftest.py | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/test/py/tests/test_fs/conftest.py b/test/py/tests/test_fs/conftest.py index 7325486cdb..0189298f91 100644 --- a/test/py/tests/test_fs/conftest.py +++ b/test/py/tests/test_fs/conftest.py @@ -209,24 +209,23 @@ def mount_fs(fs_type, device, mount_point): """ global fuse_mounted - fuse_mounted = False try: - if tool_is_in_path('guestmount'): - fuse_mounted = True - check_call('guestmount -a %s -m /dev/sda %s' - % (device, mount_point), shell=True) - else: - mount_opt = 'loop,rw' - if re.match('fat', fs_type): - mount_opt += ',umask=0000' - - check_call('sudo mount -o %s %s %s' - % (mount_opt, device, mount_point), shell=True) - - # may not be effective for some file systems - check_call('sudo chmod a+rw %s' % mount_point, shell=True) + check_call('guestmount -a %s -m /dev/sda %s' + % (device, mount_point), shell=True) + fuse_mounted = True + return except CalledProcessError: - raise + fuse_mounted = False + + mount_opt = 'loop,rw' + if re.match('fat', fs_type): + mount_opt += ',umask=0000' + + check_call('sudo mount -o %s %s %s' + % (mount_opt, device, mount_point), shell=True) + + # may not be effective for some file systems + check_call('sudo chmod a+rw %s' % mount_point, shell=True) def umount_fs(mount_point): """Unmount a volume. From 99f5303c8b857320b00a63dca81d1f6f7c75c636 Mon Sep 17 00:00:00 2001 From: Alper Nebi Yasak Date: Fri, 4 Jun 2021 22:04:46 +0300 Subject: [PATCH 03/10] test/py: Wait for guestmount worker to exit after running guestunmount Some filesystem tests are failing when their image is prepared with guestmount, but succeeding if loop mounts are used instead. The reason seems to be a race condition the guestmount(1) manual page explains: When guestunmount(1)/fusermount(1) exits, guestmount may still be running and cleaning up the mountpoint. The disk image will not be fully finalized. This means that scripts like the following have a nasty race condition: guestmount -a disk.img -i /mnt # copy things into /mnt guestunmount /mnt # immediately try to use 'disk.img' ** UNSAFE ** The solution is to use the --pid-file option to write the guestmount PID to a file, then after guestunmount spin waiting for this PID to exit. The Python standard library has an os.waitpid() function for waiting a child to terminate, but it cannot wait on non-child processes. Implement a utility function that can do this by polling the process repeatedly for a given duration, optionally killing the process if it won't terminate on its own. Apply the suggested solution with this utility function, which makes the failing tests succeed again. Signed-off-by: Alper Nebi Yasak Reviewed-by: Simon Glass --- test/py/tests/test_fs/conftest.py | 13 ++++++++++- test/py/u_boot_utils.py | 36 +++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/test/py/tests/test_fs/conftest.py b/test/py/tests/test_fs/conftest.py index 0189298f91..b638284e07 100644 --- a/test/py/tests/test_fs/conftest.py +++ b/test/py/tests/test_fs/conftest.py @@ -8,6 +8,7 @@ import pytest import re from subprocess import call, check_call, check_output, CalledProcessError from fstest_defs import * +import u_boot_utils as util supported_fs_basic = ['fat16', 'fat32', 'ext4'] supported_fs_ext = ['fat16', 'fat32'] @@ -210,7 +211,7 @@ def mount_fs(fs_type, device, mount_point): global fuse_mounted try: - check_call('guestmount -a %s -m /dev/sda %s' + check_call('guestmount --pid-file guestmount.pid -a %s -m /dev/sda %s' % (device, mount_point), shell=True) fuse_mounted = True return @@ -239,6 +240,16 @@ def umount_fs(mount_point): if fuse_mounted: call('sync') call('guestunmount %s' % mount_point, shell=True) + + try: + with open("guestmount.pid", "r") as pidfile: + pid = int(pidfile.read()) + util.waitpid(pid, kill=True) + os.remove("guestmount.pid") + + except FileNotFoundError: + pass + else: call('sudo umount %s' % mount_point, shell=True) diff --git a/test/py/u_boot_utils.py b/test/py/u_boot_utils.py index 939d82eec1..e816c7fbb6 100644 --- a/test/py/u_boot_utils.py +++ b/test/py/u_boot_utils.py @@ -8,6 +8,7 @@ import inspect import os import os.path import pytest +import signal import sys import time import re @@ -339,3 +340,38 @@ def crc32(u_boot_console, address, count): assert m, 'CRC32 operation failed.' return m.group(1) + +def waitpid(pid, timeout=60, kill=False): + """Wait a process to terminate by its PID + + This is an alternative to a os.waitpid(pid, 0) call that works on + processes that aren't children of the python process. + + Args: + pid: PID of a running process. + timeout: Time in seconds to wait. + kill: Whether to forcibly kill the process after timeout. + + Returns: + True, if the process ended on its own. + False, if the process was killed by this function. + + Raises: + TimeoutError, if the process is still running after timeout. + """ + try: + for _ in range(timeout): + os.kill(pid, 0) + time.sleep(1) + + if kill: + os.kill(pid, signal.SIGKILL) + return False + + except ProcessLookupError: + return True + + raise TimeoutError( + "Process with PID {} did not terminate after {} seconds." + .format(pid, timeout) + ) From f9abaa53ec82613ce0f578e9742092a987e8c6e6 Mon Sep 17 00:00:00 2001 From: Alper Nebi Yasak Date: Mon, 21 Jun 2021 21:51:54 +0300 Subject: [PATCH 04/10] tools: docker: Install a readable kernel for libguestfs-tools The filesystem and EFI (capsule and secure boot) test setups try to use guestmount and virt-make-fs respectively to prepare disk images to run tests on. However, these libguestfs tools need a kernel image and fail with the following message (revealed in debug/trace mode) if it can't find one: supermin: failed to find a suitable kernel (host_cpu=x86_64). I looked for kernels in /boot and modules in /lib/modules. If this is a Xen guest, and you only have Xen domU kernels installed, try installing a fullvirt kernel (only for supermin use, you shouldn't boot the Xen guest with it). This failure then causes these tests to be skipped in CIs. Install a kernel package in the Docker containers so the CIs can run these tests with libguestfs tools again (assuming the container is run with necessary host devices and privileges). As this kernel would be only used for virtualization, we can use the kernel package specialized for that. On Ubuntu systems kernel images are not readable by non-root users, so explicitly add read permissions with chmod as well. Signed-off-by: Alper Nebi Yasak Acked-by: Heinrich Schuchardt --- tools/docker/Dockerfile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tools/docker/Dockerfile b/tools/docker/Dockerfile index d2f0074ee8..579df82b56 100644 --- a/tools/docker/Dockerfile +++ b/tools/docker/Dockerfile @@ -71,6 +71,7 @@ RUN apt-get update && apt-get install -y \ libssl-dev \ libudev-dev \ libusb-1.0-0-dev \ + linux-image-kvm \ lzma-alone \ lzop \ mount \ @@ -99,6 +100,9 @@ RUN apt-get update && apt-get install -y \ zip \ && rm -rf /var/lib/apt/lists/* +# Make kernels readable for libguestfs tools to work correctly +RUN chmod +r /boot/vmlinu* + # Manually install libmpfr4 for the toolchains RUN wget http://mirrors.kernel.org/ubuntu/pool/main/m/mpfr4/libmpfr4_3.1.4-1_amd64.deb && dpkg -i libmpfr4_3.1.4-1_amd64.deb && rm libmpfr4_3.1.4-1_amd64.deb From 1aaaf60d200c6bdc556626d586d8b57b8a86d832 Mon Sep 17 00:00:00 2001 From: Alper Nebi Yasak Date: Mon, 21 Jun 2021 21:51:55 +0300 Subject: [PATCH 05/10] Azure: Add fuse device for test.py tests The EFI secure boot and capsule test setups need to prepare disk images for their tests using virt-make-fs, which requires access to the host fuse device. This is not exposed to the docker container by default and has to be added explicitly. Add it. Signed-off-by: Alper Nebi Yasak --- .azure-pipelines.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.azure-pipelines.yml b/.azure-pipelines.yml index 221e600806..e8f3c9baa6 100644 --- a/.azure-pipelines.yml +++ b/.azure-pipelines.yml @@ -318,7 +318,8 @@ jobs: # as sandbox testing need create files like spi flash images, etc. # (TODO: clean up this in the future) chmod 777 . - docker run -v $PWD:$(work_dir) $(ci_runner_image) /bin/bash $(work_dir)/test.sh + # Some tests using libguestfs-tools need the fuse device to run + docker run --device /dev/fuse:/dev/fuse -v $PWD:$(work_dir) $(ci_runner_image) /bin/bash $(work_dir)/test.sh - job: build_the_world displayName: 'Build the World' From e22ec9c6927bb565b89556eb7dc7856790778e46 Mon Sep 17 00:00:00 2001 From: Alper Nebi Yasak Date: Mon, 21 Jun 2021 21:51:56 +0300 Subject: [PATCH 06/10] Azure: Add loop devices and CAP_SYS_ADMIN for sandbox test.py tests The filesystem test setup needs to prepare disk images for its tests, with either guestmount or loop mounts. The former requires access to the host fuse device (added in a previous patch), the latter requires access to host loop devices. Both mounts also need additional privileges since docker's default configuration prevents the containers from mounting filesystems (for host security). Add any available loop devices to the container and try to add as few privileges as possible to run these tests, which narrow down to adding SYS_ADMIN capability and disabling apparmor confinement. However, this much still seems to be insecure enough to let malicious container processes escape as root on the host system [1]. [1] https://blog.trailofbits.com/2019/07/19/understanding-docker-container-escapes/ Since the mentioned tests are marked to run only on the sandbox board, add these additional devices and privileges only when testing with that. An alternative to using mounts is modifying the filesystem tests to use virt-make-fs (like some EFI tests do), but it fails to generate a partitionless FAT filesystem image on Debian systems. Other more feasible alternatives are using guestfish or directly using libguestfs Python bindings to create and populate the images, but switching the test setups to these is nontrivial and is left as future work. Signed-off-by: Alper Nebi Yasak --- .azure-pipelines.yml | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/.azure-pipelines.yml b/.azure-pipelines.yml index e8f3c9baa6..99964be53c 100644 --- a/.azure-pipelines.yml +++ b/.azure-pipelines.yml @@ -318,8 +318,22 @@ jobs: # as sandbox testing need create files like spi flash images, etc. # (TODO: clean up this in the future) chmod 777 . + # Filesystem tests need extra docker args to run + set -- + if [[ "${TEST_PY_BD}" == "sandbox" ]]; then + # mount -o loop needs the loop devices + if modprobe loop; then + for d in $(find /dev -maxdepth 1 -name 'loop*'); do + set -- "$@" --device $d:$d + done + fi + # Needed for mount syscall (for guestmount as well) + set -- "$@" --cap-add SYS_ADMIN + # Default apparmor profile denies mounts + set -- "$@" --security-opt apparmor=unconfined + fi # Some tests using libguestfs-tools need the fuse device to run - docker run --device /dev/fuse:/dev/fuse -v $PWD:$(work_dir) $(ci_runner_image) /bin/bash $(work_dir)/test.sh + docker run "$@" --device /dev/fuse:/dev/fuse -v $PWD:$(work_dir) $(ci_runner_image) /bin/bash $(work_dir)/test.sh - job: build_the_world displayName: 'Build the World' From 04c9813e951ffa9391daec5a8dcda90539ca2989 Mon Sep 17 00:00:00 2001 From: Joao Marcos Costa Date: Wed, 30 Jun 2021 19:45:03 -0300 Subject: [PATCH 07/10] test/py: rewrite common tools for SquashFS tests Remove the previous OOP approach, which was confusing and incomplete. Add more test cases by making SquashFS images with various options, concerning file fragmentation and its compression. Add comments to properly document the code. Reviewed-by: Simon Glass Tested-by: Simon Glass [on sandbox] Signed-off-by: Joao Marcos Costa --- .../test_fs/test_squashfs/sqfs_common.py | 237 ++++++++++++++---- 1 file changed, 183 insertions(+), 54 deletions(-) diff --git a/test/py/tests/test_fs/test_squashfs/sqfs_common.py b/test/py/tests/test_fs/test_squashfs/sqfs_common.py index c96f92c1d8..267c4b57d1 100644 --- a/test/py/tests/test_fs/test_squashfs/sqfs_common.py +++ b/test/py/tests/test_fs/test_squashfs/sqfs_common.py @@ -3,74 +3,203 @@ # Author: Joao Marcos Costa import os -import random -import string +import shutil import subprocess -def sqfs_get_random_letters(size): - letters = [] - for i in range(0, size): - letters.append(random.choice(string.ascii_letters)) +""" standard test images table: Each table item is a key:value pair +representing the output image name and its respective mksquashfs options. +This table should be modified only when adding support for new compression +algorithms. The 'default' case takes no options but the input and output +names, so it must be assigned with an empty string. +""" +STANDARD_TABLE = { + 'default' : '', + 'lzo_comp_frag' : '', + 'lzo_frag' : '', + 'lzo_no_frag' : '', + 'zstd_comp_frag' : '', + 'zstd_frag' : '', + 'zstd_no_frag' : '', + 'gzip_comp_frag' : '', + 'gzip_frag' : '', + 'gzip_no_frag' : '' +} - return ''.join(letters) +""" EXTRA_TABLE: Set this table's keys and values if you want to make squashfs +images with your own customized options. +""" +EXTRA_TABLE = {} -def sqfs_generate_file(path, size): - content = sqfs_get_random_letters(size) - file = open(path, "w") +# path to source directory used to make squashfs test images +SQFS_SRC_DIR = 'sqfs_src_dir' + +def get_opts_list(): + """ Combines fragmentation and compression options into a list of strings. + + opts_list's firts item is an empty string as STANDARD_TABLE's first item is + the 'default' case. + + Returns: + A list of strings whose items are formed by a compression and a + fragmentation option joined by a whitespace. + """ + # supported compression options only + comp_opts = ['-comp lzo', '-comp zstd', '-comp gzip'] + # file fragmentation options + frag_opts = ['-always-use-fragments', '-always-use-fragments -noF', '-no-fragments'] + + opts_list = [' '] + for comp_opt in comp_opts: + for frag_opt in frag_opts: + opts_list.append(' '.join([comp_opt, frag_opt])) + + return opts_list + +def init_standard_table(): + """ Initializes STANDARD_TABLE values. + + STANDARD_TABLE's keys are pre-defined, and init_standard_table() assigns + the right value for each one of them. + """ + opts_list = get_opts_list() + + for key, value in zip(STANDARD_TABLE.keys(), opts_list): + STANDARD_TABLE[key] = value + +def generate_file(file_name, file_size): + """ Generates a file filled with 'x'. + + Args: + file_name: the file's name. + file_size: the content's length and therefore the file size. + """ + content = 'x' * file_size + + file = open(file_name, 'w') file.write(content) file.close() -class Compression: - def __init__(self, name, files, sizes, block_size = 4096): - self.name = name - self.files = files - self.sizes = sizes - self.mksquashfs_opts = " -b " + str(block_size) + " -comp " + self.name +def generate_sqfs_src_dir(build_dir): + """ Generates the source directory used to make the SquashFS images. - def add_opt(self, opt): - self.mksquashfs_opts += " " + opt + The source directory is generated at build_dir, and it has the following + structure: + sqfs_src_dir/ + ├── empty-dir/ + ├── f1000 + ├── f4096 + ├── f5096 + ├── subdir/ + │   └── subdir-file + └── sym -> subdir - def gen_image(self, build_dir): - src = os.path.join(build_dir, "sqfs_src/") - os.mkdir(src) - for (f, s) in zip(self.files, self.sizes): - sqfs_generate_file(src + f, s) + 3 directories, 4 files - # the symbolic link always targets the first file - os.symlink(self.files[0], src + "sym") + The files in the root dir. are prefixed with an 'f' followed by its size. - sqfs_img = os.path.join(build_dir, "sqfs-" + self.name) - i_o = src + " " + sqfs_img - opts = self.mksquashfs_opts - try: - subprocess.run(["mksquashfs " + i_o + opts], shell = True, check = True) - except: - print("mksquashfs error. Compression type: " + self.name) - raise RuntimeError + Args: + build_dir: u-boot's build-sandbox directory. + """ - def clean_source(self, build_dir): - src = os.path.join(build_dir, "sqfs_src/") - for f in self.files: - os.remove(src + f) - os.remove(src + "sym") - os.rmdir(src) + root = os.path.join(build_dir, SQFS_SRC_DIR) + # make root directory + os.makedirs(root) - def cleanup(self, build_dir): - self.clean_source(build_dir) - sqfs_img = os.path.join(build_dir, "sqfs-" + self.name) - os.remove(sqfs_img) + # 4096: minimum block size + file_name = 'f4096' + generate_file(os.path.join(root, file_name), 4096) -files = ["blks_only", "blks_frag", "frag_only"] -sizes = [4096, 5100, 100] -gzip = Compression("gzip", files, sizes) -zstd = Compression("zstd", files, sizes) -lzo = Compression("lzo", files, sizes) + # 5096: minimum block size + 1000 chars (fragment) + file_name = 'f5096' + generate_file(os.path.join(root, file_name), 5096) -# use fragment blocks for files larger than block_size -gzip.add_opt("-always-use-fragments") -zstd.add_opt("-always-use-fragments") + # 1000: less than minimum block size (fragment only) + file_name = 'f1000' + generate_file(os.path.join(root, file_name), 1000) -# avoid fragments if lzo is used -lzo.add_opt("-no-fragments") + # sub-directory with a single file inside + subdir_path = os.path.join(root, 'subdir') + os.makedirs(subdir_path) + generate_file(os.path.join(subdir_path, 'subdir-file'), 100) -comp_opts = [gzip, zstd, lzo] + # symlink (target: sub-directory) + os.symlink('subdir', os.path.join(root, 'sym')) + + # empty directory + os.makedirs(os.path.join(root, 'empty-dir')) + +def mksquashfs(args): + """ Runs mksquashfs command. + + Args: + args: mksquashfs options (e.g.: compression and fragmentation). + """ + subprocess.run(['mksquashfs ' + args], shell=True, check=True, + stdout=subprocess.DEVNULL) + +def get_mksquashfs_version(): + """ Parses the output of mksquashfs -version. + + Returns: + mksquashfs's version as a float. + """ + out = subprocess.run(['mksquashfs -version'], shell=True, check=True, + capture_output=True, text=True) + # 'out' is: mksquashfs version X (yyyy/mm/dd) ... + return float(out.stdout.split()[2]) + +def check_mksquashfs_version(): + """ Checks if mksquashfs meets the required version. """ + + required_version = 4.4 + if get_mksquashfs_version() < required_version: + print('Error: mksquashfs is too old.') + print('Required version: {}'.format(required_version)) + raise AssertionError + +def make_all_images(build_dir): + """ Makes the SquashFS images used in the test suite. + + The image names and respective mksquashfs options are defined in STANDARD_TABLE + and EXTRA_TABLE. The destination is defined by 'build_dir'. + + Args: + build_dir: u-boot's build-sandbox directory. + """ + + init_standard_table() + input_path = os.path.join(build_dir, SQFS_SRC_DIR) + + # make squashfs images according to STANDARD_TABLE + for out, opts in zip(STANDARD_TABLE.keys(), STANDARD_TABLE.values()): + output_path = os.path.join(build_dir, out) + mksquashfs(' '.join([input_path, output_path, opts])) + + # make squashfs images according to EXTRA_TABLE + for out, opts in zip(EXTRA_TABLE.keys(), EXTRA_TABLE.values()): + output_path = os.path.join(build_dir, out) + mksquashfs(' '.join([input_path, output_path, opts])) + +def clean_all_images(build_dir): + """ Deletes the SquashFS images at build_dir. + + Args: + build_dir: u-boot's build-sandbox directory. + """ + + for image_name in STANDARD_TABLE: + image_path = os.path.join(build_dir, image_name) + os.remove(image_path) + + for image_name in EXTRA_TABLE: + image_path = os.path.join(build_dir, image_name) + os.remove(image_path) + +def clean_sqfs_src_dir(build_dir): + """ Deletes the source directory at build_dir. + + Args: + build_dir: u-boot's build-sandbox directory. + """ + path = os.path.join(build_dir, SQFS_SRC_DIR) + shutil.rmtree(path) From 208eb2a4dcd37c0bb1b76b9d71f7bf28b85a899f Mon Sep 17 00:00:00 2001 From: Joao Marcos Costa Date: Wed, 30 Jun 2021 19:45:04 -0300 Subject: [PATCH 08/10] test/py: rewrite sqfsload command test suite The previous strategy to know if a file was correctly loaded was to check for how many bytes were read and compare it against the file's original size. Since this is not a good solution, replace it by comparing the checksum of the loaded bytes against the original file's checksum. Add more test cases: files at a sub-directory and non-existent file. Reviewed-by: Simon Glass Tested-by: Simon Glass [on sandbox] Signed-off-by: Joao Marcos Costa --- .../test_fs/test_squashfs/test_sqfs_load.py | 166 +++++++++++++++--- 1 file changed, 137 insertions(+), 29 deletions(-) diff --git a/test/py/tests/test_fs/test_squashfs/test_sqfs_load.py b/test/py/tests/test_fs/test_squashfs/test_sqfs_load.py index 9e90062384..6ec6ccec6c 100644 --- a/test/py/tests/test_fs/test_squashfs/test_sqfs_load.py +++ b/test/py/tests/test_fs/test_squashfs/test_sqfs_load.py @@ -3,8 +3,118 @@ # Author: Joao Marcos Costa import os +import subprocess import pytest -from sqfs_common import * + +from sqfs_common import SQFS_SRC_DIR, STANDARD_TABLE +from sqfs_common import generate_sqfs_src_dir, make_all_images +from sqfs_common import clean_sqfs_src_dir, clean_all_images +from sqfs_common import check_mksquashfs_version + +@pytest.mark.requiredtool('md5sum') +def original_md5sum(path): + """ Runs md5sum command. + + Args: + path: path to original file. + Returns: + The original file's checksum as a string. + """ + + out = subprocess.run(['md5sum ' + path], shell=True, check=True, + capture_output=True, text=True) + checksum = out.stdout.split()[0] + + return checksum + +def uboot_md5sum(u_boot_console, address, count): + """ Runs U-Boot's md5sum command. + + Args: + u_boot_console: provides the means to interact with U-Boot's console. + address: address where the file was loaded (e.g.: $kernel_addr_r). + count: file's size. It was named 'count' to match md5sum's respective + argument name. + Returns: + The checksum of the file loaded with sqfsload as a string. + """ + + out = u_boot_console.run_command('md5sum {} {}'.format(address, count)) + checksum = out.split()[-1] + + return checksum + +def sqfs_load_files(u_boot_console, files, sizes, address): + """ Loads files and asserts their checksums. + + Args: + u_boot_console: provides the means to interact with U-Boot's console. + files: list of files to be loaded. + sizes: the sizes of each file. + address: the address where the files should be loaded. + """ + build_dir = u_boot_console.config.build_dir + for (file, size) in zip(files, sizes): + out = u_boot_console.run_command('sqfsload host 0 {} {}'.format(address, file)) + + # check if the right amount of bytes was read + assert size in out + + # compare original file's checksum against u-boot's + u_boot_checksum = uboot_md5sum(u_boot_console, address, hex(int(size))) + original_file_path = os.path.join(build_dir, SQFS_SRC_DIR + '/' + file) + original_checksum = original_md5sum(original_file_path) + assert u_boot_checksum == original_checksum + +def sqfs_load_files_at_root(u_boot_console): + """ Calls sqfs_load_files passing the files at the SquashFS image's root. + + Args: + u_boot_console: provides the means to interact with U-Boot's console. + """ + + files = ['f4096', 'f5096', 'f1000'] + sizes = ['4096', '5096', '1000'] + address = '$kernel_addr_r' + sqfs_load_files(u_boot_console, files, sizes, address) + +def sqfs_load_files_at_subdir(u_boot_console): + """ Calls sqfs_load_files passing the files at the SquashFS image's subdir. + + This test checks if the path resolution works, since the file is not at the + root directory. + + Args: + u_boot_console: provides the means to interact with U-Boot's console. + """ + files = ['subdir/subdir-file'] + sizes = ['100'] + address = '$kernel_addr_r' + sqfs_load_files(u_boot_console, files, sizes, address) + +def sqfs_load_non_existent_file(u_boot_console): + """ Calls sqfs_load_files passing an non-existent file to raise an error. + + This test checks if the SquashFS support won't crash if it doesn't find the + specified file. + + Args: + u_boot_console: provides the means to interact with U-Boot's console. + """ + address = '$kernel_addr_r' + file = 'non-existent' + out = u_boot_console.run_command('sqfsload host 0 {} {}'.format(address, file)) + assert 'Failed to load' in out + +def sqfs_run_all_load_tests(u_boot_console): + """ Runs all the previously defined test cases. + + Args: + u_boot_console: provides the means to interact with U-Boot's console. + """ + sqfs_load_files_at_root(u_boot_console) + sqfs_load_files_at_subdir(u_boot_console) + sqfs_load_non_existent_file(u_boot_console) @pytest.mark.boardspec('sandbox') @pytest.mark.buildconfigspec('cmd_fs_generic') @@ -12,35 +122,33 @@ from sqfs_common import * @pytest.mark.buildconfigspec('fs_squashfs') @pytest.mark.requiredtool('mksquashfs') def test_sqfs_load(u_boot_console): + """ Executes the sqfsload test suite. + + First, it generates the SquashFS images, then it runs the test cases and + finally cleans the workspace. If an exception is raised, the workspace is + cleaned before exiting. + + Args: + u_boot_console: provides the means to interact with U-Boot's console. + """ build_dir = u_boot_console.config.build_dir - command = "sqfsload host 0 $kernel_addr_r " - for opt in comp_opts: - # generate and load the squashfs image + # setup test environment + check_mksquashfs_version() + generate_sqfs_src_dir(build_dir) + make_all_images(build_dir) + + # run all tests for each image + for image in STANDARD_TABLE: try: - opt.gen_image(build_dir) - except RuntimeError: - opt.clean_source(build_dir) - # skip unsupported compression types - continue + image_path = os.path.join(build_dir, image) + u_boot_console.run_command('host bind 0 {}'.format(image_path)) + sqfs_run_all_load_tests(u_boot_console) + except: + clean_all_images(build_dir) + clean_sqfs_src_dir(build_dir) + raise AssertionError - path = os.path.join(build_dir, "sqfs-" + opt.name) - output = u_boot_console.run_command("host bind 0 " + path) - - output = u_boot_console.run_command(command + "xxx") - assert "File not found." in output - - for (f, s) in zip(opt.files, opt.sizes): - try: - output = u_boot_console.run_command(command + f) - assert str(s) in output - except: - assert False - opt.cleanup(build_dir) - - # test symbolic link - output = u_boot_console.run_command(command + "sym") - assert str(opt.sizes[0]) in output - - # remove generated files - opt.cleanup(build_dir) + # clean test environment + clean_all_images(build_dir) + clean_sqfs_src_dir(build_dir) From 9bde9b5e29a31bf07a6cd4e00643a7de65c5212d Mon Sep 17 00:00:00 2001 From: Joao Marcos Costa Date: Wed, 30 Jun 2021 19:45:05 -0300 Subject: [PATCH 09/10] test/py: rewrite sqfsls command test suite Add more details to test cases by comparing each expected line with the command's output. Add new test cases: - sqfsls at an empty directory - sqfsls at a sub-directory Reviewed-by: Simon Glass Tested-by: Simon Glass [on sandbox] Signed-off-by: Joao Marcos Costa --- .../test_fs/test_squashfs/test_sqfs_ls.py | 142 +++++++++++++++--- 1 file changed, 122 insertions(+), 20 deletions(-) diff --git a/test/py/tests/test_fs/test_squashfs/test_sqfs_ls.py b/test/py/tests/test_fs/test_squashfs/test_sqfs_ls.py index a0dca2e2fc..9eb00d6888 100644 --- a/test/py/tests/test_fs/test_squashfs/test_sqfs_ls.py +++ b/test/py/tests/test_fs/test_squashfs/test_sqfs_ls.py @@ -4,7 +4,101 @@ import os import pytest -from sqfs_common import * + +from sqfs_common import STANDARD_TABLE +from sqfs_common import generate_sqfs_src_dir, make_all_images +from sqfs_common import clean_sqfs_src_dir, clean_all_images +from sqfs_common import check_mksquashfs_version + +def sqfs_ls_at_root(u_boot_console): + """ Runs sqfsls at image's root. + + This test checks if all the present files and directories were listed. Also, + it checks if passing the slash or not changes the output, which it shouldn't. + + Args: + u_boot_console: provides the means to interact with U-Boot's console. + """ + + no_slash = u_boot_console.run_command('sqfsls host 0') + slash = u_boot_console.run_command('sqfsls host 0 /') + assert no_slash == slash + + expected_lines = ['empty-dir/', '1000 f1000', '4096 f4096', '5096 f5096', + 'subdir/', ' sym', '4 file(s), 2 dir(s)'] + + output = u_boot_console.run_command('sqfsls host 0') + for line in expected_lines: + assert line in output + +def sqfs_ls_at_empty_dir(u_boot_console): + """ Runs sqfsls at an empty directory. + + This tests checks if sqfsls will print anything other than the 'Empty directory' + message. + + Args: + u_boot_console: provides the means to interact with U-Boot's console. + """ + assert u_boot_console.run_command('sqfsls host 0 empty-dir') == 'Empty directory.' + +def sqfs_ls_at_subdir(u_boot_console): + """ Runs sqfsls at the SquashFS image's subdir. + + This test checks if the path resolution works, since the directory is not the + root. + + Args: + u_boot_console: provides the means to interact with U-Boot's console. + """ + expected_lines = ['100 subdir-file', '1 file(s), 0 dir(s)'] + output = u_boot_console.run_command('sqfsls host 0 subdir') + for line in expected_lines: + assert line in output + +def sqfs_ls_at_symlink(u_boot_console): + """ Runs sqfsls at a SquashFS image's symbolic link. + + This test checks if the symbolic link's target resolution works. + + Args: + u_boot_console: provides the means to interact with U-Boot's console. + """ + # since sym -> subdir, the following outputs must be equal + output = u_boot_console.run_command('sqfsls host 0 sym') + output_subdir = u_boot_console.run_command('sqfsls host 0 subdir') + assert output == output_subdir + + expected_lines = ['100 subdir-file', '1 file(s), 0 dir(s)'] + for line in expected_lines: + assert line in output + +def sqfs_ls_at_non_existent_dir(u_boot_console): + """ Runs sqfsls at a file and at a non-existent directory. + + This test checks if the SquashFS support won't crash if it doesn't find the + specified directory or if it takes a file as an input instead of an actual + directory. In both cases, the output should be the same. + + Args: + u_boot_console: provides the means to interact with U-Boot's console. + """ + out_non_existent = u_boot_console.run_command('sqfsls host 0 fff') + out_not_dir = u_boot_console.run_command('sqfsls host 0 f1000') + assert out_non_existent == out_not_dir + assert '** Cannot find directory. **' in out_non_existent + +def sqfs_run_all_ls_tests(u_boot_console): + """ Runs all the previously defined test cases. + + Args: + u_boot_console: provides the means to interact with U-Boot's console. + """ + sqfs_ls_at_root(u_boot_console) + sqfs_ls_at_empty_dir(u_boot_console) + sqfs_ls_at_subdir(u_boot_console) + sqfs_ls_at_symlink(u_boot_console) + sqfs_ls_at_non_existent_dir(u_boot_console) @pytest.mark.boardspec('sandbox') @pytest.mark.buildconfigspec('cmd_fs_generic') @@ -12,25 +106,33 @@ from sqfs_common import * @pytest.mark.buildconfigspec('fs_squashfs') @pytest.mark.requiredtool('mksquashfs') def test_sqfs_ls(u_boot_console): - build_dir = u_boot_console.config.build_dir - for opt in comp_opts: - try: - opt.gen_image(build_dir) - except RuntimeError: - opt.clean_source(build_dir) - # skip unsupported compression types - continue - path = os.path.join(build_dir, "sqfs-" + opt.name) - output = u_boot_console.run_command("host bind 0 " + path) + """ Executes the sqfsls test suite. + First, it generates the SquashFS images, then it runs the test cases and + finally cleans the workspace. If an exception is raised, the workspace is + cleaned before exiting. + + Args: + u_boot_console: provides the means to interact with U-Boot's console. + """ + build_dir = u_boot_console.config.build_dir + + # setup test environment + check_mksquashfs_version() + generate_sqfs_src_dir(build_dir) + make_all_images(build_dir) + + # run all tests for each image + for image in STANDARD_TABLE: try: - # list files in root directory - output = u_boot_console.run_command("sqfsls host 0") - assert str(len(opt.files) + 1) + " file(s), 0 dir(s)" in output - assert " sym" in output - output = u_boot_console.run_command("sqfsls host 0 xxx") - assert "** Cannot find directory. **" in output + image_path = os.path.join(build_dir, image) + u_boot_console.run_command('host bind 0 {}'.format(image_path)) + sqfs_run_all_ls_tests(u_boot_console) except: - opt.cleanup(build_dir) - assert False - opt.cleanup(build_dir) + clean_all_images(build_dir) + clean_sqfs_src_dir(build_dir) + raise AssertionError + + # clean test environment + clean_all_images(build_dir) + clean_sqfs_src_dir(build_dir) From b1c2102db1774686282474aee3c2dd06df92f175 Mon Sep 17 00:00:00 2001 From: Tom Rini Date: Thu, 10 Jun 2021 10:57:36 -0400 Subject: [PATCH 10/10] Docker/CI: Update to "focal" and latest build Move us up to being based on Ubuntu 20.04 "focal" and the latest tag from Ubuntu for this release. For this, we make sure that "python" is now python3 but still include python2.7 for the rx51 qemu build as that is very old and does not support python3. Signed-off-by: Tom Rini --- .azure-pipelines.yml | 2 +- .gitlab-ci.yml | 2 +- test/nokia_rx51_test.sh | 2 +- tools/docker/Dockerfile | 20 +++++++++----------- 4 files changed, 12 insertions(+), 14 deletions(-) diff --git a/.azure-pipelines.yml b/.azure-pipelines.yml index 99964be53c..06c4a2ffd0 100644 --- a/.azure-pipelines.yml +++ b/.azure-pipelines.yml @@ -2,7 +2,7 @@ variables: windows_vm: vs2017-win2016 ubuntu_vm: ubuntu-18.04 macos_vm: macOS-10.15 - ci_runner_image: trini/u-boot-gitlab-ci-runner:bionic-20200807-02Sep2020 + ci_runner_image: trini/u-boot-gitlab-ci-runner:focal-20210609-01Jul2021 # Add '-u 0' options for Azure pipelines, otherwise we get "permission # denied" error when it tries to "useradd -m -u 1001 vsts_azpcontainer", # since our $(ci_runner_image) user is not root. diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d072e833a3..94c7333056 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,7 +2,7 @@ # Grab our configured image. The source for this is found at: # https://source.denx.de/u-boot/gitlab-ci-runner -image: trini/u-boot-gitlab-ci-runner:bionic-20200807-02Sep2020 +image: trini/u-boot-gitlab-ci-runner:focal-20210609-01Jul2021 # We run some tests in different order, to catch some failures quicker. stages: diff --git a/test/nokia_rx51_test.sh b/test/nokia_rx51_test.sh index ee45e8d6dd..ff840c1e6c 100755 --- a/test/nokia_rx51_test.sh +++ b/test/nokia_rx51_test.sh @@ -56,7 +56,7 @@ if ! test -f qemu-system-arm; then test -d qemu-linaro || git clone https://git.linaro.org/qemu/qemu-linaro.git cd qemu-linaro git checkout 8f8d8e0796efe1a6f34cdd83fb798f3c41217ec1 - ./configure --enable-system --target-list=arm-softmmu --disable-sdl --disable-gtk --disable-curses --audio-drv-list= --audio-card-list= --disable-werror --disable-xen --disable-xen-pci-passthrough --disable-brlapi --disable-vnc --disable-curl --disable-slirp --disable-kvm --disable-user --disable-linux-user --disable-bsd-user --disable-guest-base --disable-uuid --disable-vde --disable-linux-aio --disable-cap-ng --disable-attr --disable-blobs --disable-docs --disable-spice --disable-libiscsi --disable-smartcard-nss --disable-usb-redir --disable-guest-agent --disable-seccomp --disable-glusterfs --disable-nptl --disable-fdt + ./configure --enable-system --target-list=arm-softmmu --python=/usr/bin/python2.7 --disable-sdl --disable-gtk --disable-curses --audio-drv-list= --audio-card-list= --disable-werror --disable-xen --disable-xen-pci-passthrough --disable-brlapi --disable-vnc --disable-curl --disable-slirp --disable-kvm --disable-user --disable-linux-user --disable-bsd-user --disable-guest-base --disable-uuid --disable-vde --disable-linux-aio --disable-cap-ng --disable-attr --disable-blobs --disable-docs --disable-spice --disable-libiscsi --disable-smartcard-nss --disable-usb-redir --disable-guest-agent --disable-seccomp --disable-glusterfs --disable-nptl --disable-fdt make -j4 cd .. ln -s qemu-linaro/arm-softmmu/qemu-system-arm . diff --git a/tools/docker/Dockerfile b/tools/docker/Dockerfile index 579df82b56..de0c6cee8c 100644 --- a/tools/docker/Dockerfile +++ b/tools/docker/Dockerfile @@ -2,7 +2,7 @@ # This Dockerfile is used to build an image containing basic stuff to be used # to build U-Boot and run our test suites. -FROM ubuntu:bionic-20200807 +FROM ubuntu:focal-20210609 MAINTAINER Tom Rini LABEL Description=" This image is for building U-Boot inside a container" @@ -12,7 +12,7 @@ ENV DEBIAN_FRONTEND=noninteractive # Add LLVM repository RUN apt-get update && apt-get install -y gnupg2 wget xz-utils && rm -rf /var/lib/apt/lists/* RUN wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - -RUN echo deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-10 main | tee /etc/apt/sources.list.d/llvm.list +RUN echo deb http://apt.llvm.org/focal/ llvm-toolchain-focal-10 main | tee /etc/apt/sources.list.d/llvm.list # Manually install the kernel.org "Crosstool" based toolchains for gcc-7.3 RUN wget -O - https://mirrors.edge.kernel.org/pub/tools/crosstool/files/bin/x86_64/9.2.0/x86_64-gcc-9.2.0-nolibc-aarch64-linux.tar.xz | tar -C /opt -xJ @@ -61,11 +61,11 @@ RUN apt-get update && apt-get install -y \ iasl \ imagemagick \ iputils-ping \ + libgit2-dev \ libguestfs-tools \ - libisl15 \ liblz4-tool \ libpixman-1-dev \ - libpython-dev \ + libpython3-dev \ libsdl1.2-dev \ libsdl2-dev \ libssl-dev \ @@ -81,12 +81,13 @@ RUN apt-get update && apt-get install -y \ picocom \ parted \ pkg-config \ - python \ - python-dev \ - python-pip \ - python-virtualenv \ + python-is-python3 \ + python2.7 \ + python3 \ + python3-dev \ python3-pip \ python3-sphinx \ + python3-virtualenv \ rpm2cpio \ sbsigntool \ sloccount \ @@ -106,9 +107,6 @@ RUN chmod +r /boot/vmlinu* # Manually install libmpfr4 for the toolchains RUN wget http://mirrors.kernel.org/ubuntu/pool/main/m/mpfr4/libmpfr4_3.1.4-1_amd64.deb && dpkg -i libmpfr4_3.1.4-1_amd64.deb && rm libmpfr4_3.1.4-1_amd64.deb -# Manually install a new enough version of efitools (must be v1.5.2 or later) -RUN wget http://mirrors.kernel.org/ubuntu/pool/universe/e/efitools/efitools_1.8.1-0ubuntu2_amd64.deb && sudo dpkg -i efitools_1.8.1-0ubuntu2_amd64.deb && rm efitools_1.8.1-0ubuntu2_amd64.deb - # Manually install a new enough version of sbsigntools (must be v0.9.4 or later) RUN git clone https://git.kernel.org/pub/scm/linux/kernel/git/jejb/sbsigntools.git /tmp/sbsigntools && \ cd /tmp/sbsigntools && \