eaw/esp-idf
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

style: format python files with isort and double-quote-string-fixer

Browse Source
This commit is contained in:
Fu Hanxi
2021-01-26 10:49:01 +08:00
parent dc8402ea61
commit 0146f258d7
276 changed files with 8241 additions and 8162 deletions
Download Patch File Download Diff File Expand all files Collapse all files

118
tools/ble/lib_ble_client.py
View File

@@ -18,25 +18,25 @@
# DBus-Bluez BLE library
from __future__ import print_function
import sys
import time
import traceback
try:
from future.moves.itertools import zip_longest
import dbus
import dbus.mainloop.glib
from future.moves.itertools import zip_longest
from gi.repository import GLib
except ImportError as e:
if 'linux' not in sys.platform:
raise e
print(e)
print("Install packages `libgirepository1.0-dev gir1.2-gtk-3.0 libcairo2-dev libdbus-1-dev libdbus-glib-1-dev` for resolving the issue")
print("Run `pip install -r $IDF_PATH/tools/ble/requirements.txt` for resolving the issue")
print('Install packages `libgirepository1.0-dev gir1.2-gtk-3.0 libcairo2-dev libdbus-1-dev libdbus-glib-1-dev` for resolving the issue')
print('Run `pip install -r $IDF_PATH/tools/ble/requirements.txt` for resolving the issue')
raise
from . import lib_gatt
from . import lib_gap
from . import lib_gap, lib_gatt
srv_added_old_cnt = 0
srv_added_new_cnt = 0
@@ -198,7 +198,7 @@ class BLE_Bluez_Client:
try:
self.bus = dbus.SystemBus()
om_iface_obj = dbus.Interface(self.bus.get_object(BLUEZ_SERVICE_NAME, "/"), DBUS_OM_IFACE)
om_iface_obj = dbus.Interface(self.bus.get_object(BLUEZ_SERVICE_NAME, '/'), DBUS_OM_IFACE)
self.ble_objs = om_iface_obj.GetManagedObjects()
except Exception as e:
@@ -206,7 +206,7 @@ class BLE_Bluez_Client:
def __del__(self):
try:
print("Test Exit")
print('Test Exit')
except Exception as e:
print(e)
sys.exit(1)
@@ -220,7 +220,7 @@ class BLE_Bluez_Client:
verify_signal_check = 0
adapter_on = False
try:
print("discovering adapter...")
print('discovering adapter...')
for path, interfaces in self.ble_objs.items():
adapter = interfaces.get(ADAPTER_IFACE)
if adapter is not None:
@@ -234,32 +234,32 @@ class BLE_Bluez_Client:
break
if self.adapter is None:
raise Exception("Bluetooth adapter not found")
raise Exception('Bluetooth adapter not found')
if self.props_iface_obj is None:
raise Exception("Properties interface not found")
raise Exception('Properties interface not found')
print("bluetooth adapter discovered")
print('bluetooth adapter discovered')
# Check if adapter is already powered on
if adapter_on:
print("Adapter already powered on")
print('Adapter already powered on')
return True
# Power On Adapter
print("powering on adapter...")
print('powering on adapter...')
self.props_iface_obj.connect_to_signal('PropertiesChanged', props_change_handler)
self.props_iface_obj.Set(ADAPTER_IFACE, "Powered", dbus.Boolean(1))
self.props_iface_obj.Set(ADAPTER_IFACE, 'Powered', dbus.Boolean(1))
signal_caught = False
GLib.timeout_add_seconds(5, verify_signal_is_caught)
event_loop.run()
if adapter_on:
print("bluetooth adapter powered on")
print('bluetooth adapter powered on')
return True
else:
raise Exception("Failure: bluetooth adapter not powered on")
raise Exception('Failure: bluetooth adapter not powered on')
except Exception:
print(traceback.format_exc())
@@ -275,7 +275,7 @@ class BLE_Bluez_Client:
device_connected = False
try:
self.adapter.StartDiscovery()
print("\nStarted Discovery")
print('\nStarted Discovery')
discovery_start = True
@@ -283,7 +283,7 @@ class BLE_Bluez_Client:
verify_signal_check = 0
try:
if self.device is None:
print("\nConnecting to device...")
print('\nConnecting to device...')
# Wait for device to be discovered
time.sleep(5)
device_found = self.get_device()
@@ -294,13 +294,13 @@ class BLE_Bluez_Client:
GLib.timeout_add_seconds(5, verify_signal_is_caught)
event_loop.run()
if device_connected:
print("\nConnected to device")
print('\nConnected to device')
return True
else:
raise Exception
except Exception as e:
print(e)
print("\nRetries left", retry_cnt - 1)
print('\nRetries left', retry_cnt - 1)
continue
# Device not found
@@ -318,7 +318,7 @@ class BLE_Bluez_Client:
'''
dev_path = None
om_iface_obj = dbus.Interface(self.bus.get_object(BLUEZ_SERVICE_NAME, "/"), DBUS_OM_IFACE)
om_iface_obj = dbus.Interface(self.bus.get_object(BLUEZ_SERVICE_NAME, '/'), DBUS_OM_IFACE)
self.ble_objs = om_iface_obj.GetManagedObjects()
for path, interfaces in self.ble_objs.items():
if DEVICE_IFACE not in interfaces.keys():
@@ -326,12 +326,12 @@ class BLE_Bluez_Client:
device_addr_iface = (path.replace('_', ':')).lower()
dev_addr = self.devaddr.lower()
if dev_addr in device_addr_iface and \
interfaces[DEVICE_IFACE].get("Name") == self.devname:
interfaces[DEVICE_IFACE].get('Name') == self.devname:
dev_path = path
break
if dev_path is None:
raise Exception("\nBLE device not found")
raise Exception('\nBLE device not found')
device_props_iface_obj = dbus.Interface(self.bus.get_object(BLUEZ_SERVICE_NAME, dev_path), DBUS_PROP_IFACE)
device_props_iface_obj.connect_to_signal('PropertiesChanged', props_change_handler)
@@ -373,7 +373,7 @@ class BLE_Bluez_Client:
signal_caught = False
try:
om_iface_obj = dbus.Interface(self.bus.get_object(BLUEZ_SERVICE_NAME, "/"), DBUS_OM_IFACE)
om_iface_obj = dbus.Interface(self.bus.get_object(BLUEZ_SERVICE_NAME, '/'), DBUS_OM_IFACE)
self.ble_objs = om_iface_obj.GetManagedObjects()
for path, interfaces in self.ble_objs.items():
self.srvc_iface_added_handler(path, interfaces)
@@ -383,12 +383,12 @@ class BLE_Bluez_Client:
om_iface_obj.connect_to_signal('InterfacesAdded', self.srvc_iface_added_handler)
event_loop.run()
if not services_resolved:
raise Exception("Services not found...")
raise Exception('Services not found...')
if service_uuid:
self.verify_service_uuid_found(service_uuid)
if not service_uuid_found:
raise Exception("Service with uuid: %s not found..." % service_uuid)
raise Exception('Service with uuid: %s not found...' % service_uuid)
# Services found
return self.srv_uuid
@@ -426,7 +426,7 @@ class BLE_Bluez_Client:
signal_caught = False
try:
om_iface_obj = dbus.Interface(self.bus.get_object(BLUEZ_SERVICE_NAME, "/"), DBUS_OM_IFACE)
om_iface_obj = dbus.Interface(self.bus.get_object(BLUEZ_SERVICE_NAME, '/'), DBUS_OM_IFACE)
self.ble_objs = om_iface_obj.GetManagedObjects()
for path, interfaces in self.ble_objs.items():
self.chrc_iface_added_handler(path, interfaces)
@@ -460,13 +460,13 @@ class BLE_Bluez_Client:
if 'read' in props[1]:
chrc_val = chrc.ReadValue({}, dbus_interface=GATT_CHRC_IFACE)
else:
print("Warning: Cannot read value. Characteristic does not have read permission.")
print('Warning: Cannot read value. Characteristic does not have read permission.')
if not (ord(write_val) == int(chrc_val[0])):
print("\nWrite Failed")
print('\nWrite Failed')
return False
self.chars[path] = chrc_val, props[1], props[2] # update value
if not char_write_props:
raise Exception("Failure: Cannot perform write operation. Characteristic does not have write permission.")
raise Exception('Failure: Cannot perform write operation. Characteristic does not have write permission.')
return self.chars
except Exception:
@@ -498,7 +498,7 @@ class BLE_Bluez_Client:
break
if srv_path is None:
raise Exception("Failure: HR UUID:", hr_srv_uuid, "not found")
raise Exception('Failure: HR UUID:', hr_srv_uuid, 'not found')
chars_ret = self.read_chars()
@@ -509,10 +509,10 @@ class BLE_Bluez_Client:
if hr_char_uuid in props[2]: # uuid
break
if chrc is None:
raise Exception("Failure: Characteristics for service: ", srv_path, "not found")
raise Exception('Failure: Characteristics for service: ', srv_path, 'not found')
# Subscribe to notifications
print("\nSubscribe to notifications: On")
print('\nSubscribe to notifications: On')
chrc.StartNotify(dbus_interface=GATT_CHRC_IFACE)
chrc_props_iface_obj = dbus.Interface(self.bus.get_object(BLUEZ_SERVICE_NAME, chrc_path), DBUS_PROP_IFACE)
@@ -524,7 +524,7 @@ class BLE_Bluez_Client:
event_loop.run()
chrc.StopNotify(dbus_interface=GATT_CHRC_IFACE)
time.sleep(2)
print("\nSubscribe to notifications: Off")
print('\nSubscribe to notifications: Off')
ble_hr_chrc = False
return True
@@ -587,7 +587,7 @@ class BLE_Bluez_Client:
lib_gap.ADV_OBJ = False
try:
print("Advertising started")
print('Advertising started')
gatt_app_ret = self.create_and_reg_gatt_app()
# Check if gatt app create and register command
@@ -609,7 +609,7 @@ class BLE_Bluez_Client:
# Get device when connected
if not self.device:
om_iface_obj = dbus.Interface(self.bus.get_object(BLUEZ_SERVICE_NAME, "/"), DBUS_OM_IFACE)
om_iface_obj = dbus.Interface(self.bus.get_object(BLUEZ_SERVICE_NAME, '/'), DBUS_OM_IFACE)
self.ble_objs = om_iface_obj.GetManagedObjects()
for path, interfaces in self.ble_objs.items():
@@ -679,13 +679,13 @@ class BLE_Bluez_Client:
# Check for success
if lib_gatt.GATT_APP_OBJ:
print("GATT Data created")
print('GATT Data created')
if gatt_app_registered:
print("GATT Application registered")
print('GATT Application registered')
gatt_checks_done = True
if gatt_app_retry_check_cnt == 20:
if not gatt_app_registered:
print("Failure: GATT Application could not be registered")
print('Failure: GATT Application could not be registered')
gatt_checks_done = True
# End polling if app is registered or cnt has reached 10
@@ -707,13 +707,13 @@ class BLE_Bluez_Client:
adv_checks_done = False
if lib_gap.ADV_OBJ:
print("Advertising data created")
print('Advertising data created')
if adv_registered or adv_active_instance:
print("Advertisement registered")
print('Advertisement registered')
adv_checks_done = True
if adv_retry_check_cnt == 10:
if not adv_registered and not adv_active_instance:
print("Failure: Advertisement could not be registered")
print('Failure: Advertisement could not be registered')
adv_checks_done = True
# End polling if success or cnt has reached 10
@@ -737,11 +737,11 @@ class BLE_Bluez_Client:
if blecent_retry_check_cnt == 10:
# check for failures
if not read_req_check:
print("Failure: Read Request not received")
print('Failure: Read Request not received')
if not write_req_check:
print("Failure: Write Request not received")
print('Failure: Write Request not received')
if not subscribe_req_check:
print("Failure: Subscribe Request not received")
print('Failure: Subscribe Request not received')
# Blecent Test failed
test_checks_pass = False
@@ -790,27 +790,27 @@ class BLE_Bluez_Client:
if blecent_retry_check_cnt == 10:
# check for failures
if not gatt_app_obj_check:
print("Warning: GATT Data could not be removed")
print('Warning: GATT Data could not be removed')
if not gatt_app_reg_check:
print("Warning: GATT Application could not be unregistered")
print('Warning: GATT Application could not be unregistered')
if not adv_data_check:
print("Warning: Advertising data could not be removed")
print('Warning: Advertising data could not be removed')
if not adv_reg_check:
print("Warning: Advertisement could not be unregistered")
print('Warning: Advertisement could not be unregistered')
# Blecent Test failed
adv_stop = False
else:
# Check for success
if not gatt_app_obj_check and not lib_gatt.GATT_APP_OBJ:
print("GATT Data removed")
print('GATT Data removed')
gatt_app_obj_check = True
if not gatt_app_reg_check and not gatt_app_registered:
print("GATT Application unregistered")
print('GATT Application unregistered')
gatt_app_reg_check = True
if not adv_data_check and not adv_reg_check and not (adv_registered or adv_active_instance or lib_gap.ADV_OBJ):
print("Advertising data removed")
print("Advertisement unregistered")
print('Advertising data removed')
print('Advertisement unregistered')
adv_data_check = True
adv_reg_check = True
# all checks passed
@@ -842,7 +842,7 @@ class BLE_Bluez_Client:
blecent_retry_check_cnt = 0
verify_signal_check = 0
print("\nexiting from test...")
print('\nexiting from test...')
self.props_iface_obj.connect_to_signal('PropertiesChanged', props_change_handler)
@@ -864,13 +864,13 @@ class BLE_Bluez_Client:
event_loop.run()
if adv_stop:
print("Stop Advertising status: ", adv_stop)
print('Stop Advertising status: ', adv_stop)
else:
print("Warning: Stop Advertising status: ", adv_stop)
print('Warning: Stop Advertising status: ', adv_stop)
# Disconnect device
if self.device:
print("disconnecting device...")
print('disconnecting device...')
self.device.Disconnect(dbus_interface=DEVICE_IFACE)
if self.adapter:
self.adapter.RemoveDevice(self.device)
@@ -885,9 +885,9 @@ class BLE_Bluez_Client:
event_loop.run()
if not device_connected:
print("device disconnected")
print('device disconnected')
else:
print("Warning: device could not be disconnected")
print('Warning: device could not be disconnected')
except Exception:
print(traceback.format_exc())

5
tools/ble/lib_gap.py
View File

@@ -18,6 +18,7 @@
# Register Advertisement
from __future__ import print_function
import sys
try:
@@ -27,8 +28,8 @@ except ImportError as e:
if 'linux' not in sys.platform:
raise e
print(e)
print("Install packages `libgirepository1.0-dev gir1.2-gtk-3.0 libcairo2-dev libdbus-1-dev libdbus-glib-1-dev` for resolving the issue")
print("Run `pip install -r $IDF_PATH/tools/ble/requirements.txt` for resolving the issue")
print('Install packages `libgirepository1.0-dev gir1.2-gtk-3.0 libcairo2-dev libdbus-1-dev libdbus-glib-1-dev` for resolving the issue')
print('Run `pip install -r $IDF_PATH/tools/ble/requirements.txt` for resolving the issue')
raise
ADV_OBJ = False

43
tools/ble/lib_gatt.py
View File

@@ -18,6 +18,7 @@
# Creating GATT Application which then becomes available to remote devices.
from __future__ import print_function
import sys
try:
@@ -27,8 +28,8 @@ except ImportError as e:
if 'linux' not in sys.platform:
raise e
print(e)
print("Install packages `libgirepository1.0-dev gir1.2-gtk-3.0 libcairo2-dev libdbus-1-dev libdbus-glib-1-dev` for resolving the issue")
print("Run `pip install -r $IDF_PATH/tools/ble/requirements.txt` for resolving the issue")
print('Install packages `libgirepository1.0-dev gir1.2-gtk-3.0 libcairo2-dev libdbus-1-dev libdbus-glib-1-dev` for resolving the issue')
print('Run `pip install -r $IDF_PATH/tools/ble/requirements.txt` for resolving the issue')
raise
alert_status_char_obj = None
@@ -216,7 +217,7 @@ class Characteristic(dbus.service.Object):
@dbus.service.signal(DBUS_PROP_IFACE,
signature='sa{sv}as')
def PropertiesChanged(self, interface, changed, invalidated):
print("\nProperties Changed")
print('\nProperties Changed')
class Descriptor(dbus.service.Object):
@@ -293,8 +294,8 @@ class SupportedNewAlertCategoryCharacteristic(Characteristic):
val_list = []
for val in self.value:
val_list.append(dbus.Byte(val))
print("Read Request received\n", "\tSupportedNewAlertCategoryCharacteristic")
print("\tValue:", "\t", val_list)
print('Read Request received\n', '\tSupportedNewAlertCategoryCharacteristic')
print('\tValue:', '\t', val_list)
return val_list
@@ -314,23 +315,23 @@ class AlertNotificationControlPointCharacteristic(Characteristic):
val_list = []
for val in self.value:
val_list.append(dbus.Byte(val))
print("Read Request received\n", "\tAlertNotificationControlPointCharacteristic")
print("\tValue:", "\t", val_list)
print('Read Request received\n', '\tAlertNotificationControlPointCharacteristic')
print('\tValue:', '\t', val_list)
return val_list
def WriteValue(self, value, options):
global CHAR_WRITE
CHAR_WRITE = True
print("Write Request received\n", "\tAlertNotificationControlPointCharacteristic")
print("\tCurrent value:", "\t", self.value)
print('Write Request received\n', '\tAlertNotificationControlPointCharacteristic')
print('\tCurrent value:', '\t', self.value)
val_list = []
for val in value:
val_list.append(val)
self.value = val_list
# Check if new value is written
print("\tNew value:", "\t", self.value)
print('\tNew value:', '\t', self.value)
if not self.value == value:
print("Failed: Write Request\n\tNew value not written\tCurrent value:", self.value)
print('Failed: Write Request\n\tNew value not written\tCurrent value:', self.value)
class UnreadAlertStatusCharacteristic(Characteristic):
@@ -355,7 +356,7 @@ class UnreadAlertStatusCharacteristic(Characteristic):
print('\nAlready notifying, nothing to do')
return
self.notifying = True
print("\nNotify Started")
print('\nNotify Started')
self.cccd_obj.WriteValue([dbus.Byte(1), dbus.Byte(0)])
self.cccd_obj.ReadValue()
@@ -364,26 +365,26 @@ class UnreadAlertStatusCharacteristic(Characteristic):
print('\nNot notifying, nothing to do')
return
self.notifying = False
print("\nNotify Stopped")
print('\nNotify Stopped')
def ReadValue(self, options):
print("Read Request received\n", "\tUnreadAlertStatusCharacteristic")
print('Read Request received\n', '\tUnreadAlertStatusCharacteristic')
val_list = []
for val in self.value:
val_list.append(dbus.Byte(val))
print("\tValue:", "\t", val_list)
print('\tValue:', '\t', val_list)
return val_list
def WriteValue(self, value, options):
print("Write Request received\n", "\tUnreadAlertStatusCharacteristic")
print('Write Request received\n', '\tUnreadAlertStatusCharacteristic')
val_list = []
for val in value:
val_list.append(val)
self.value = val_list
# Check if new value is written
print("\tNew value:", "\t", self.value)
print('\tNew value:', '\t', self.value)
if not self.value == value:
print("Failed: Write Request\n\tNew value not written\tCurrent value:", self.value)
print('Failed: Write Request\n\tNew value not written\tCurrent value:', self.value)
class ClientCharacteristicConfigurationDescriptor(Descriptor):
@@ -398,7 +399,7 @@ class ClientCharacteristicConfigurationDescriptor(Descriptor):
characteristic)
def ReadValue(self):
print("\tValue on read:", "\t", self.value)
print('\tValue on read:', '\t', self.value)
return self.value
def WriteValue(self, value):
@@ -407,6 +408,6 @@ class ClientCharacteristicConfigurationDescriptor(Descriptor):
val_list.append(val)
self.value = val_list
# Check if new value is written
print("New value on write:", "\t", self.value)
print('New value on write:', '\t', self.value)
if not self.value == value:
print("Failed: Write Request\n\tNew value not written\tCurrent value:", self.value)
print('Failed: Write Request\n\tNew value not written\tCurrent value:', self.value)

92
tools/build_apps.py
View File

@@ -8,74 +8,74 @@ import argparse
import logging
import sys
from find_build_apps import BuildItem, BuildError, setup_logging, BUILD_SYSTEMS
from find_build_apps.common import rmdir, SIZE_JSON_FN
from find_build_apps import BUILD_SYSTEMS, BuildError, BuildItem, setup_logging
from find_build_apps.common import SIZE_JSON_FN, rmdir
def main():
parser = argparse.ArgumentParser(description="ESP-IDF app builder")
parser = argparse.ArgumentParser(description='ESP-IDF app builder')
parser.add_argument(
"-v",
"--verbose",
action="count",
help="Increase the logging level of the script. Can be specified multiple times.",
'-v',
'--verbose',
action='count',
help='Increase the logging level of the script. Can be specified multiple times.',
)
parser.add_argument(
"--build-verbose",
action="store_true",
help="Enable verbose output from build system.",
'--build-verbose',
action='store_true',
help='Enable verbose output from build system.',
)
parser.add_argument(
"--log-file",
type=argparse.FileType("w"),
help="Write the script log to the specified file, instead of stderr",
'--log-file',
type=argparse.FileType('w'),
help='Write the script log to the specified file, instead of stderr',
)
parser.add_argument(
"--parallel-count",
'--parallel-count',
default=1,
type=int,
help="Number of parallel build jobs. Note that this script doesn't start the jobs, " +
"it needs to be executed multiple times with same value of --parallel-count and " +
"different values of --parallel-index.",
'it needs to be executed multiple times with same value of --parallel-count and ' +
'different values of --parallel-index.',
)
parser.add_argument(
"--parallel-index",
'--parallel-index',
default=1,
type=int,
help="Index (1-based) of the job, out of the number specified by --parallel-count.",
help='Index (1-based) of the job, out of the number specified by --parallel-count.',
)
parser.add_argument(
"--format",
default="json",
choices=["json"],
help="Format to read the list of builds",
'--format',
default='json',
choices=['json'],
help='Format to read the list of builds',
)
parser.add_argument(
"--dry-run",
action="store_true",
'--dry-run',
action='store_true',
help="Don't actually build, only print the build commands",
)
parser.add_argument(
"--keep-going",
action="store_true",
'--keep-going',
action='store_true',
help="Don't exit immediately when a build fails.",
)
parser.add_argument(
"--output-build-list",
type=argparse.FileType("w"),
help="If specified, the list of builds (with all the placeholders expanded) will be written to this file.",
'--output-build-list',
type=argparse.FileType('w'),
help='If specified, the list of builds (with all the placeholders expanded) will be written to this file.',
)
parser.add_argument(
"--size-info",
type=argparse.FileType("a"),
help="If specified, the test case name and size info json will be written to this file"
'--size-info',
type=argparse.FileType('a'),
help='If specified, the test case name and size info json will be written to this file'
)
parser.add_argument(
"build_list",
type=argparse.FileType("r"),
nargs="?",
'build_list',
type=argparse.FileType('r'),
nargs='?',
default=sys.stdin,
help="Name of the file to read the list of builds from. If not specified, read from stdin.",
help='Name of the file to read the list of builds from. If not specified, read from stdin.',
)
args = parser.parse_args()
@@ -83,7 +83,7 @@ def main():
build_items = [BuildItem.from_json(line) for line in args.build_list]
if not build_items:
logging.warning("Empty build list")
logging.warning('Empty build list')
SystemExit(0)
num_builds = len(build_items)
@@ -92,12 +92,12 @@ def main():
num_builds_per_job = (num_builds + num_jobs - 1) // num_jobs
min_job_index = num_builds_per_job * job_index
if min_job_index >= num_builds:
logging.warn("Nothing to do for job {} (build total: {}, per job: {})".format(
logging.warn('Nothing to do for job {} (build total: {}, per job: {})'.format(
job_index + 1, num_builds, num_builds_per_job))
raise SystemExit(0)
max_job_index = min(num_builds_per_job * (job_index + 1) - 1, num_builds - 1)
logging.info("Total {} builds, max. {} builds per job, running builds {}-{}".format(
logging.info('Total {} builds, max. {} builds per job, running builds {}-{}'.format(
num_builds, num_builds_per_job, min_job_index + 1, max_job_index + 1))
builds_for_current_job = build_items[min_job_index:max_job_index + 1]
@@ -107,13 +107,13 @@ def main():
build_info.dry_run = args.dry_run
build_info.verbose = args.build_verbose
build_info.keep_going = args.keep_going
logging.debug(" Build {}: {}".format(index, repr(build_info)))
logging.debug(' Build {}: {}'.format(index, repr(build_info)))
if args.output_build_list:
args.output_build_list.write(build_info.to_json_expanded() + "\n")
args.output_build_list.write(build_info.to_json_expanded() + '\n')
failed_builds = []
for build_info in builds_for_current_job:
logging.info("Running build {}: {}".format(build_info.index, repr(build_info)))
logging.info('Running build {}: {}'.format(build_info.index, repr(build_info)))
build_system_class = BUILD_SYSTEMS[build_info.build_system]
try:
build_system_class.build(build_info)
@@ -127,16 +127,16 @@ def main():
if args.size_info:
build_info.write_size_info(args.size_info)
if not build_info.preserve:
logging.info("Removing build directory {}".format(build_info.build_path))
logging.info('Removing build directory {}'.format(build_info.build_path))
# we only remove binaries here, log files are still needed by check_build_warnings.py
rmdir(build_info.build_path, exclude_file_pattern=SIZE_JSON_FN)
if failed_builds:
logging.error("The following build have failed:")
logging.error('The following build have failed:')
for build in failed_builds:
logging.error(" {}".format(build))
logging.error(' {}'.format(build))
raise SystemExit(1)
if __name__ == "__main__":
if __name__ == '__main__':
main()

22
tools/check_python_dependencies.py
View File

@@ -29,15 +29,15 @@ except Exception:
def escape_backslash(path):
if sys.platform == "win32":
if sys.platform == 'win32':
# escaped backslashes are necessary in order to be able to copy-paste the printed path
return path.replace("\\", "\\\\")
return path.replace('\\', '\\\\')
else:
return path
if __name__ == "__main__":
idf_path = os.getenv("IDF_PATH")
if __name__ == '__main__':
idf_path = os.getenv('IDF_PATH')
default_requirements_path = os.path.join(idf_path, 'requirements.txt')
@@ -72,30 +72,30 @@ if __name__ == "__main__":
elif os.environ.get('IDF_PYTHON_ENV_PATH'):
# We are running inside a private virtual environment under IDF_TOOLS_PATH,
# ask the user to run install.bat again.
if sys.platform == "win32" and not os.environ.get("MSYSTEM"):
if sys.platform == 'win32' and not os.environ.get('MSYSTEM'):
install_script = 'install.bat'
else:
install_script = 'install.sh'
print('To install the missing packages, please run "%s"' % os.path.join(idf_path, install_script))
elif sys.platform == "win32" and os.environ.get("MSYSTEM", None) == "MINGW32" and "/mingw32/bin/python" in sys.executable:
elif sys.platform == 'win32' and os.environ.get('MSYSTEM', None) == 'MINGW32' and '/mingw32/bin/python' in sys.executable:
print("The recommended way to install a packages is via \"pacman\". Please run \"pacman -Ss <package_name>\" for"
" searching the package database and if found then "
' searching the package database and if found then '
"\"pacman -S mingw-w64-i686-python-<package_name>\" for installing it.")
print("NOTE: You may need to run \"pacman -Syu\" if your package database is older and run twice if the "
"previous run updated \"pacman\" itself.")
print("Please read https://github.com/msys2/msys2/wiki/Using-packages for further information about using "
print('Please read https://github.com/msys2/msys2/wiki/Using-packages for further information about using '
"\"pacman\"")
# Special case for MINGW32 Python, needs some packages
# via MSYS2 not via pip or system breaks...
for requirement in not_satisfied:
if requirement.startswith('cryptography'):
print("WARNING: The cryptography package have dependencies on system packages so please make sure "
print('WARNING: The cryptography package have dependencies on system packages so please make sure '
"you run \"pacman -Syu\" followed by \"pacman -S mingw-w64-i686-python{}-cryptography\"."
"".format(sys.version_info[0],))
''.format(sys.version_info[0],))
continue
elif requirement.startswith('setuptools'):
print("Please run the following command to install MSYS2's MINGW Python setuptools package:")
print("pacman -S mingw-w64-i686-python-setuptools")
print('pacman -S mingw-w64-i686-python-setuptools')
continue
else:
print('Please follow the instructions found in the "Set up the tools" section of '

1
tools/check_term.py
View File

@@ -15,6 +15,7 @@
# limitations under the License.
from __future__ import print_function
import os
import sys

13
tools/ci/apply_bot_filter.py
View File

@@ -3,15 +3,14 @@
# internal use only
# called by CI jobs to determine if it need to be executed
import json
import os
import re
import sys
import json
RE_FILTER_PATTERN = re.compile(r'^r"(.+)?"$')
RE_TYPE = type(re.compile("", 0))
RE_TYPE = type(re.compile('', 0))
def parse_filter(filter_name):
@@ -50,13 +49,13 @@ def process_filter(execute_by_default, filter_name, ci_name):
return execute
if __name__ == "__main__":
if __name__ == '__main__':
execute_by_default = True
if os.getenv("BOT_NEEDS_TRIGGER_BY_NAME", "0") == "1":
if os.getenv('BOT_NEEDS_TRIGGER_BY_NAME', '0') == '1':
execute_by_default = False
need_to_execute = process_filter(True, "BOT_STAGE_FILTER", os.getenv("CI_JOB_STAGE")) and process_filter(execute_by_default,
"BOT_JOB_FILTER", os.getenv("CI_JOB_NAME"))
need_to_execute = process_filter(True, 'BOT_STAGE_FILTER', os.getenv('CI_JOB_STAGE')) and process_filter(execute_by_default,
'BOT_JOB_FILTER', os.getenv('CI_JOB_NAME'))
if need_to_execute:
sys.exit(0)
else:

18
tools/ci/check_artifacts_expire_time.py
View File

@@ -12,42 +12,42 @@ try:
except ImportError:
from yaml import Loader as Loader
IDF_PATH = os.getenv("IDF_PATH")
IDF_PATH = os.getenv('IDF_PATH')
if not IDF_PATH:
print("Please set IDF_PATH before running this script")
print('Please set IDF_PATH before running this script')
raise SystemExit(-1)
GITLAB_CONFIG_FILE = os.path.join(os.getenv("IDF_PATH"), ".gitlab-ci.yml")
GITLAB_CONFIG_FILE = os.path.join(os.getenv('IDF_PATH'), '.gitlab-ci.yml')
def check_artifacts_expire_time():
with open(GITLAB_CONFIG_FILE, "r") as f:
with open(GITLAB_CONFIG_FILE, 'r') as f:
config = yaml.load(f, Loader=Loader)
errors = []
print("expire time for jobs:")
print('expire time for jobs:')
job_names = list(config.keys())
job_names.sort()
for job_name in job_names:
if job_name.startswith("."):
if job_name.startswith('.'):
# skip ignored jobs
continue
try:
if "expire_in" not in config[job_name]["artifacts"]:
if 'expire_in' not in config[job_name]['artifacts']:
errors.append(job_name)
else:
print("{}: {}".format(job_name, config[job_name]["artifacts"]["expire_in"]))
print('{}: {}'.format(job_name, config[job_name]['artifacts']['expire_in']))
except (KeyError, TypeError):
# this is not job, or the job does not have artifacts
pass
if errors:
print("\n\nThe following jobs did not set expire time for its artifacts")
print('\n\nThe following jobs did not set expire time for its artifacts')
for error in errors:
print(error)
raise SystemExit(-2)

66
tools/ci/check_build_warnings.py
View File

@@ -15,23 +15,23 @@ import sys
try:
from find_build_apps import BuildItem, setup_logging
except ImportError:
sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
from find_build_apps import BuildItem, setup_logging
WARNING_REGEX = re.compile(r"(?:error|warning)[^\w]", re.MULTILINE | re.IGNORECASE)
WARNING_REGEX = re.compile(r'(?:error|warning)[^\w]', re.MULTILINE | re.IGNORECASE)
IGNORE_WARNS = [
re.compile(r_str) for r_str in [
r"library/error\.o",
r".*error.*\.c\.obj",
r"-Werror",
r"error\.d",
r"reassigning to symbol",
r"changes choice state",
r"crosstool_version_check\.cmake",
r"CryptographyDeprecationWarning",
r"Python 3 versions older than 3.6 are not supported.",
r"Support for Python 2 is deprecated and will be removed in future versions.",
r'library/error\.o',
r'.*error.*\.c\.obj',
r'-Werror',
r'error\.d',
r'reassigning to symbol',
r'changes choice state',
r'crosstool_version_check\.cmake',
r'CryptographyDeprecationWarning',
r'Python 3 versions older than 3.6 are not supported.',
r'Support for Python 2 is deprecated and will be removed in future versions.',
]
]
@@ -50,54 +50,54 @@ def line_has_warnings(line): # type: (str) -> bool
def main():
parser = argparse.ArgumentParser(description="ESP-IDF app builder")
parser = argparse.ArgumentParser(description='ESP-IDF app builder')
parser.add_argument(
"-v",
"--verbose",
action="count",
help="Increase the logging level of the script. Can be specified multiple times.",
'-v',
'--verbose',
action='count',
help='Increase the logging level of the script. Can be specified multiple times.',
)
parser.add_argument(
"--log-file",
type=argparse.FileType("w"),
help="Write the script log to the specified file, instead of stderr",
'--log-file',
type=argparse.FileType('w'),
help='Write the script log to the specified file, instead of stderr',
)
parser.add_argument(
"build_list",
type=argparse.FileType("r"),
nargs="?",
'build_list',
type=argparse.FileType('r'),
nargs='?',
default=sys.stdin,
help="Name of the file to read the list of builds from. If not specified, read from stdin.",
help='Name of the file to read the list of builds from. If not specified, read from stdin.',
)
args = parser.parse_args()
setup_logging(args)
build_items = [BuildItem.from_json(line) for line in args.build_list]
if not build_items:
logging.warning("Empty build list")
logging.warning('Empty build list')
SystemExit(0)
found_warnings = 0
for build_item in build_items:
if not build_item.build_log_path:
logging.debug("No log file for {}".format(build_item.work_dir))
logging.debug('No log file for {}'.format(build_item.work_dir))
continue
with open(build_item.build_log_path, "r") as log_file:
with open(build_item.build_log_path, 'r') as log_file:
for line_no, line in enumerate(log_file):
if line_has_warnings(line):
logging.error("Issue in app {}, config {}:".format(build_item.app_dir, build_item.config_name))
logging.error(line.rstrip("\n"))
logging.error("See {}:{} for details".format(os.path.basename(build_item.build_log_path),
logging.error('Issue in app {}, config {}:'.format(build_item.app_dir, build_item.config_name))
logging.error(line.rstrip('\n'))
logging.error('See {}:{} for details'.format(os.path.basename(build_item.build_log_path),
line_no + 1))
found_warnings += 1
break
if found_warnings:
logging.error("Checked {} builds, found {} warnings".format(len(build_items), found_warnings))
logging.error('Checked {} builds, found {} warnings'.format(len(build_items), found_warnings))
raise SystemExit(1)
logging.info("No warnings found")
logging.info('No warnings found')
if __name__ == "__main__":
if __name__ == '__main__':
main()

112
tools/ci/check_callgraph.py
View File

@@ -16,20 +16,20 @@
# limitations under the License.
import argparse
from functools import partial
import os
import re
from functools import partial
import elftools
from elftools.elf import elffile
try:
from typing import List, Optional, BinaryIO, Tuple, Generator, Dict, Callable
from typing import BinaryIO, Callable, Dict, Generator, List, Optional, Tuple
except ImportError:
pass
FUNCTION_REGEX = re.compile(
r"^;; Function (?P<mangle>.*)\s+\((?P<function>\S+)(,.*)?\).*$"
r'^;; Function (?P<mangle>.*)\s+\((?P<function>\S+)(,.*)?\).*$'
)
CALL_REGEX = re.compile(r'^.*\(call.*"(?P<target>.*)".*$')
SYMBOL_REF_REGEX = re.compile(r'^.*\(symbol_ref[^()]*\("(?P<target>.*)"\).*$')
@@ -52,24 +52,24 @@ class SectionAddressRange(object):
self.high = addr + size
def __str__(self):
return "{}: 0x{:08x} - 0x{:08x}".format(self.name, self.low, self.high)
return '{}: 0x{:08x} - 0x{:08x}'.format(self.name, self.low, self.high)
def contains_address(self, addr):
return self.low <= addr < self.high
TARGET_SECTIONS = {
"esp32": [
SectionAddressRange(".rom.text", 0x40000000, 0x70000),
SectionAddressRange(".rom.rodata", 0x3ff96000, 0x9018)
'esp32': [
SectionAddressRange('.rom.text', 0x40000000, 0x70000),
SectionAddressRange('.rom.rodata', 0x3ff96000, 0x9018)
],
"esp32s2": [
SectionAddressRange(".rom.text", 0x40000000, 0x1bed0),
SectionAddressRange(".rom.rodata", 0x3ffac600, 0x392c)
'esp32s2': [
SectionAddressRange('.rom.text', 0x40000000, 0x1bed0),
SectionAddressRange('.rom.rodata', 0x3ffac600, 0x392c)
],
"esp32s3": [
SectionAddressRange(".rom.text", 0x40000000, 0x568d0),
SectionAddressRange(".rom.rodata", 0x3ff071c0, 0x8e30)
'esp32s3': [
SectionAddressRange('.rom.text', 0x40000000, 0x568d0),
SectionAddressRange('.rom.rodata', 0x3ff071c0, 0x8e30)
]
} # type: Dict[str, List[SectionAddressRange]]
@@ -85,11 +85,11 @@ class Symbol(object):
self.referred_from = list() # type: List[Symbol]
def __str__(self):
return "{} @0x{:08x} [{}]{} {}".format(
return '{} @0x{:08x} [{}]{} {}'.format(
self.name,
self.addr,
self.section or "unknown",
" (local)" if self.local else "",
self.section or 'unknown',
' (local)' if self.local else '',
self.filename
)
@@ -100,7 +100,7 @@ class Reference(object):
self.to_sym = to_sym
def __str__(self):
return "{} @0x{:08x} ({}) -> {} @0x{:08x} ({})".format(
return '{} @0x{:08x} ({}) -> {} @0x{:08x} ({})'.format(
self.from_sym.name,
self.from_sym.addr,
self.from_sym.section,
@@ -124,12 +124,12 @@ class ElfInfo(object):
continue
filename = None
for sym in s.iter_symbols():
sym_type = sym.entry["st_info"]["type"]
if sym_type == "STT_FILE":
sym_type = sym.entry['st_info']['type']
if sym_type == 'STT_FILE':
filename = sym.name
if sym_type in ["STT_NOTYPE", "STT_FUNC", "STT_OBJECT"]:
local = sym.entry["st_info"]["bind"] == "STB_LOCAL"
addr = sym.entry["st_value"]
if sym_type in ['STT_NOTYPE', 'STT_FUNC', 'STT_OBJECT']:
local = sym.entry['st_info']['bind'] == 'STB_LOCAL'
addr = sym.entry['st_value']
symbols.append(
Symbol(
sym.name,
@@ -144,17 +144,17 @@ class ElfInfo(object):
def _load_sections(self): # type: () -> List[SectionAddressRange]
result = []
for segment in self.elf_obj.iter_segments():
if segment["p_type"] == "PT_LOAD":
if segment['p_type'] == 'PT_LOAD':
for section in self.elf_obj.iter_sections():
if not segment.section_in_segment(section):
continue
result.append(
SectionAddressRange(
section.name, section["sh_addr"], section["sh_size"]
section.name, section['sh_addr'], section['sh_size']
)
)
target = os.environ.get("IDF_TARGET")
target = os.environ.get('IDF_TARGET')
if target in TARGET_SECTIONS:
result += TARGET_SECTIONS[target]
@@ -180,7 +180,7 @@ def load_rtl_file(rtl_filename, tu_filename, functions): # type: (str, str, Lis
# Find function definition
match = re.match(FUNCTION_REGEX, line)
if match:
function_name = match.group("function")
function_name = match.group('function')
last_function = RtlFunction(function_name, rtl_filename, tu_filename)
functions.append(last_function)
continue
@@ -189,7 +189,7 @@ def load_rtl_file(rtl_filename, tu_filename, functions): # type: (str, str, Lis
# Find direct function calls
match = re.match(CALL_REGEX, line)
if match:
target = match.group("target")
target = match.group('target')
if target not in last_function.calls:
last_function.calls.append(target)
continue
@@ -197,7 +197,7 @@ def load_rtl_file(rtl_filename, tu_filename, functions): # type: (str, str, Lis
# Find symbol references
match = re.match(SYMBOL_REF_REGEX, line)
if match:
target = match.group("target")
target = match.group('target')
if target not in last_function.refs:
last_function.refs.append(target)
continue
@@ -298,7 +298,7 @@ def match_rtl_funcs_to_symbols(rtl_functions, elfinfo): # type: (List[RtlFuncti
symbols.append(sym_from)
for target_rtl_func_name in source_rtl_func.calls + source_rtl_func.refs:
if "*.LC" in target_rtl_func_name: # skip local labels
if '*.LC' in target_rtl_func_name: # skip local labels
continue
maybe_sym_to = find_symbol_by_name(target_rtl_func_name, elfinfo, partial(match_local_target_func, source_rtl_func.rtl_filename, sym_from))
@@ -351,68 +351,68 @@ def main():
parser = argparse.ArgumentParser()
parser.add_argument(
"--rtl-list",
help="File with the list of RTL files",
type=argparse.FileType("r"),
'--rtl-list',
help='File with the list of RTL files',
type=argparse.FileType('r'),
)
parser.add_argument(
"--rtl-dir", help="Directory where to look for RTL files, recursively"
'--rtl-dir', help='Directory where to look for RTL files, recursively'
)
parser.add_argument(
"--elf-file",
'--elf-file',
required=True,
help="Program ELF file",
type=argparse.FileType("rb"),
help='Program ELF file',
type=argparse.FileType('rb'),
)
action_sub = parser.add_subparsers(dest="action")
action_sub = parser.add_subparsers(dest='action')
find_refs_parser = action_sub.add_parser(
"find-refs",
help="List the references coming from a given list of source sections"
"to a given list of target sections.",
'find-refs',
help='List the references coming from a given list of source sections'
'to a given list of target sections.',
)
find_refs_parser.add_argument(
"--from-sections", help="comma-separated list of source sections"
'--from-sections', help='comma-separated list of source sections'
)
find_refs_parser.add_argument(
"--to-sections", help="comma-separated list of target sections"
'--to-sections', help='comma-separated list of target sections'
)
find_refs_parser.add_argument(
"--exit-code",
action="store_true",
help="If set, exits with non-zero code when any references found",
'--exit-code',
action='store_true',
help='If set, exits with non-zero code when any references found',
)
action_sub.add_parser(
"all-refs",
help="Print the list of all references",
'all-refs',
help='Print the list of all references',
)
parser.parse_args()
args = parser.parse_args()
if args.rtl_list:
with open(args.rtl_list, "r") as rtl_list_file:
with open(args.rtl_list, 'r') as rtl_list_file:
rtl_list = [line.strip for line in rtl_list_file]
else:
if not args.rtl_dir:
raise RuntimeError("Either --rtl-list or --rtl-dir must be specified")
rtl_list = list(find_files_recursive(args.rtl_dir, ".expand"))
raise RuntimeError('Either --rtl-list or --rtl-dir must be specified')
rtl_list = list(find_files_recursive(args.rtl_dir, '.expand'))
if not rtl_list:
raise RuntimeError("No RTL files specified")
raise RuntimeError('No RTL files specified')
_, refs = get_symbols_and_refs(rtl_list, args.elf_file)
if args.action == "find-refs":
from_sections = args.from_sections.split(",") if args.from_sections else []
to_sections = args.to_sections.split(",") if args.to_sections else []
if args.action == 'find-refs':
from_sections = args.from_sections.split(',') if args.from_sections else []
to_sections = args.to_sections.split(',') if args.to_sections else []
found = list_refs_from_to_sections(
refs, from_sections, to_sections
)
if args.exit_code and found:
raise SystemExit(1)
elif args.action == "all-refs":
elif args.action == 'all-refs':
for r in refs:
print(str(r))
if __name__ == "__main__":
if __name__ == '__main__':
main()

58
tools/ci/check_codeowners.py
View File

@@ -24,8 +24,8 @@ import sys
from idf_ci_utils import IDF_PATH
CODEOWNERS_PATH = os.path.join(IDF_PATH, ".gitlab", "CODEOWNERS")
CODEOWNER_GROUP_PREFIX = "@esp-idf-codeowners/"
CODEOWNERS_PATH = os.path.join(IDF_PATH, '.gitlab', 'CODEOWNERS')
CODEOWNER_GROUP_PREFIX = '@esp-idf-codeowners/'
def get_all_files():
@@ -33,7 +33,7 @@ def get_all_files():
Get list of all file paths in the repository.
"""
# only split on newlines, since file names may contain spaces
return subprocess.check_output(["git", "ls-files"], cwd=IDF_PATH).decode("utf-8").strip().split('\n')
return subprocess.check_output(['git', 'ls-files'], cwd=IDF_PATH).decode('utf-8').strip().split('\n')
def pattern_to_regex(pattern):
@@ -93,7 +93,7 @@ def action_identify(args):
with open(CODEOWNERS_PATH) as f:
for line in f:
line = line.strip()
if not line or line.startswith("#"):
if not line or line.startswith('#'):
continue
tokens = line.split()
path_pattern = tokens[0]
@@ -121,18 +121,18 @@ def action_ci_check(args):
errors = []
def add_error(msg):
errors.append("{}:{}: {}".format(CODEOWNERS_PATH, line_no, msg))
errors.append('{}:{}: {}'.format(CODEOWNERS_PATH, line_no, msg))
all_files = get_all_files()
prev_path_pattern = ""
prev_path_pattern = ''
with open(CODEOWNERS_PATH) as f:
for line_no, line in enumerate(f, start=1):
# Skip empty lines and comments
line = line.strip()
if line.startswith("# sort-order-reset"):
prev_path_pattern = ""
if line.startswith('# sort-order-reset'):
prev_path_pattern = ''
if not line or line.startswith("#"):
if not line or line.startswith('#'):
continue
# Each line has a form of "<path> <owners>+"
@@ -140,18 +140,18 @@ def action_ci_check(args):
path_pattern = tokens[0]
owners = tokens[1:]
if not owners:
add_error("no owners specified for {}".format(path_pattern))
add_error('no owners specified for {}'.format(path_pattern))
# Check that the file is sorted by path patterns
path_pattern_for_cmp = path_pattern.replace("-", "_") # ignore difference between _ and - for ordering
path_pattern_for_cmp = path_pattern.replace('-', '_') # ignore difference between _ and - for ordering
if prev_path_pattern and path_pattern_for_cmp < prev_path_pattern:
add_error("file is not sorted: {} < {}".format(path_pattern_for_cmp, prev_path_pattern))
add_error('file is not sorted: {} < {}'.format(path_pattern_for_cmp, prev_path_pattern))
prev_path_pattern = path_pattern_for_cmp
# Check that the pattern matches at least one file
files = files_by_pattern(all_files, path_pattern)
if not files:
add_error("no files matched by pattern {}".format(path_pattern))
add_error('no files matched by pattern {}'.format(path_pattern))
for o in owners:
# Sanity-check the owner group name
@@ -159,9 +159,9 @@ def action_ci_check(args):
add_error("owner {} doesn't start with {}".format(o, CODEOWNER_GROUP_PREFIX))
if not errors:
print("No errors found.")
print('No errors found.')
else:
print("Errors found!")
print('Errors found!')
for e in errors:
print(e)
raise SystemExit(1)
@@ -169,29 +169,29 @@ def action_ci_check(args):
def main():
parser = argparse.ArgumentParser(
sys.argv[0], description="Internal helper script for working with the CODEOWNERS file."
sys.argv[0], description='Internal helper script for working with the CODEOWNERS file.'
)
subparsers = parser.add_subparsers(dest="action")
subparsers = parser.add_subparsers(dest='action')
identify = subparsers.add_parser(
"identify",
help="List the owners of the specified path within IDF."
'identify',
help='List the owners of the specified path within IDF.'
"This command doesn't support files inside submodules, or files not added to git repository.",
)
identify.add_argument("path", help="Path of the file relative to the root of the repository")
identify.add_argument('path', help='Path of the file relative to the root of the repository')
subparsers.add_parser(
"ci-check",
help="Check CODEOWNERS file: every line should match at least one file, sanity-check group names, "
"check that the file is sorted by paths",
'ci-check',
help='Check CODEOWNERS file: every line should match at least one file, sanity-check group names, '
'check that the file is sorted by paths',
)
test_pattern = subparsers.add_parser(
"test-pattern",
help="Print files in the repository for a given CODEOWNERS pattern. Useful when adding new rules."
'test-pattern',
help='Print files in the repository for a given CODEOWNERS pattern. Useful when adding new rules.'
)
test_pattern.add_argument("--regex", action="store_true", help="Print the equivalent regular expression instead of the file list.")
test_pattern.add_argument("pattern", help="Path pattern to get the list of files for")
test_pattern.add_argument('--regex', action='store_true', help='Print the equivalent regular expression instead of the file list.')
test_pattern.add_argument('pattern', help='Path pattern to get the list of files for')
args = parser.parse_args()
@@ -199,10 +199,10 @@ def main():
parser.print_help()
parser.exit(1)
action_func_name = "action_" + args.action.replace("-", "_")
action_func_name = 'action_' + args.action.replace('-', '_')
action_func = globals()[action_func_name]
action_func(args)
if __name__ == "__main__":
if __name__ == '__main__':
main()

7
tools/ci/check_deprecated_kconfigs.py
View File

@@ -14,8 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import print_function
from __future__ import unicode_literals
from __future__ import print_function, unicode_literals
import argparse
import os
@@ -47,7 +46,7 @@ def _parse_path(path, sep=None):
def _valid_directory(path):
if not os.path.isdir(path):
raise argparse.ArgumentTypeError("{} is not a valid directory!".format(path))
raise argparse.ArgumentTypeError('{} is not a valid directory!'.format(path))
return path
@@ -114,5 +113,5 @@ def main():
return 0
if __name__ == "__main__":
if __name__ == '__main__':
sys.exit(main())

38
tools/ci/check_examples_cmake_make.py
View File

@@ -1,10 +1,10 @@
#!/usr/bin/env python
import os
import sys
import pprint
import json
import os
import pprint
import subprocess
import sys
# =============================================================================
# Service funcs
@@ -12,7 +12,7 @@ import subprocess
def _build_path(path, *paths):
return str(os.path.normpath(os.path.join(path, *paths)).replace("\\", "/"))
return str(os.path.normpath(os.path.join(path, *paths)).replace('\\', '/'))
def _unify_paths(path_list):
@@ -20,7 +20,7 @@ def _unify_paths(path_list):
def _exclude_by_pat_list(path_list, ignore_list):
print("- Applying ignore list")
print('- Applying ignore list')
path_list_res = list(path_list)
for ign in ignore_list:
if len(ign.strip()):
@@ -50,7 +50,7 @@ def get_idf_path(path, *paths):
def _get_apps(target, build_system):
print("- Getting paths of apps")
print('- Getting paths of apps')
args = [sys.executable,
get_idf_path('tools/find_apps.py'),
'-p',
@@ -79,21 +79,21 @@ def get_apps(target, build_system, ignorelist):
def get_cmake_ignore_list():
print("- Getting CMake ignore list")
print('- Getting CMake ignore list')
return _file2linelist(
get_idf_path("tools", "ci",
"check_examples_cmake_make-cmake_ignore.txt"))
get_idf_path('tools', 'ci',
'check_examples_cmake_make-cmake_ignore.txt'))
def get_make_ignore_list():
print("- Getting Make ignore list")
print('- Getting Make ignore list')
return _file2linelist(
get_idf_path("tools", "ci",
"check_examples_cmake_make-make_ignore.txt"))
get_idf_path('tools', 'ci',
'check_examples_cmake_make-make_ignore.txt'))
def diff(first, second):
print("- Comparing...")
print('- Comparing...')
first = set(first)
second = set(second)
res = list(first - second) + list(second - first)
@@ -103,21 +103,21 @@ def diff(first, second):
def main():
cmake_ignore = get_cmake_ignore_list()
make_ignore = get_make_ignore_list()
cmakes = get_apps("esp32", "cmake", cmake_ignore)
makes = get_apps("esp32", "make", make_ignore)
cmakes = get_apps('esp32', 'cmake', cmake_ignore)
makes = get_apps('esp32', 'make', make_ignore)
res = diff(cmakes, makes)
if len(res):
pp = pprint.PrettyPrinter(indent=4)
print(
"[ ERROR ] Some projects are not containing Make and Cmake project files:"
'[ ERROR ] Some projects are not containing Make and Cmake project files:'
)
pp.pprint(res)
raise ValueError("Test is not passed")
raise ValueError('Test is not passed')
else:
print("[ DONE ]")
print('[ DONE ]')
if __name__ == "__main__":
if __name__ == '__main__':
main()

15
tools/ci/check_kconfigs.py
View File

@@ -14,8 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import print_function
from __future__ import unicode_literals
from __future__ import print_function, unicode_literals
import argparse
import os
@@ -23,7 +22,7 @@ import re
import sys
from io import open
from idf_ci_utils import get_submodule_dirs, IDF_PATH
from idf_ci_utils import IDF_PATH, get_submodule_dirs
# regular expression for matching Kconfig files
RE_KCONFIG = r'^Kconfig(\.projbuild)?(\.in)?$'
@@ -101,7 +100,7 @@ class SourceChecker(BaseChecker):
if path in ['$COMPONENT_KCONFIGS_SOURCE_FILE', '$COMPONENT_KCONFIGS_PROJBUILD_SOURCE_FILE']:
pass
elif not filename.startswith('Kconfig.'):
raise InputError(self.path_in_idf, line_number, "only filenames starting with Kconfig.* can be sourced",
raise InputError(self.path_in_idf, line_number, 'only filenames starting with Kconfig.* can be sourced',
line.replace(path, os.path.join(os.path.dirname(path), 'Kconfig.' + filename)))
@@ -124,7 +123,7 @@ class LineRuleChecker(BaseChecker):
if rule[2]:
line = rule[0].sub(rule[2], line)
if len(errors) > 0:
raise InputError(self.path_in_idf, line_number, "; ".join(errors), line)
raise InputError(self.path_in_idf, line_number, '; '.join(errors), line)
class IndentAndNameChecker(BaseChecker):
@@ -369,7 +368,7 @@ class IndentAndNameChecker(BaseChecker):
def valid_directory(path):
if not os.path.isdir(path):
raise argparse.ArgumentTypeError("{} is not a valid directory!".format(path))
raise argparse.ArgumentTypeError('{} is not a valid directory!'.format(path))
return path
@@ -394,7 +393,7 @@ def validate_kconfig_file(kconfig_full_path, verbose=False): # type: (str, bool
fail = True
f_o.write(e.suggested_line)
except UnicodeDecodeError:
raise ValueError("The encoding of {} is not Unicode.".format(kconfig_full_path))
raise ValueError('The encoding of {} is not Unicode.'.format(kconfig_full_path))
if fail:
print('\t{} has been saved with suggestions for resolving the issues.\n'
@@ -475,5 +474,5 @@ def main():
return 0
if __name__ == "__main__":
if __name__ == '__main__':
sys.exit(main())

134
tools/ci/check_public_headers.py
View File

@@ -17,18 +17,18 @@
# limitations under the License.
#
from __future__ import print_function
from __future__ import unicode_literals
import re
import os
import subprocess
import json
import fnmatch
from __future__ import print_function, unicode_literals
import argparse
import fnmatch
import json
import os
import queue
from threading import Thread, Event
import re
import subprocess
import tempfile
from io import open
from threading import Event, Thread
class HeaderFailed(Exception):
@@ -38,30 +38,30 @@ class HeaderFailed(Exception):
class HeaderFailedSdkconfig(HeaderFailed):
def __str__(self):
return "Sdkconfig Error"
return 'Sdkconfig Error'
class HeaderFailedBuildError(HeaderFailed):
def __str__(self):
return "Header Build Error"
return 'Header Build Error'
class HeaderFailedCppGuardMissing(HeaderFailed):
def __str__(self):
return "Header Missing C++ Guard"
return 'Header Missing C++ Guard'
class HeaderFailedContainsCode(HeaderFailed):
def __str__(self):
return "Header Produced non-zero object"
return 'Header Produced non-zero object'
# Creates a temp file and returns both output as a string and a file name
#
def exec_cmd_to_temp_file(what, suffix=""):
def exec_cmd_to_temp_file(what, suffix=''):
out_file = tempfile.NamedTemporaryFile(suffix=suffix, delete=False)
rc, out, err = exec_cmd(what, out_file)
with open(out_file.name, "r", encoding='utf-8') as f:
with open(out_file.name, 'r', encoding='utf-8') as f:
out = f.read()
return rc, out, err, out_file.name
@@ -90,14 +90,14 @@ class PublicHeaderChecker:
print(message)
def __init__(self, verbose=False, jobs=1, prefix=None):
self.gcc = "{}gcc".format(prefix)
self.gpp = "{}g++".format(prefix)
self.gcc = '{}gcc'.format(prefix)
self.gpp = '{}g++'.format(prefix)
self.verbose = verbose
self.jobs = jobs
self.prefix = prefix
self.extern_c = re.compile(r'extern "C"')
self.error_macro = re.compile(r'#error')
self.error_orphan_kconfig = re.compile(r"#error CONFIG_VARS_USED_WHILE_SDKCONFIG_NOT_INCLUDED")
self.error_orphan_kconfig = re.compile(r'#error CONFIG_VARS_USED_WHILE_SDKCONFIG_NOT_INCLUDED')
self.kconfig_macro = re.compile(r'\bCONFIG_[A-Z0-9_]+')
self.assembly_nocode = r'^\s*(\.file|\.text|\.ident).*$'
self.check_threads = []
@@ -129,10 +129,10 @@ class PublicHeaderChecker:
try:
self.check_one_header(task, num)
except HeaderFailed as e:
self.failed_queue.put("{}: Failed! {}".format(task, e))
self.failed_queue.put('{}: Failed! {}'.format(task, e))
except Exception as e:
# Makes sure any unexpected exceptions causes the program to terminate
self.failed_queue.put("{}: Failed! {}".format(task, e))
self.failed_queue.put('{}: Failed! {}'.format(task, e))
self.terminate.set()
raise
@@ -174,7 +174,7 @@ class PublicHeaderChecker:
self.compile_one_header(header)
temp_header = None
try:
_, _, _, temp_header = exec_cmd_to_temp_file(["sed", "/#include/d; /#error/d", header], suffix=".h")
_, _, _, temp_header = exec_cmd_to_temp_file(['sed', '/#include/d; /#error/d', header], suffix='.h')
res = self.preprocess_one_header(temp_header, num, ignore_sdkconfig_issue=True)
if res == self.PREPROC_OUT_SAME_HRD_FAILED:
raise HeaderFailedCppGuardMissing()
@@ -185,53 +185,53 @@ class PublicHeaderChecker:
os.unlink(temp_header)
def compile_one_header(self, header):
rc, out, err = exec_cmd([self.gcc, "-S", "-o-", "-include", header, self.main_c] + self.include_dir_flags)
rc, out, err = exec_cmd([self.gcc, '-S', '-o-', '-include', header, self.main_c] + self.include_dir_flags)
if rc == 0:
if not re.sub(self.assembly_nocode, '', out, flags=re.M).isspace():
raise HeaderFailedContainsCode()
return # Header OK: produced zero code
self.log("{}: FAILED: compilation issue".format(header), True)
self.log('{}: FAILED: compilation issue'.format(header), True)
self.log(err, True)
raise HeaderFailedBuildError()
def preprocess_one_header(self, header, num, ignore_sdkconfig_issue=False):
all_compilation_flags = ["-w", "-P", "-E", "-DESP_PLATFORM", "-include", header, self.main_c] + self.include_dir_flags
all_compilation_flags = ['-w', '-P', '-E', '-DESP_PLATFORM', '-include', header, self.main_c] + self.include_dir_flags
if not ignore_sdkconfig_issue:
# just strip commnets to check for CONFIG_... macros
rc, out, err = exec_cmd([self.gcc, "-fpreprocessed", "-dD", "-P", "-E", header] + self.include_dir_flags)
rc, out, err = exec_cmd([self.gcc, '-fpreprocessed', '-dD', '-P', '-E', header] + self.include_dir_flags)
if re.search(self.kconfig_macro, out):
# enable defined #error if sdkconfig.h not included
all_compilation_flags.append("-DIDF_CHECK_SDKCONFIG_INCLUDED")
all_compilation_flags.append('-DIDF_CHECK_SDKCONFIG_INCLUDED')
try:
# compile with C++, check for errors, outputs for a temp file
rc, cpp_out, err, cpp_out_file = exec_cmd_to_temp_file([self.gpp, "--std=c++17"] + all_compilation_flags)
rc, cpp_out, err, cpp_out_file = exec_cmd_to_temp_file([self.gpp, '--std=c++17'] + all_compilation_flags)
if rc != 0:
if re.search(self.error_macro, err):
if re.search(self.error_orphan_kconfig, err):
self.log("{}: CONFIG_VARS_USED_WHILE_SDKCONFIG_NOT_INCLUDED".format(header), True)
self.log('{}: CONFIG_VARS_USED_WHILE_SDKCONFIG_NOT_INCLUDED'.format(header), True)
return self.COMPILE_ERR_REF_CONFIG_HDR_FAILED
self.log("{}: Error directive failure: OK".format(header))
self.log('{}: Error directive failure: OK'.format(header))
return self.COMPILE_ERR_ERROR_MACRO_HDR_OK
self.log("{}: FAILED: compilation issue".format(header), True)
self.log('{}: FAILED: compilation issue'.format(header), True)
self.log(err)
return self.COMPILE_ERR_HDR_FAILED
# compile with C compiler, outputs to another temp file
rc, c99_out, err, c99_out_file = exec_cmd_to_temp_file([self.gcc, "--std=c99"] + all_compilation_flags)
rc, c99_out, err, c99_out_file = exec_cmd_to_temp_file([self.gcc, '--std=c99'] + all_compilation_flags)
if rc != 0:
self.log("{} FAILED should never happen".format(header))
self.log('{} FAILED should never happen'.format(header))
return self.COMPILE_ERR_HDR_FAILED
# diff the two outputs
rc, diff, err = exec_cmd(["diff", c99_out_file, cpp_out_file])
rc, diff, err = exec_cmd(['diff', c99_out_file, cpp_out_file])
if not diff or diff.isspace():
if not cpp_out or cpp_out.isspace():
self.log("{} The same, but empty out - OK".format(header))
self.log('{} The same, but empty out - OK'.format(header))
return self.PREPROC_OUT_ZERO_HDR_OK
self.log("{} FAILED C and C++ preprocessor output is the same!".format(header), True)
self.log('{} FAILED C and C++ preprocessor output is the same!'.format(header), True)
return self.PREPROC_OUT_SAME_HRD_FAILED
if re.search(self.extern_c, diff):
self.log("{} extern C present - OK".format(header))
self.log('{} extern C present - OK'.format(header))
return self.PREPROC_OUT_DIFFERENT_WITH_EXT_C_HDR_OK
self.log("{} Different but no extern C - FAILED".format(header), True)
self.log('{} Different but no extern C - FAILED'.format(header), True)
return self.PREPROC_OUT_DIFFERENT_NO_EXT_C_HDR_FAILED
finally:
os.unlink(cpp_out_file)
@@ -243,32 +243,32 @@ class PublicHeaderChecker:
# Get compilation data from an example to list all public header files
def list_public_headers(self, ignore_dirs, ignore_files, only_dir=None):
idf_path = os.getenv('IDF_PATH')
project_dir = os.path.join(idf_path, "examples", "get-started", "blink")
subprocess.check_call(["idf.py", "reconfigure"], cwd=project_dir)
build_commands_json = os.path.join(project_dir, "build", "compile_commands.json")
with open(build_commands_json, "r", encoding='utf-8') as f:
build_command = json.load(f)[0]["command"].split()
project_dir = os.path.join(idf_path, 'examples', 'get-started', 'blink')
subprocess.check_call(['idf.py', 'reconfigure'], cwd=project_dir)
build_commands_json = os.path.join(project_dir, 'build', 'compile_commands.json')
with open(build_commands_json, 'r', encoding='utf-8') as f:
build_command = json.load(f)[0]['command'].split()
include_dir_flags = []
include_dirs = []
# process compilation flags (includes and defines)
for item in build_command:
if item.startswith("-I"):
if item.startswith('-I'):
include_dir_flags.append(item)
if "components" in item:
if 'components' in item:
include_dirs.append(item[2:]) # Removing the leading "-I"
if item.startswith("-D"):
if item.startswith('-D'):
include_dir_flags.append(item.replace('\\','')) # removes escaped quotes, eg: -DMBEDTLS_CONFIG_FILE=\\\"mbedtls/esp_config.h\\\"
include_dir_flags.append("-I" + os.path.join(project_dir, "build", "config"))
include_dir_flags.append("-DCI_HEADER_CHECK")
sdkconfig_h = os.path.join(project_dir, "build", "config", "sdkconfig.h")
include_dir_flags.append('-I' + os.path.join(project_dir, 'build', 'config'))
include_dir_flags.append('-DCI_HEADER_CHECK')
sdkconfig_h = os.path.join(project_dir, 'build', 'config', 'sdkconfig.h')
# prepares a main_c file for easier sdkconfig checks and avoid compilers warning when compiling headers directly
with open(sdkconfig_h, "a") as f:
f.write("#define IDF_SDKCONFIG_INCLUDED")
main_c = os.path.join(project_dir, "build", "compile.c")
with open(main_c, "w") as f:
f.write("#if defined(IDF_CHECK_SDKCONFIG_INCLUDED) && ! defined(IDF_SDKCONFIG_INCLUDED)\n"
"#error CONFIG_VARS_USED_WHILE_SDKCONFIG_NOT_INCLUDED\n"
"#endif")
with open(sdkconfig_h, 'a') as f:
f.write('#define IDF_SDKCONFIG_INCLUDED')
main_c = os.path.join(project_dir, 'build', 'compile.c')
with open(main_c, 'w') as f:
f.write('#if defined(IDF_CHECK_SDKCONFIG_INCLUDED) && ! defined(IDF_SDKCONFIG_INCLUDED)\n'
'#error CONFIG_VARS_USED_WHILE_SDKCONFIG_NOT_INCLUDED\n'
'#endif')
# processes public include dirs, removing ignored files
all_include_files = []
files_to_check = []
@@ -277,7 +277,7 @@ class PublicHeaderChecker:
self.log('{} - directory ignored (not in "{}")'.format(d, only_dir))
continue
if os.path.relpath(d, idf_path).startswith(tuple(ignore_dirs)):
self.log("{} - directory ignored".format(d))
self.log('{} - directory ignored'.format(d))
continue
for root, dirnames, filenames in os.walk(d):
for filename in fnmatch.filter(filenames, '*.h'):
@@ -289,10 +289,10 @@ class PublicHeaderChecker:
for f in all_include_files:
rel_path_file = os.path.relpath(f, idf_path)
if any([os.path.commonprefix([d, rel_path_file]) == d for d in ignore_dirs]):
self.log("{} - file ignored (inside ignore dir)".format(f))
self.log('{} - file ignored (inside ignore dir)'.format(f))
continue
if rel_path_file in ignore_files:
self.log("{} - file ignored".format(f))
self.log('{} - file ignored'.format(f))
continue
files_to_check.append(f)
# removes duplicates and places headers to a work queue
@@ -302,22 +302,22 @@ class PublicHeaderChecker:
def check_all_headers():
parser = argparse.ArgumentParser("Public header checker file")
parser.add_argument("--verbose", "-v", help="enables verbose mode", action="store_true")
parser.add_argument("--jobs", "-j", help="number of jobs to run checker", default=1, type=int)
parser.add_argument("--prefix", "-p", help="compiler prefix", default="xtensa-esp32-elf-", type=str)
parser.add_argument("--exclude-file", "-e", help="exception file", default="check_public_headers_exceptions.txt", type=str)
parser.add_argument("--only-dir", "-d", help="reduce the analysis to this directory only", default=None, type=str)
parser = argparse.ArgumentParser('Public header checker file')
parser.add_argument('--verbose', '-v', help='enables verbose mode', action='store_true')
parser.add_argument('--jobs', '-j', help='number of jobs to run checker', default=1, type=int)
parser.add_argument('--prefix', '-p', help='compiler prefix', default='xtensa-esp32-elf-', type=str)
parser.add_argument('--exclude-file', '-e', help='exception file', default='check_public_headers_exceptions.txt', type=str)
parser.add_argument('--only-dir', '-d', help='reduce the analysis to this directory only', default=None, type=str)
args = parser.parse_args()
# process excluded files and dirs
exclude_file = os.path.join(os.path.dirname(__file__), args.exclude_file)
with open(exclude_file, "r", encoding='utf-8') as f:
with open(exclude_file, 'r', encoding='utf-8') as f:
lines = [line.rstrip() for line in f]
ignore_files = []
ignore_dirs = []
for line in lines:
if not line or line.isspace() or line.startswith("#"):
if not line or line.isspace() or line.startswith('#'):
continue
if os.path.isdir(line):
ignore_dirs.append(line)
@@ -334,9 +334,9 @@ def check_all_headers():
for failed in failures:
print(failed)
exit(1)
print("No errors found")
print('No errors found')
except KeyboardInterrupt:
print("Keyboard interrupt")
print('Keyboard interrupt')
if __name__ == '__main__':

34
tools/ci/check_readme_links.py
View File

@@ -17,15 +17,15 @@
# limitations under the License.
#
import os
import re
import os.path
import urllib.request
import urllib.error
import concurrent.futures
import argparse
import concurrent.futures
import os
import os.path
import re
import urllib.error
import urllib.request
from collections import defaultdict, namedtuple
from pathlib import Path
from collections import namedtuple, defaultdict
EXCLUDE_DOCS_LIST = ['examples/peripherals/secure_element/atecc608_ecdsa/components/esp-cryptoauthlib/cryptoauthlib/**']
@@ -43,7 +43,7 @@ class ReadmeLinkError(Exception):
class RelativeLinkError(ReadmeLinkError):
def __str__(self):
return "Relative link error, file - {} not found, linked from {}".format(self.url, self.file)
return 'Relative link error, file - {} not found, linked from {}'.format(self.url, self.file)
class UrlLinkError(ReadmeLinkError):
@@ -53,7 +53,7 @@ class UrlLinkError(ReadmeLinkError):
def __str__(self):
files = [str(f) for f in self.file]
return "URL error, url - {} in files - {} is not accessible, request returned {}".format(self.url, ", ".join(files), self.error_code)
return 'URL error, url - {} in files - {} is not accessible, request returned {}'.format(self.url, ', '.join(files), self.error_code)
# we do not want a failed test just due to bad network conditions, for non 404 errors we simply print a warning
@@ -65,9 +65,9 @@ def check_url(url, files, timeout):
if e.code == 404:
raise UrlLinkError(files, url, str(e))
else:
print("Unable to access {}, err = {}".format(url, str(e)))
print('Unable to access {}, err = {}'.format(url, str(e)))
except Exception as e:
print("Unable to access {}, err = {}".format(url, str(e)))
print('Unable to access {}, err = {}'.format(url, str(e)))
def check_web_links(web_links):
@@ -93,19 +93,19 @@ def check_file_links(file_links):
if not Path.exists(link_path):
errors.append(RelativeLinkError(link.file, link.url))
print("Found {} errors with relative links".format(len(errors)))
print('Found {} errors with relative links'.format(len(errors)))
return errors
def get_md_links(folder):
MD_LINK_RE = r"\[.+?\]\((.+?)(#.+)?\)"
MD_LINK_RE = r'\[.+?\]\((.+?)(#.+)?\)'
idf_path = Path(os.getenv('IDF_PATH'))
links = []
for path in (idf_path / folder).rglob('*.md'):
if any([path.relative_to(idf_path).match(exclude_doc) for exclude_doc in EXCLUDE_DOCS_LIST]):
print("{} - excluded".format(path))
print('{} - excluded'.format(path))
continue
with path.open(encoding='utf8') as f:
@@ -123,7 +123,7 @@ def get_md_links(folder):
def check_readme_links(args):
links = get_md_links('examples')
print("Found {} links".format(len(links)))
print('Found {} links'.format(len(links)))
errors = []
@@ -145,7 +145,7 @@ def check_readme_links(args):
if not args.skip_weburl:
errors.extend(check_web_links(web_links))
print("Found {} errors:".format(len(errors)))
print('Found {} errors:'.format(len(errors)))
for e in errors:
print(e)
if errors:
@@ -155,7 +155,7 @@ def check_readme_links(args):
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='check_readme_links.py: Checks for dead links in example READMEs', prog='check_readme_links.py')
parser.add_argument("--skip-weburl", "-w", action='store_true', help="Skip checking of web URLs, only check links to local files")
parser.add_argument('--skip-weburl', '-w', action='store_true', help='Skip checking of web URLs, only check links to local files')
args = parser.parse_args()
check_readme_links(args)

49
tools/ci/checkout_project_ref.py
View File

@@ -3,14 +3,13 @@
# internal use only
# called by CI jobs when it uses a project related to IDF
import os
import json
import argparse
import subprocess
import json
import os
import re
import subprocess
IDF_GIT_DESCRIBE_PATTERN = re.compile(r"^v(\d)\.(\d)")
IDF_GIT_DESCRIBE_PATTERN = re.compile(r'^v(\d)\.(\d)')
RETRY_COUNT = 3
@@ -18,8 +17,8 @@ def get_customized_project_revision(proj_name):
"""
get customized project revision defined in bot message
"""
revision = ""
customized_project_revisions = os.getenv("BOT_CUSTOMIZED_REVISION")
revision = ''
customized_project_revisions = os.getenv('BOT_CUSTOMIZED_REVISION')
if customized_project_revisions:
customized_project_revisions = json.loads(customized_project_revisions)
try:
@@ -35,9 +34,9 @@ def target_branch_candidates(proj_name):
"""
candidates = [
# branch name (or tag name) of current IDF
os.getenv("CI_COMMIT_REF_NAME"),
os.getenv('CI_COMMIT_REF_NAME'),
# CI_MERGE_REQUEST_TARGET_BRANCH_NAME
os.getenv("CI_MERGE_REQUEST_TARGET_BRANCH_NAME"),
os.getenv('CI_MERGE_REQUEST_TARGET_BRANCH_NAME'),
]
customized_candidate = get_customized_project_revision(proj_name)
if customized_candidate:
@@ -46,16 +45,16 @@ def target_branch_candidates(proj_name):
# branch name read from IDF
try:
git_describe = subprocess.check_output(["git", "describe", "HEAD"])
git_describe = subprocess.check_output(['git', 'describe', 'HEAD'])
match = IDF_GIT_DESCRIBE_PATTERN.search(git_describe.decode())
if match:
major_revision = match.group(1)
minor_revision = match.group(2)
# release branch
candidates.append("release/v{}.{}".format(major_revision, minor_revision))
candidates.append('release/v{}.{}'.format(major_revision, minor_revision))
# branch to match all major branches, like v3.x or v3
candidates.append("release/v{}.x".format(major_revision))
candidates.append("release/v{}".format(major_revision))
candidates.append('release/v{}.x'.format(major_revision))
candidates.append('release/v{}'.format(major_revision))
except subprocess.CalledProcessError:
# this should not happen as IDF should have describe message
pass
@@ -63,14 +62,14 @@ def target_branch_candidates(proj_name):
return [c for c in candidates if c] # filter out null value
if __name__ == "__main__":
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument("project",
help="the name of project")
parser.add_argument("project_relative_path",
help="relative path of project to IDF repository directory")
parser.add_argument('project',
help='the name of project')
parser.add_argument('project_relative_path',
help='relative path of project to IDF repository directory')
parser.add_argument('--customized_only', action='store_true',
help="Only to find customized revision")
help='Only to find customized revision')
args = parser.parse_args()
@@ -83,10 +82,10 @@ if __name__ == "__main__":
# change to project dir for checkout
os.chdir(args.project_relative_path)
ref_to_use = ""
ref_to_use = ''
for candidate in candidate_branches:
# check if candidate branch exists
branch_match = subprocess.check_output(["git", "branch", "-a", "--list", "origin/" + candidate])
branch_match = subprocess.check_output(['git', 'branch', '-a', '--list', 'origin/' + candidate])
if branch_match:
ref_to_use = candidate
break
@@ -95,13 +94,13 @@ if __name__ == "__main__":
for _ in range(RETRY_COUNT):
# Add retry for projects with git-lfs
try:
subprocess.check_call(["git", "checkout", "-f", ref_to_use], stdout=subprocess.PIPE) # not print the stdout
print("CI using ref {} for project {}".format(ref_to_use, args.project))
subprocess.check_call(['git', 'checkout', '-f', ref_to_use], stdout=subprocess.PIPE) # not print the stdout
print('CI using ref {} for project {}'.format(ref_to_use, args.project))
break
except subprocess.CalledProcessError:
pass
else:
print("Failed to use ref {} for project {}".format(ref_to_use, args.project))
print('Failed to use ref {} for project {}'.format(ref_to_use, args.project))
exit(1)
else:
print("using default branch")
print('using default branch')

42
tools/ci/ci_fetch_submodule.py
View File

@@ -3,26 +3,26 @@
# internal use only for CI
# download archive of one commit instead of cloning entire submodule repo
import re
import os
import subprocess
import argparse
import os
import re
import shutil
import subprocess
import time
import gitlab_api
SUBMODULE_PATTERN = re.compile(r"\[submodule \"([^\"]+)\"]")
PATH_PATTERN = re.compile(r"path\s+=\s+(\S+)")
URL_PATTERN = re.compile(r"url\s+=\s+(\S+)")
PATH_PATTERN = re.compile(r'path\s+=\s+(\S+)')
URL_PATTERN = re.compile(r'url\s+=\s+(\S+)')
SUBMODULE_ARCHIVE_TEMP_FOLDER = "submodule_archive"
SUBMODULE_ARCHIVE_TEMP_FOLDER = 'submodule_archive'
class SubModule(object):
# We don't need to support recursive submodule clone now
GIT_LS_TREE_OUTPUT_PATTERN = re.compile(r"\d+\s+commit\s+([0-9a-f]+)\s+")
GIT_LS_TREE_OUTPUT_PATTERN = re.compile(r'\d+\s+commit\s+([0-9a-f]+)\s+')
def __init__(self, gitlab_inst, path, url):
self.path = path
@@ -31,7 +31,7 @@ class SubModule(object):
self.commit_id = self._get_commit_id(path)
def _get_commit_id(self, path):
output = subprocess.check_output(["git", "ls-tree", "HEAD", path])
output = subprocess.check_output(['git', 'ls-tree', 'HEAD', path])
output = output.decode()
# example output: 160000 commit d88a262fbdf35e5abb372280eb08008749c3faa0 components/esp_wifi/lib
match = self.GIT_LS_TREE_OUTPUT_PATTERN.search(output)
@@ -40,11 +40,11 @@ class SubModule(object):
def _get_project_id(self, url):
base_name = os.path.basename(url)
project_id = self.gitlab_inst.get_project_id(os.path.splitext(base_name)[0], # remove .git
namespace="espressif")
namespace='espressif')
return project_id
def download_archive(self):
print("Update submodule: {}: {}".format(self.path, self.commit_id))
print('Update submodule: {}: {}'.format(self.path, self.commit_id))
path_name = self.gitlab_inst.download_archive(self.commit_id, SUBMODULE_ARCHIVE_TEMP_FOLDER,
self.project_id)
renamed_path = os.path.join(os.path.dirname(path_name), os.path.basename(self.path))
@@ -56,7 +56,7 @@ class SubModule(object):
def update_submodule(git_module_file, submodules_to_update):
gitlab_inst = gitlab_api.Gitlab()
submodules = []
with open(git_module_file, "r") as f:
with open(git_module_file, 'r') as f:
data = f.read()
match = SUBMODULE_PATTERN.search(data)
while True:
@@ -90,18 +90,18 @@ def update_submodule(git_module_file, submodules_to_update):
if __name__ == '__main__':
start_time = time.time()
parser = argparse.ArgumentParser()
parser.add_argument("--repo_path", "-p", default=".", help="repo path")
parser.add_argument("--submodule", "-s", default="all",
help="Submodules to update. By default update all submodules. "
"For multiple submodules, separate them with `;`. "
"`all` and `none` are special values that indicates we fetch all / none submodules")
parser.add_argument('--repo_path', '-p', default='.', help='repo path')
parser.add_argument('--submodule', '-s', default='all',
help='Submodules to update. By default update all submodules. '
'For multiple submodules, separate them with `;`. '
'`all` and `none` are special values that indicates we fetch all / none submodules')
args = parser.parse_args()
if args.submodule == "none":
if args.submodule == 'none':
print("don't need to update submodules")
exit(0)
if args.submodule == "all":
if args.submodule == 'all':
_submodules = []
else:
_submodules = args.submodule.split(";")
update_submodule(os.path.join(args.repo_path, ".gitmodules"), _submodules)
print("total time spent on update submodule: {:.02f}s".format(time.time() - start_time))
_submodules = args.submodule.split(';')
update_submodule(os.path.join(args.repo_path, '.gitmodules'), _submodules)
print('total time spent on update submodule: {:.02f}s'.format(time.time() - start_time))

105
tools/ci/deploy_docs.py
View File

@@ -22,9 +22,10 @@ import os
import os.path
import re
import stat
import sys
import subprocess
import sys
import tarfile
import packaging.version
@@ -34,86 +35,86 @@ def env(variable, default=None):
# import sanitize_version from the docs directory, shared with here
sys.path.append(os.path.join(env("IDF_PATH"), "docs"))
sys.path.append(os.path.join(env('IDF_PATH'), 'docs'))
from sanitize_version import sanitize_version # noqa
def main():
# if you get KeyErrors on the following lines, it's probably because you're not running in Gitlab CI
git_ver = env("GIT_VER") # output of git describe --always
ci_ver = env("CI_COMMIT_REF_NAME", git_ver) # branch or tag we're building for (used for 'release' & URL)
git_ver = env('GIT_VER') # output of git describe --always
ci_ver = env('CI_COMMIT_REF_NAME', git_ver) # branch or tag we're building for (used for 'release' & URL)
version = sanitize_version(ci_ver)
print("Git version: {}".format(git_ver))
print("CI Version: {}".format(ci_ver))
print("Deployment version: {}".format(version))
print('Git version: {}'.format(git_ver))
print('CI Version: {}'.format(ci_ver))
print('Deployment version: {}'.format(version))
if not version:
raise RuntimeError("A version is needed to deploy")
raise RuntimeError('A version is needed to deploy')
build_dir = env("DOCS_BUILD_DIR") # top-level local build dir, where docs have already been built
build_dir = env('DOCS_BUILD_DIR') # top-level local build dir, where docs have already been built
if not build_dir:
raise RuntimeError("Valid DOCS_BUILD_DIR is needed to deploy")
raise RuntimeError('Valid DOCS_BUILD_DIR is needed to deploy')
url_base = env("DOCS_DEPLOY_URL_BASE") # base for HTTP URLs, used to print the URL to the log after deploying
url_base = env('DOCS_DEPLOY_URL_BASE') # base for HTTP URLs, used to print the URL to the log after deploying
docs_server = env("DOCS_DEPLOY_SERVER") # ssh server to deploy to
docs_user = env("DOCS_DEPLOY_SERVER_USER")
docs_path = env("DOCS_DEPLOY_PATH") # filesystem path on DOCS_SERVER
docs_server = env('DOCS_DEPLOY_SERVER') # ssh server to deploy to
docs_user = env('DOCS_DEPLOY_SERVER_USER')
docs_path = env('DOCS_DEPLOY_PATH') # filesystem path on DOCS_SERVER
if not docs_server:
raise RuntimeError("Valid DOCS_DEPLOY_SERVER is needed to deploy")
raise RuntimeError('Valid DOCS_DEPLOY_SERVER is needed to deploy')
if not docs_user:
raise RuntimeError("Valid DOCS_DEPLOY_SERVER_USER is needed to deploy")
raise RuntimeError('Valid DOCS_DEPLOY_SERVER_USER is needed to deploy')
docs_server = "{}@{}".format(docs_user, docs_server)
docs_server = '{}@{}'.format(docs_user, docs_server)
if not docs_path:
raise RuntimeError("Valid DOCS_DEPLOY_PATH is needed to deploy")
raise RuntimeError('Valid DOCS_DEPLOY_PATH is needed to deploy')
print("DOCS_DEPLOY_SERVER {} DOCS_DEPLOY_PATH {}".format(docs_server, docs_path))
print('DOCS_DEPLOY_SERVER {} DOCS_DEPLOY_PATH {}'.format(docs_server, docs_path))
tarball_path, version_urls = build_doc_tarball(version, git_ver, build_dir)
deploy(version, tarball_path, docs_path, docs_server)
print("Docs URLs:")
print('Docs URLs:')
doc_deploy_type = os.getenv('TYPE')
for vurl in version_urls:
language, _, target = vurl.split('/')
tag = '{}_{}'.format(language, target)
url = "{}/{}/index.html".format(url_base, vurl) # (index.html needed for the preview server)
url = re.sub(r"([^:])//", r"\1/", url) # get rid of any // that isn't in the https:// part
url = '{}/{}/index.html'.format(url_base, vurl) # (index.html needed for the preview server)
url = re.sub(r'([^:])//', r'\1/', url) # get rid of any // that isn't in the https:// part
print('[document {}][{}] {}'.format(doc_deploy_type, tag, url))
# note: it would be neater to use symlinks for stable, but because of the directory order
# (language first) it's kind of a pain to do on a remote server, so we just repeat the
# process but call the version 'stable' this time
if is_stable_version(version):
print("Deploying again as stable version...")
tarball_path, version_urls = build_doc_tarball("stable", git_ver, build_dir)
deploy("stable", tarball_path, docs_path, docs_server)
print('Deploying again as stable version...')
tarball_path, version_urls = build_doc_tarball('stable', git_ver, build_dir)
deploy('stable', tarball_path, docs_path, docs_server)
def deploy(version, tarball_path, docs_path, docs_server):
def run_ssh(commands):
""" Log into docs_server and run a sequence of commands using ssh """
print("Running ssh: {}".format(commands))
subprocess.run(["ssh", "-o", "BatchMode=yes", docs_server, "-x", " && ".join(commands)], check=True)
print('Running ssh: {}'.format(commands))
subprocess.run(['ssh', '-o', 'BatchMode=yes', docs_server, '-x', ' && '.join(commands)], check=True)
# copy the version tarball to the server
run_ssh(["mkdir -p {}".format(docs_path)])
print("Running scp {} to {}".format(tarball_path, "{}:{}".format(docs_server, docs_path)))
subprocess.run(["scp", "-B", tarball_path, "{}:{}".format(docs_server, docs_path)], check=True)
run_ssh(['mkdir -p {}'.format(docs_path)])
print('Running scp {} to {}'.format(tarball_path, '{}:{}'.format(docs_server, docs_path)))
subprocess.run(['scp', '-B', tarball_path, '{}:{}'.format(docs_server, docs_path)], check=True)
tarball_name = os.path.basename(tarball_path)
run_ssh(["cd {}".format(docs_path),
"rm -rf ./*/{}".format(version), # remove any pre-existing docs matching this version
"tar -zxvf {}".format(tarball_name), # untar the archive with the new docs
"rm {}".format(tarball_name)])
run_ssh(['cd {}'.format(docs_path),
'rm -rf ./*/{}'.format(version), # remove any pre-existing docs matching this version
'tar -zxvf {}'.format(tarball_name), # untar the archive with the new docs
'rm {}'.format(tarball_name)])
# Note: deleting and then extracting the archive is a bit awkward for updating stable/latest/etc
# as the version will be invalid for a window of time. Better to do it atomically, but this is
@@ -124,21 +125,21 @@ def build_doc_tarball(version, git_ver, build_dir):
""" Make a tar.gz archive of the docs, in the directory structure used to deploy as
the given version """
version_paths = []
tarball_path = "{}/{}.tar.gz".format(build_dir, version)
tarball_path = '{}/{}.tar.gz'.format(build_dir, version)
# find all the 'html/' directories under build_dir
html_dirs = glob.glob("{}/**/html/".format(build_dir), recursive=True)
print("Found %d html directories" % len(html_dirs))
html_dirs = glob.glob('{}/**/html/'.format(build_dir), recursive=True)
print('Found %d html directories' % len(html_dirs))
pdfs = glob.glob("{}/**/latex/build/*.pdf".format(build_dir), recursive=True)
print("Found %d PDFs in latex directories" % len(pdfs))
pdfs = glob.glob('{}/**/latex/build/*.pdf'.format(build_dir), recursive=True)
print('Found %d PDFs in latex directories' % len(pdfs))
# add symlink for stable and latest and adds them to PDF blob
symlinks = create_and_add_symlinks(version, git_ver, pdfs)
def not_sources_dir(ti):
""" Filter the _sources directories out of the tarballs """
if ti.name.endswith("/_sources"):
if ti.name.endswith('/_sources'):
return None
ti.mode |= stat.S_IWGRP # make everything group-writeable
@@ -149,7 +150,7 @@ def build_doc_tarball(version, git_ver, build_dir):
except OSError:
pass
with tarfile.open(tarball_path, "w:gz") as tarball:
with tarfile.open(tarball_path, 'w:gz') as tarball:
for html_dir in html_dirs:
# html_dir has the form '<ignored>/<language>/<target>/html/'
target_dirname = os.path.dirname(os.path.dirname(html_dir))
@@ -157,7 +158,7 @@ def build_doc_tarball(version, git_ver, build_dir):
language = os.path.basename(os.path.dirname(target_dirname))
# when deploying, we want the top-level directory layout 'language/version/target'
archive_path = "{}/{}/{}".format(language, version, target)
archive_path = '{}/{}/{}'.format(language, version, target)
print("Archiving '{}' as '{}'...".format(html_dir, archive_path))
tarball.add(html_dir, archive_path, filter=not_sources_dir)
version_paths.append(archive_path)
@@ -171,7 +172,7 @@ def build_doc_tarball(version, git_ver, build_dir):
language = os.path.basename(os.path.dirname(target_dirname))
# when deploying, we want the layout 'language/version/target/pdf'
archive_path = "{}/{}/{}/{}".format(language, version, target, pdf_filename)
archive_path = '{}/{}/{}/{}'.format(language, version, target, pdf_filename)
print("Archiving '{}' as '{}'...".format(pdf_path, archive_path))
tarball.add(pdf_path, archive_path)
@@ -192,34 +193,34 @@ def create_and_add_symlinks(version, git_ver, pdfs):
symlinks.append(symlink_path)
pdfs.extend(symlinks)
print("Found %d PDFs in latex directories after adding symlink" % len(pdfs))
print('Found %d PDFs in latex directories after adding symlink' % len(pdfs))
return symlinks
def is_stable_version(version):
""" Heuristic for whether this is the latest stable release """
if not version.startswith("v"):
if not version.startswith('v'):
return False # branch name
if "-" in version:
if '-' in version:
return False # prerelease tag
git_out = subprocess.check_output(["git", "tag", "-l"]).decode("utf-8")
git_out = subprocess.check_output(['git', 'tag', '-l']).decode('utf-8')
versions = [v.strip() for v in git_out.split("\n")]
versions = [v for v in versions if re.match(r"^v[\d\.]+$", v)] # include vX.Y.Z only
versions = [v.strip() for v in git_out.split('\n')]
versions = [v for v in versions if re.match(r'^v[\d\.]+$', v)] # include vX.Y.Z only
versions = [packaging.version.parse(v) for v in versions]
max_version = max(versions)
if max_version.public != version[1:]:
print("Stable version is v{}. This version is {}.".format(max_version.public, version))
print('Stable version is v{}. This version is {}.'.format(max_version.public, version))
return False
else:
print("This version {} is the stable version".format(version))
print('This version {} is the stable version'.format(version))
return True
if __name__ == "__main__":
if __name__ == '__main__':
main()

2
tools/ci/normalize_clangtidy_path.py
View File

@@ -1,7 +1,7 @@
#!/usr/bin/env python
import argparse
import re
from os.path import join, normpath, dirname, relpath
from os.path import dirname, join, normpath, relpath
CLANG_TIDY_REGEX = re.compile(r'(.+|[a-zA-Z]:\\\\.+):([0-9]+):([0-9]+): ([^:]+): (.+)')

62
tools/ci/python_packages/gitlab_api.py
View File

@@ -1,8 +1,8 @@
import argparse
import os
import re
import argparse
import tempfile
import tarfile
import tempfile
import zipfile
from functools import wraps
@@ -10,21 +10,21 @@ import gitlab
class Gitlab(object):
JOB_NAME_PATTERN = re.compile(r"(\w+)(\s+(\d+)/(\d+))?")
JOB_NAME_PATTERN = re.compile(r'(\w+)(\s+(\d+)/(\d+))?')
DOWNLOAD_ERROR_MAX_RETRIES = 3
def __init__(self, project_id=None):
config_data_from_env = os.getenv("PYTHON_GITLAB_CONFIG")
config_data_from_env = os.getenv('PYTHON_GITLAB_CONFIG')
if config_data_from_env:
# prefer to load config from env variable
with tempfile.NamedTemporaryFile("w", delete=False) as temp_file:
with tempfile.NamedTemporaryFile('w', delete=False) as temp_file:
temp_file.write(config_data_from_env)
config_files = [temp_file.name]
else:
# otherwise try to use config file at local filesystem
config_files = None
gitlab_id = os.getenv("LOCAL_GITLAB_HTTPS_HOST") # if None, will use the default gitlab server
gitlab_id = os.getenv('LOCAL_GITLAB_HTTPS_HOST') # if None, will use the default gitlab server
self.gitlab_inst = gitlab.Gitlab.from_config(gitlab_id=gitlab_id, config_files=config_files)
self.gitlab_inst.auth()
if project_id:
@@ -46,7 +46,7 @@ class Gitlab(object):
if len(projects) == 1:
project_id = project.id
break
if project.namespace["path"] == namespace:
if project.namespace['path'] == namespace:
project_id = project.id
break
else:
@@ -65,7 +65,7 @@ class Gitlab(object):
with tempfile.NamedTemporaryFile(delete=False) as temp_file:
job.artifacts(streamed=True, action=temp_file.write)
with zipfile.ZipFile(temp_file.name, "r") as archive_file:
with zipfile.ZipFile(temp_file.name, 'r') as archive_file:
archive_file.extractall(destination)
def retry_download(func):
@@ -120,12 +120,12 @@ class Gitlab(object):
except OSError:
# already exists
pass
with open(file_path, "wb") as f:
with open(file_path, 'wb') as f:
f.write(data)
return raw_data_list
def find_job_id(self, job_name, pipeline_id=None, job_status="success"):
def find_job_id(self, job_name, pipeline_id=None, job_status='success'):
"""
Get Job ID from job name of specific pipeline
@@ -137,14 +137,14 @@ class Gitlab(object):
"""
job_id_list = []
if pipeline_id is None:
pipeline_id = os.getenv("CI_PIPELINE_ID")
pipeline_id = os.getenv('CI_PIPELINE_ID')
pipeline = self.project.pipelines.get(pipeline_id)
jobs = pipeline.jobs.list(all=True)
for job in jobs:
match = self.JOB_NAME_PATTERN.match(job.name)
if match:
if match.group(1) == job_name and job.status == job_status:
job_id_list.append({"id": job.id, "parallel_num": match.group(3)})
job_id_list.append({'id': job.id, 'parallel_num': match.group(3)})
return job_id_list
@retry_download
@@ -166,12 +166,12 @@ class Gitlab(object):
try:
project.repository_archive(sha=ref, streamed=True, action=temp_file.write)
except gitlab.GitlabGetError as e:
print("Failed to archive from project {}".format(project_id))
print('Failed to archive from project {}'.format(project_id))
raise e
print("archive size: {:.03f}MB".format(float(os.path.getsize(temp_file.name)) / (1024 * 1024)))
print('archive size: {:.03f}MB'.format(float(os.path.getsize(temp_file.name)) / (1024 * 1024)))
with tarfile.open(temp_file.name, "r") as archive_file:
with tarfile.open(temp_file.name, 'r') as archive_file:
root_name = archive_file.getnames()[0]
archive_file.extractall(destination)
@@ -180,27 +180,27 @@ class Gitlab(object):
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument("action")
parser.add_argument("project_id", type=int)
parser.add_argument("--pipeline_id", "-i", type=int, default=None)
parser.add_argument("--ref", "-r", default="master")
parser.add_argument("--job_id", "-j", type=int, default=None)
parser.add_argument("--job_name", "-n", default=None)
parser.add_argument("--project_name", "-m", default=None)
parser.add_argument("--destination", "-d", default=None)
parser.add_argument("--artifact_path", "-a", nargs="*", default=None)
parser.add_argument('action')
parser.add_argument('project_id', type=int)
parser.add_argument('--pipeline_id', '-i', type=int, default=None)
parser.add_argument('--ref', '-r', default='master')
parser.add_argument('--job_id', '-j', type=int, default=None)
parser.add_argument('--job_name', '-n', default=None)
parser.add_argument('--project_name', '-m', default=None)
parser.add_argument('--destination', '-d', default=None)
parser.add_argument('--artifact_path', '-a', nargs='*', default=None)
args = parser.parse_args()
gitlab_inst = Gitlab(args.project_id)
if args.action == "download_artifacts":
if args.action == 'download_artifacts':
gitlab_inst.download_artifacts(args.job_id, args.destination)
if args.action == "download_artifact":
if args.action == 'download_artifact':
gitlab_inst.download_artifact(args.job_id, args.artifact_path, args.destination)
elif args.action == "find_job_id":
elif args.action == 'find_job_id':
job_ids = gitlab_inst.find_job_id(args.job_name, args.pipeline_id)
print(";".join([",".join([str(j["id"]), j["parallel_num"]]) for j in job_ids]))
elif args.action == "download_archive":
print(';'.join([','.join([str(j['id']), j['parallel_num']]) for j in job_ids]))
elif args.action == 'download_archive':
gitlab_inst.download_archive(args.ref, args.destination)
elif args.action == "get_project_id":
elif args.action == 'get_project_id':
ret = gitlab_inst.get_project_id(args.project_name)
print("project id: {}".format(ret))
print('project id: {}'.format(ret))

57
tools/ci/python_packages/idf_http_server_test/adder.py
View File

@@ -14,12 +14,11 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import print_function
from __future__ import unicode_literals
from builtins import str
from builtins import range
import http.client
from __future__ import print_function, unicode_literals
import argparse
import http.client
from builtins import range, str
from tiny_test_fw import Utility
@@ -33,41 +32,41 @@ def end_session(conn):
def getreq(conn, path, verbose=False):
conn.request("GET", path)
conn.request('GET', path)
resp = conn.getresponse()
data = resp.read()
if verbose:
Utility.console_log("GET : " + path)
Utility.console_log("Status : " + resp.status)
Utility.console_log("Reason : " + resp.reason)
Utility.console_log("Data length : " + str(len(data)))
Utility.console_log("Data content : " + data)
Utility.console_log('GET : ' + path)
Utility.console_log('Status : ' + resp.status)
Utility.console_log('Reason : ' + resp.reason)
Utility.console_log('Data length : ' + str(len(data)))
Utility.console_log('Data content : ' + data)
return data
def postreq(conn, path, data, verbose=False):
conn.request("POST", path, data)
conn.request('POST', path, data)
resp = conn.getresponse()
data = resp.read()
if verbose:
Utility.console_log("POST : " + data)
Utility.console_log("Status : " + resp.status)
Utility.console_log("Reason : " + resp.reason)
Utility.console_log("Data length : " + str(len(data)))
Utility.console_log("Data content : " + data)
Utility.console_log('POST : ' + data)
Utility.console_log('Status : ' + resp.status)
Utility.console_log('Reason : ' + resp.reason)
Utility.console_log('Data length : ' + str(len(data)))
Utility.console_log('Data content : ' + data)
return data
def putreq(conn, path, body, verbose=False):
conn.request("PUT", path, body)
conn.request('PUT', path, body)
resp = conn.getresponse()
data = resp.read()
if verbose:
Utility.console_log("PUT : " + path, body)
Utility.console_log("Status : " + resp.status)
Utility.console_log("Reason : " + resp.reason)
Utility.console_log("Data length : " + str(len(data)))
Utility.console_log("Data content : " + data)
Utility.console_log('PUT : ' + path, body)
Utility.console_log('Status : ' + resp.status)
Utility.console_log('Reason : ' + resp.reason)
Utility.console_log('Data length : ' + str(len(data)))
Utility.console_log('Data content : ' + data)
return data
@@ -85,22 +84,22 @@ if __name__ == '__main__':
N = args['N']
# Establish HTTP connection
Utility.console_log("Connecting to => " + ip + ":" + port)
Utility.console_log('Connecting to => ' + ip + ':' + port)
conn = start_session(ip, port)
# Reset adder context to specified value(0)
# -- Not needed as new connection will always
# -- have zero value of the accumulator
Utility.console_log("Reset the accumulator to 0")
putreq(conn, "/adder", str(0))
Utility.console_log('Reset the accumulator to 0')
putreq(conn, '/adder', str(0))
# Sum numbers from 1 to specified value(N)
Utility.console_log("Summing numbers from 1 to " + str(N))
Utility.console_log('Summing numbers from 1 to ' + str(N))
for i in range(1, N + 1):
postreq(conn, "/adder", str(i))
postreq(conn, '/adder', str(i))
# Fetch the result
Utility.console_log("Result :" + getreq(conn, "/adder"))
Utility.console_log('Result :' + getreq(conn, '/adder'))
# Close HTTP connection
end_session(conn)

153
tools/ci/python_packages/idf_http_server_test/client.py
View File

@@ -14,12 +14,11 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import print_function
from __future__ import unicode_literals
from builtins import str
import http.client
import argparse
from __future__ import print_function, unicode_literals
import argparse
import http.client
from builtins import str
from tiny_test_fw import Utility
@@ -31,28 +30,28 @@ def verbose_print(verbosity, *args):
def test_val(text, expected, received):
if expected != received:
Utility.console_log(" Fail!")
Utility.console_log(" [reason] " + text + ":")
Utility.console_log(" expected: " + str(expected))
Utility.console_log(" received: " + str(received))
Utility.console_log(' Fail!')
Utility.console_log(' [reason] ' + text + ':')
Utility.console_log(' expected: ' + str(expected))
Utility.console_log(' received: ' + str(received))
return False
return True
def test_get_handler(ip, port, verbosity=False):
verbose_print(verbosity, "======== GET HANDLER TEST =============")
verbose_print(verbosity, '======== GET HANDLER TEST =============')
# Establish HTTP connection
verbose_print(verbosity, "Connecting to => " + ip + ":" + port)
sess = http.client.HTTPConnection(ip + ":" + port, timeout=15)
verbose_print(verbosity, 'Connecting to => ' + ip + ':' + port)
sess = http.client.HTTPConnection(ip + ':' + port, timeout=15)
uri = "/hello?query1=value1&query2=value2&query3=value3"
uri = '/hello?query1=value1&query2=value2&query3=value3'
# GET hello response
test_headers = {"Test-Header-1":"Test-Value-1", "Test-Header-2":"Test-Value-2"}
verbose_print(verbosity, "Sending GET to URI : ", uri)
verbose_print(verbosity, "Sending additional headers : ")
test_headers = {'Test-Header-1':'Test-Value-1', 'Test-Header-2':'Test-Value-2'}
verbose_print(verbosity, 'Sending GET to URI : ', uri)
verbose_print(verbosity, 'Sending additional headers : ')
for k, v in test_headers.items():
verbose_print(verbosity, "\t", k, ": ", v)
sess.request("GET", url=uri, headers=test_headers)
verbose_print(verbosity, '\t', k, ': ', v)
sess.request('GET', url=uri, headers=test_headers)
resp = sess.getresponse()
resp_hdrs = resp.getheaders()
resp_data = resp.read().decode()
@@ -60,100 +59,100 @@ def test_get_handler(ip, port, verbosity=False):
sess.close()
if not (
test_val("Status code mismatch", 200, resp.status) and
test_val("Response mismatch", "Custom-Value-1", resp.getheader("Custom-Header-1")) and
test_val("Response mismatch", "Custom-Value-2", resp.getheader("Custom-Header-2")) and
test_val("Response mismatch", "Hello World!", resp_data)
test_val('Status code mismatch', 200, resp.status) and
test_val('Response mismatch', 'Custom-Value-1', resp.getheader('Custom-Header-1')) and
test_val('Response mismatch', 'Custom-Value-2', resp.getheader('Custom-Header-2')) and
test_val('Response mismatch', 'Hello World!', resp_data)
):
return False
verbose_print(verbosity, "vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv")
verbose_print(verbosity, "Server response to GET /hello")
verbose_print(verbosity, "Response Headers : ")
verbose_print(verbosity, 'vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv')
verbose_print(verbosity, 'Server response to GET /hello')
verbose_print(verbosity, 'Response Headers : ')
for k, v in resp_hdrs:
verbose_print(verbosity, "\t", k, ": ", v)
verbose_print(verbosity, "Response Data : " + resp_data)
verbose_print(verbosity, "========================================\n")
verbose_print(verbosity, '\t', k, ': ', v)
verbose_print(verbosity, 'Response Data : ' + resp_data)
verbose_print(verbosity, '========================================\n')
return True
def test_post_handler(ip, port, msg, verbosity=False):
verbose_print(verbosity, "======== POST HANDLER TEST ============")
verbose_print(verbosity, '======== POST HANDLER TEST ============')
# Establish HTTP connection
verbose_print(verbosity, "Connecting to => " + ip + ":" + port)
sess = http.client.HTTPConnection(ip + ":" + port, timeout=15)
verbose_print(verbosity, 'Connecting to => ' + ip + ':' + port)
sess = http.client.HTTPConnection(ip + ':' + port, timeout=15)
# POST message to /echo and get back response
sess.request("POST", url="/echo", body=msg)
sess.request('POST', url='/echo', body=msg)
resp = sess.getresponse()
resp_data = resp.read().decode()
verbose_print(verbosity, "Server response to POST /echo (" + msg + ")")
verbose_print(verbosity, "vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv")
verbose_print(verbosity, 'Server response to POST /echo (' + msg + ')')
verbose_print(verbosity, 'vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv')
verbose_print(verbosity, resp_data)
verbose_print(verbosity, "========================================\n")
verbose_print(verbosity, '========================================\n')
# Close HTTP connection
sess.close()
return test_val("Response mismatch", msg, resp_data)
return test_val('Response mismatch', msg, resp_data)
def test_put_handler(ip, port, verbosity=False):
verbose_print(verbosity, "======== PUT HANDLER TEST =============")
verbose_print(verbosity, '======== PUT HANDLER TEST =============')
# Establish HTTP connection
verbose_print(verbosity, "Connecting to => " + ip + ":" + port)
sess = http.client.HTTPConnection(ip + ":" + port, timeout=15)
verbose_print(verbosity, 'Connecting to => ' + ip + ':' + port)
sess = http.client.HTTPConnection(ip + ':' + port, timeout=15)
# PUT message to /ctrl to disable /hello and /echo URI handlers
# and set 404 error handler to custom http_404_error_handler()
verbose_print(verbosity, "Disabling /hello and /echo handlers")
sess.request("PUT", url="/ctrl", body="0")
verbose_print(verbosity, 'Disabling /hello and /echo handlers')
sess.request('PUT', url='/ctrl', body='0')
resp = sess.getresponse()
resp.read()
try:
# Send HTTP request to /hello URI
sess.request("GET", url="/hello")
sess.request('GET', url='/hello')
resp = sess.getresponse()
resp_data = resp.read().decode()
# 404 Error must be returned from server as URI /hello is no longer available.
# But the custom error handler http_404_error_handler() will not close the
# session if the requested URI is /hello
if not test_val("Status code mismatch", 404, resp.status):
if not test_val('Status code mismatch', 404, resp.status):
raise AssertionError
# Compare error response string with expectation
verbose_print(verbosity, "Response on GET /hello : " + resp_data)
if not test_val("Response mismatch", "/hello URI is not available", resp_data):
verbose_print(verbosity, 'Response on GET /hello : ' + resp_data)
if not test_val('Response mismatch', '/hello URI is not available', resp_data):
raise AssertionError
# Using same session for sending an HTTP request to /echo, as it is expected
# that the custom error handler http_404_error_handler() would not have closed
# the session
sess.request("POST", url="/echo", body="Some content")
sess.request('POST', url='/echo', body='Some content')
resp = sess.getresponse()
resp_data = resp.read().decode()
# 404 Error must be returned from server as URI /hello is no longer available.
# The custom error handler http_404_error_handler() will close the session
# this time as the requested URI is /echo
if not test_val("Status code mismatch", 404, resp.status):
if not test_val('Status code mismatch', 404, resp.status):
raise AssertionError
# Compare error response string with expectation
verbose_print(verbosity, "Response on POST /echo : " + resp_data)
if not test_val("Response mismatch", "/echo URI is not available", resp_data):
verbose_print(verbosity, 'Response on POST /echo : ' + resp_data)
if not test_val('Response mismatch', '/echo URI is not available', resp_data):
raise AssertionError
try:
# Using same session should fail as by now the session would have closed
sess.request("POST", url="/hello", body="Some content")
sess.request('POST', url='/hello', body='Some content')
resp = sess.getresponse()
resp.read().decode()
# If control reaches this point then the socket was not closed.
# This is not expected
verbose_print(verbosity, "Socket not closed by server")
verbose_print(verbosity, 'Socket not closed by server')
raise AssertionError
except http.client.HTTPException:
@@ -161,7 +160,7 @@ def test_put_handler(ip, port, verbosity=False):
pass
except http.client.HTTPException:
verbose_print(verbosity, "Socket closed by server")
verbose_print(verbosity, 'Socket closed by server')
return False
except AssertionError:
@@ -171,47 +170,47 @@ def test_put_handler(ip, port, verbosity=False):
# Close HTTP connection
sess.close()
verbose_print(verbosity, "Enabling /hello handler")
verbose_print(verbosity, 'Enabling /hello handler')
# Create new connection
sess = http.client.HTTPConnection(ip + ":" + port, timeout=15)
sess = http.client.HTTPConnection(ip + ':' + port, timeout=15)
# PUT message to /ctrl to enable /hello URI handler
# and restore 404 error handler to default
sess.request("PUT", url="/ctrl", body="1")
sess.request('PUT', url='/ctrl', body='1')
resp = sess.getresponse()
resp.read()
# Close HTTP connection
sess.close()
# Create new connection
sess = http.client.HTTPConnection(ip + ":" + port, timeout=15)
sess = http.client.HTTPConnection(ip + ':' + port, timeout=15)
try:
# Sending HTTP request to /hello should work now
sess.request("GET", url="/hello")
sess.request('GET', url='/hello')
resp = sess.getresponse()
resp_data = resp.read().decode()
if not test_val("Status code mismatch", 200, resp.status):
if not test_val('Status code mismatch', 200, resp.status):
raise AssertionError
verbose_print(verbosity, "Response on GET /hello : " + resp_data)
if not test_val("Response mismatch", "Hello World!", resp_data):
verbose_print(verbosity, 'Response on GET /hello : ' + resp_data)
if not test_val('Response mismatch', 'Hello World!', resp_data):
raise AssertionError
# 404 Error handler should have been restored to default
sess.request("GET", url="/invalid")
sess.request('GET', url='/invalid')
resp = sess.getresponse()
resp_data = resp.read().decode()
if not test_val("Status code mismatch", 404, resp.status):
if not test_val('Status code mismatch', 404, resp.status):
raise AssertionError
verbose_print(verbosity, "Response on GET /invalid : " + resp_data)
if not test_val("Response mismatch", "This URI does not exist", resp_data):
verbose_print(verbosity, 'Response on GET /invalid : ' + resp_data)
if not test_val('Response mismatch', 'This URI does not exist', resp_data):
raise AssertionError
except http.client.HTTPException:
verbose_print(verbosity, "Socket closed by server")
verbose_print(verbosity, 'Socket closed by server')
return False
except AssertionError:
@@ -225,26 +224,26 @@ def test_put_handler(ip, port, verbosity=False):
def test_custom_uri_query(ip, port, query, verbosity=False):
verbose_print(verbosity, "======== GET HANDLER TEST =============")
verbose_print(verbosity, '======== GET HANDLER TEST =============')
# Establish HTTP connection
verbose_print(verbosity, "Connecting to => " + ip + ":" + port)
sess = http.client.HTTPConnection(ip + ":" + port, timeout=15)
verbose_print(verbosity, 'Connecting to => ' + ip + ':' + port)
sess = http.client.HTTPConnection(ip + ':' + port, timeout=15)
uri = "/hello?" + query
uri = '/hello?' + query
# GET hello response
verbose_print(verbosity, "Sending GET to URI : ", uri)
sess.request("GET", url=uri, headers={})
verbose_print(verbosity, 'Sending GET to URI : ', uri)
sess.request('GET', url=uri, headers={})
resp = sess.getresponse()
resp_data = resp.read().decode()
verbose_print(verbosity, "vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv")
verbose_print(verbosity, "Server response to GET /hello")
verbose_print(verbosity, "Response Data : " + resp_data)
verbose_print(verbosity, "========================================\n")
verbose_print(verbosity, 'vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv')
verbose_print(verbosity, 'Server response to GET /hello')
verbose_print(verbosity, 'Response Data : ' + resp_data)
verbose_print(verbosity, '========================================\n')
# Close HTTP connection
sess.close()
return "Hello World!" == resp_data
return 'Hello World!' == resp_data
if __name__ == '__main__':
@@ -265,4 +264,4 @@ if __name__ == '__main__':
test_put_handler(ip, port, True) and
test_post_handler(ip, port, msg, True)
):
Utility.console_log("Failed!")
Utility.console_log('Failed!')

478
tools/ci/python_packages/idf_http_server_test/test.py
View File

@@ -129,19 +129,17 @@
# - Simple GET on /hello/restart_results (returns the leak results)
from __future__ import division
from __future__ import print_function
from builtins import str
from builtins import range
from builtins import object
import threading
import socket
import time
from __future__ import division, print_function
import argparse
import http.client
import sys
import string
import random
import socket
import string
import sys
import threading
import time
from builtins import object, range, str
from tiny_test_fw import Utility
@@ -165,32 +163,32 @@ class Session(object):
self.client.sendall(data.encode())
except socket.error as err:
self.client.close()
Utility.console_log("Socket Error in send :", err)
Utility.console_log('Socket Error in send :', err)
rval = False
return rval
def send_get(self, path, headers=None):
request = "GET " + path + " HTTP/1.1\r\nHost: " + self.target
request = 'GET ' + path + ' HTTP/1.1\r\nHost: ' + self.target
if headers:
for field, value in headers.items():
request += "\r\n" + field + ": " + value
request += "\r\n\r\n"
request += '\r\n' + field + ': ' + value
request += '\r\n\r\n'
return self.send_err_check(request)
def send_put(self, path, data, headers=None):
request = "PUT " + path + " HTTP/1.1\r\nHost: " + self.target
request = 'PUT ' + path + ' HTTP/1.1\r\nHost: ' + self.target
if headers:
for field, value in headers.items():
request += "\r\n" + field + ": " + value
request += "\r\nContent-Length: " + str(len(data)) + "\r\n\r\n"
request += '\r\n' + field + ': ' + value
request += '\r\nContent-Length: ' + str(len(data)) + '\r\n\r\n'
return self.send_err_check(request, data)
def send_post(self, path, data, headers=None):
request = "POST " + path + " HTTP/1.1\r\nHost: " + self.target
request = 'POST ' + path + ' HTTP/1.1\r\nHost: ' + self.target
if headers:
for field, value in headers.items():
request += "\r\n" + field + ": " + value
request += "\r\nContent-Length: " + str(len(data)) + "\r\n\r\n"
request += '\r\n' + field + ': ' + value
request += '\r\nContent-Length: ' + str(len(data)) + '\r\n\r\n'
return self.send_err_check(request, data)
def read_resp_hdrs(self):
@@ -234,7 +232,7 @@ class Session(object):
return headers
except socket.error as err:
self.client.close()
Utility.console_log("Socket Error in recv :", err)
Utility.console_log('Socket Error in recv :', err)
return None
def read_resp_data(self):
@@ -263,9 +261,9 @@ class Session(object):
rem_len -= len(new_data)
chunk_data_buf = ''
# Fetch remaining CRLF
if self.client.recv(2) != "\r\n":
if self.client.recv(2) != '\r\n':
# Error in packet
Utility.console_log("Error in chunked data")
Utility.console_log('Error in chunked data')
return None
if not chunk_len:
# If last chunk
@@ -278,7 +276,7 @@ class Session(object):
return read_data
except socket.error as err:
self.client.close()
Utility.console_log("Socket Error in recv :", err)
Utility.console_log('Socket Error in recv :', err)
return None
def close(self):
@@ -287,10 +285,10 @@ class Session(object):
def test_val(text, expected, received):
if expected != received:
Utility.console_log(" Fail!")
Utility.console_log(" [reason] " + text + ":")
Utility.console_log(" expected: " + str(expected))
Utility.console_log(" received: " + str(received))
Utility.console_log(' Fail!')
Utility.console_log(' [reason] ' + text + ':')
Utility.console_log(' expected: ' + str(expected))
Utility.console_log(' received: ' + str(received))
return False
return True
@@ -308,7 +306,7 @@ class adder_thread (threading.Thread):
# Pipeline 3 requests
if (_verbose_):
Utility.console_log(" Thread: Using adder start " + str(self.id))
Utility.console_log(' Thread: Using adder start ' + str(self.id))
for _ in range(self.depth):
self.session.send_post('/adder', str(self.id))
@@ -320,10 +318,10 @@ class adder_thread (threading.Thread):
def adder_result(self):
if len(self.response) != self.depth:
Utility.console_log("Error : missing response packets")
Utility.console_log('Error : missing response packets')
return False
for i in range(len(self.response)):
if not test_val("Thread" + str(self.id) + " response[" + str(i) + "]",
if not test_val('Thread' + str(self.id) + ' response[' + str(i) + ']',
str(self.id * (i + 1)), str(self.response[i])):
return False
return True
@@ -336,177 +334,177 @@ def get_hello(dut, port):
# GET /hello should return 'Hello World!'
Utility.console_log("[test] GET /hello returns 'Hello World!' =>", end=' ')
conn = http.client.HTTPConnection(dut, int(port), timeout=15)
conn.request("GET", "/hello")
conn.request('GET', '/hello')
resp = conn.getresponse()
if not test_val("status_code", 200, resp.status):
if not test_val('status_code', 200, resp.status):
conn.close()
return False
if not test_val("data", "Hello World!", resp.read().decode()):
if not test_val('data', 'Hello World!', resp.read().decode()):
conn.close()
return False
if not test_val("data", "text/html", resp.getheader('Content-Type')):
if not test_val('data', 'text/html', resp.getheader('Content-Type')):
conn.close()
return False
Utility.console_log("Success")
Utility.console_log('Success')
conn.close()
return True
def put_hello(dut, port):
# PUT /hello returns 405'
Utility.console_log("[test] PUT /hello returns 405 =>", end=' ')
Utility.console_log('[test] PUT /hello returns 405 =>', end=' ')
conn = http.client.HTTPConnection(dut, int(port), timeout=15)
conn.request("PUT", "/hello", "Hello")
conn.request('PUT', '/hello', 'Hello')
resp = conn.getresponse()
if not test_val("status_code", 405, resp.status):
if not test_val('status_code', 405, resp.status):
conn.close()
return False
Utility.console_log("Success")
Utility.console_log('Success')
conn.close()
return True
def post_hello(dut, port):
# POST /hello returns 405'
Utility.console_log("[test] POST /hello returns 405 =>", end=' ')
Utility.console_log('[test] POST /hello returns 405 =>', end=' ')
conn = http.client.HTTPConnection(dut, int(port), timeout=15)
conn.request("POST", "/hello", "Hello")
conn.request('POST', '/hello', 'Hello')
resp = conn.getresponse()
if not test_val("status_code", 405, resp.status):
if not test_val('status_code', 405, resp.status):
conn.close()
return False
Utility.console_log("Success")
Utility.console_log('Success')
conn.close()
return True
def post_echo(dut, port):
# POST /echo echoes data'
Utility.console_log("[test] POST /echo echoes data =>", end=' ')
Utility.console_log('[test] POST /echo echoes data =>', end=' ')
conn = http.client.HTTPConnection(dut, int(port), timeout=15)
conn.request("POST", "/echo", "Hello")
conn.request('POST', '/echo', 'Hello')
resp = conn.getresponse()
if not test_val("status_code", 200, resp.status):
if not test_val('status_code', 200, resp.status):
conn.close()
return False
if not test_val("data", "Hello", resp.read().decode()):
if not test_val('data', 'Hello', resp.read().decode()):
conn.close()
return False
Utility.console_log("Success")
Utility.console_log('Success')
conn.close()
return True
def put_echo(dut, port):
# PUT /echo echoes data'
Utility.console_log("[test] PUT /echo echoes data =>", end=' ')
Utility.console_log('[test] PUT /echo echoes data =>', end=' ')
conn = http.client.HTTPConnection(dut, int(port), timeout=15)
conn.request("PUT", "/echo", "Hello")
conn.request('PUT', '/echo', 'Hello')
resp = conn.getresponse()
if not test_val("status_code", 200, resp.status):
if not test_val('status_code', 200, resp.status):
conn.close()
return False
if not test_val("data", "Hello", resp.read().decode()):
if not test_val('data', 'Hello', resp.read().decode()):
conn.close()
return False
Utility.console_log("Success")
Utility.console_log('Success')
conn.close()
return True
def get_echo(dut, port):
# GET /echo returns 404'
Utility.console_log("[test] GET /echo returns 405 =>", end=' ')
Utility.console_log('[test] GET /echo returns 405 =>', end=' ')
conn = http.client.HTTPConnection(dut, int(port), timeout=15)
conn.request("GET", "/echo")
conn.request('GET', '/echo')
resp = conn.getresponse()
if not test_val("status_code", 405, resp.status):
if not test_val('status_code', 405, resp.status):
conn.close()
return False
Utility.console_log("Success")
Utility.console_log('Success')
conn.close()
return True
def get_test_headers(dut, port):
# GET /test_header returns data of Header2'
Utility.console_log("[test] GET /test_header =>", end=' ')
Utility.console_log('[test] GET /test_header =>', end=' ')
conn = http.client.HTTPConnection(dut, int(port), timeout=15)
custom_header = {"Header1": "Value1", "Header3": "Value3"}
header2_values = ["", " ", "Value2", " Value2", "Value2 ", " Value2 "]
custom_header = {'Header1': 'Value1', 'Header3': 'Value3'}
header2_values = ['', ' ', 'Value2', ' Value2', 'Value2 ', ' Value2 ']
for val in header2_values:
custom_header["Header2"] = val
conn.request("GET", "/test_header", headers=custom_header)
custom_header['Header2'] = val
conn.request('GET', '/test_header', headers=custom_header)
resp = conn.getresponse()
if not test_val("status_code", 200, resp.status):
if not test_val('status_code', 200, resp.status):
conn.close()
return False
hdr_val_start_idx = val.find("Value2")
hdr_val_start_idx = val.find('Value2')
if hdr_val_start_idx == -1:
if not test_val("header: Header2", "", resp.read().decode()):
if not test_val('header: Header2', '', resp.read().decode()):
conn.close()
return False
else:
if not test_val("header: Header2", val[hdr_val_start_idx:], resp.read().decode()):
if not test_val('header: Header2', val[hdr_val_start_idx:], resp.read().decode()):
conn.close()
return False
resp.read()
Utility.console_log("Success")
Utility.console_log('Success')
conn.close()
return True
def get_hello_type(dut, port):
# GET /hello/type_html returns text/html as Content-Type'
Utility.console_log("[test] GET /hello/type_html has Content-Type of text/html =>", end=' ')
Utility.console_log('[test] GET /hello/type_html has Content-Type of text/html =>', end=' ')
conn = http.client.HTTPConnection(dut, int(port), timeout=15)
conn.request("GET", "/hello/type_html")
conn.request('GET', '/hello/type_html')
resp = conn.getresponse()
if not test_val("status_code", 200, resp.status):
if not test_val('status_code', 200, resp.status):
conn.close()
return False
if not test_val("data", "Hello World!", resp.read().decode()):
if not test_val('data', 'Hello World!', resp.read().decode()):
conn.close()
return False
if not test_val("data", "text/html", resp.getheader('Content-Type')):
if not test_val('data', 'text/html', resp.getheader('Content-Type')):
conn.close()
return False
Utility.console_log("Success")
Utility.console_log('Success')
conn.close()
return True
def get_hello_status(dut, port):
# GET /hello/status_500 returns status 500'
Utility.console_log("[test] GET /hello/status_500 returns status 500 =>", end=' ')
Utility.console_log('[test] GET /hello/status_500 returns status 500 =>', end=' ')
conn = http.client.HTTPConnection(dut, int(port), timeout=15)
conn.request("GET", "/hello/status_500")
conn.request('GET', '/hello/status_500')
resp = conn.getresponse()
if not test_val("status_code", 500, resp.status):
if not test_val('status_code', 500, resp.status):
conn.close()
return False
Utility.console_log("Success")
Utility.console_log('Success')
conn.close()
return True
def get_false_uri(dut, port):
# GET /false_uri returns status 404'
Utility.console_log("[test] GET /false_uri returns status 404 =>", end=' ')
Utility.console_log('[test] GET /false_uri returns status 404 =>', end=' ')
conn = http.client.HTTPConnection(dut, int(port), timeout=15)
conn.request("GET", "/false_uri")
conn.request('GET', '/false_uri')
resp = conn.getresponse()
if not test_val("status_code", 404, resp.status):
if not test_val('status_code', 404, resp.status):
conn.close()
return False
Utility.console_log("Success")
Utility.console_log('Success')
conn.close()
return True
def parallel_sessions_adder(dut, port, max_sessions):
# POSTs on /adder in parallel sessions
Utility.console_log("[test] POST {pipelined} on /adder in " + str(max_sessions) + " sessions =>", end=' ')
Utility.console_log('[test] POST {pipelined} on /adder in ' + str(max_sessions) + ' sessions =>', end=' ')
t = []
# Create all sessions
for i in range(max_sessions):
@@ -520,90 +518,90 @@ def parallel_sessions_adder(dut, port, max_sessions):
res = True
for i in range(len(t)):
if not test_val("Thread" + str(i) + " Failed", t[i].adder_result(), True):
if not test_val('Thread' + str(i) + ' Failed', t[i].adder_result(), True):
res = False
t[i].close()
if (res):
Utility.console_log("Success")
Utility.console_log('Success')
return res
def async_response_test(dut, port):
# Test that an asynchronous work is executed in the HTTPD's context
# This is tested by reading two responses over the same session
Utility.console_log("[test] Test HTTPD Work Queue (Async response) =>", end=' ')
Utility.console_log('[test] Test HTTPD Work Queue (Async response) =>', end=' ')
s = Session(dut, port)
s.send_get('/async_data')
s.read_resp_hdrs()
if not test_val("First Response", "Hello World!", s.read_resp_data()):
if not test_val('First Response', 'Hello World!', s.read_resp_data()):
s.close()
return False
s.read_resp_hdrs()
if not test_val("Second Response", "Hello Double World!", s.read_resp_data()):
if not test_val('Second Response', 'Hello Double World!', s.read_resp_data()):
s.close()
return False
s.close()
Utility.console_log("Success")
Utility.console_log('Success')
return True
def leftover_data_test(dut, port):
# Leftover data in POST is purged (valid and invalid URIs)
Utility.console_log("[test] Leftover data in POST is purged (valid and invalid URIs) =>", end=' ')
s = http.client.HTTPConnection(dut + ":" + port, timeout=15)
Utility.console_log('[test] Leftover data in POST is purged (valid and invalid URIs) =>', end=' ')
s = http.client.HTTPConnection(dut + ':' + port, timeout=15)
s.request("POST", url='/leftover_data', body="abcdefghijklmnopqrstuvwxyz\r\nabcdefghijklmnopqrstuvwxyz")
s.request('POST', url='/leftover_data', body='abcdefghijklmnopqrstuvwxyz\r\nabcdefghijklmnopqrstuvwxyz')
resp = s.getresponse()
if not test_val("Partial data", "abcdefghij", resp.read().decode()):
if not test_val('Partial data', 'abcdefghij', resp.read().decode()):
s.close()
return False
s.request("GET", url='/hello')
s.request('GET', url='/hello')
resp = s.getresponse()
if not test_val("Hello World Data", "Hello World!", resp.read().decode()):
if not test_val('Hello World Data', 'Hello World!', resp.read().decode()):
s.close()
return False
s.request("POST", url='/false_uri', body="abcdefghijklmnopqrstuvwxyz\r\nabcdefghijklmnopqrstuvwxyz")
s.request('POST', url='/false_uri', body='abcdefghijklmnopqrstuvwxyz\r\nabcdefghijklmnopqrstuvwxyz')
resp = s.getresponse()
if not test_val("False URI Status", str(404), str(resp.status)):
if not test_val('False URI Status', str(404), str(resp.status)):
s.close()
return False
# socket would have been closed by server due to error
s.close()
s = http.client.HTTPConnection(dut + ":" + port, timeout=15)
s.request("GET", url='/hello')
s = http.client.HTTPConnection(dut + ':' + port, timeout=15)
s.request('GET', url='/hello')
resp = s.getresponse()
if not test_val("Hello World Data", "Hello World!", resp.read().decode()):
if not test_val('Hello World Data', 'Hello World!', resp.read().decode()):
s.close()
return False
s.close()
Utility.console_log("Success")
Utility.console_log('Success')
return True
def spillover_session(dut, port, max_sess):
# Session max_sess_sessions + 1 is rejected
Utility.console_log("[test] Session max_sess_sessions (" + str(max_sess) + ") + 1 is rejected =>", end=' ')
Utility.console_log('[test] Session max_sess_sessions (' + str(max_sess) + ') + 1 is rejected =>', end=' ')
s = []
_verbose_ = True
for i in range(max_sess + 1):
if (_verbose_):
Utility.console_log("Executing " + str(i))
Utility.console_log('Executing ' + str(i))
try:
a = http.client.HTTPConnection(dut + ":" + port, timeout=15)
a.request("GET", url='/hello')
a = http.client.HTTPConnection(dut + ':' + port, timeout=15)
a.request('GET', url='/hello')
resp = a.getresponse()
if not test_val("Connection " + str(i), "Hello World!", resp.read().decode()):
if not test_val('Connection ' + str(i), 'Hello World!', resp.read().decode()):
a.close()
break
s.append(a)
except Exception:
if (_verbose_):
Utility.console_log("Connection " + str(i) + " rejected")
Utility.console_log('Connection ' + str(i) + ' rejected')
a.close()
break
@@ -612,134 +610,134 @@ def spillover_session(dut, port, max_sess):
a.close()
# Check if number of connections is equal to max_sess
Utility.console_log(["Fail","Success"][len(s) == max_sess])
Utility.console_log(['Fail','Success'][len(s) == max_sess])
return (len(s) == max_sess)
def recv_timeout_test(dut, port):
Utility.console_log("[test] Timeout occurs if partial packet sent =>", end=' ')
Utility.console_log('[test] Timeout occurs if partial packet sent =>', end=' ')
s = Session(dut, port)
s.client.sendall(b"GE")
s.client.sendall(b'GE')
s.read_resp_hdrs()
resp = s.read_resp_data()
if not test_val("Request Timeout", "Server closed this connection", resp):
if not test_val('Request Timeout', 'Server closed this connection', resp):
s.close()
return False
s.close()
Utility.console_log("Success")
Utility.console_log('Success')
return True
def packet_size_limit_test(dut, port, test_size):
Utility.console_log("[test] send size limit test =>", end=' ')
Utility.console_log('[test] send size limit test =>', end=' ')
retry = 5
while (retry):
retry -= 1
Utility.console_log("data size = ", test_size)
s = http.client.HTTPConnection(dut + ":" + port, timeout=15)
Utility.console_log('data size = ', test_size)
s = http.client.HTTPConnection(dut + ':' + port, timeout=15)
random_data = ''.join(string.printable[random.randint(0,len(string.printable)) - 1] for _ in list(range(test_size)))
path = "/echo"
s.request("POST", url=path, body=random_data)
path = '/echo'
s.request('POST', url=path, body=random_data)
resp = s.getresponse()
if not test_val("Error", "200", str(resp.status)):
if test_val("Error", "500", str(resp.status)):
Utility.console_log("Data too large to be allocated")
if not test_val('Error', '200', str(resp.status)):
if test_val('Error', '500', str(resp.status)):
Utility.console_log('Data too large to be allocated')
test_size = test_size // 10
else:
Utility.console_log("Unexpected error")
Utility.console_log('Unexpected error')
s.close()
Utility.console_log("Retry...")
Utility.console_log('Retry...')
continue
resp = resp.read().decode()
result = (resp == random_data)
if not result:
test_val("Data size", str(len(random_data)), str(len(resp)))
test_val('Data size', str(len(random_data)), str(len(resp)))
s.close()
Utility.console_log("Retry...")
Utility.console_log('Retry...')
continue
s.close()
Utility.console_log("Success")
Utility.console_log('Success')
return True
Utility.console_log("Failed")
Utility.console_log('Failed')
return False
def arbitrary_termination_test(dut, port):
Utility.console_log("[test] Arbitrary termination test =>", end=' ')
Utility.console_log('[test] Arbitrary termination test =>', end=' ')
cases = [
{
"request": "POST /echo HTTP/1.1\r\nHost: " + dut + "\r\nCustom: SomeValue\r\n\r\n",
"code": "200",
"header": "SomeValue"
'request': 'POST /echo HTTP/1.1\r\nHost: ' + dut + '\r\nCustom: SomeValue\r\n\r\n',
'code': '200',
'header': 'SomeValue'
},
{
"request": "POST /echo HTTP/1.1\nHost: " + dut + "\r\nCustom: SomeValue\r\n\r\n",
"code": "200",
"header": "SomeValue"
'request': 'POST /echo HTTP/1.1\nHost: ' + dut + '\r\nCustom: SomeValue\r\n\r\n',
'code': '200',
'header': 'SomeValue'
},
{
"request": "POST /echo HTTP/1.1\r\nHost: " + dut + "\nCustom: SomeValue\r\n\r\n",
"code": "200",
"header": "SomeValue"
'request': 'POST /echo HTTP/1.1\r\nHost: ' + dut + '\nCustom: SomeValue\r\n\r\n',
'code': '200',
'header': 'SomeValue'
},
{
"request": "POST /echo HTTP/1.1\r\nHost: " + dut + "\r\nCustom: SomeValue\n\r\n",
"code": "200",
"header": "SomeValue"
'request': 'POST /echo HTTP/1.1\r\nHost: ' + dut + '\r\nCustom: SomeValue\n\r\n',
'code': '200',
'header': 'SomeValue'
},
{
"request": "POST /echo HTTP/1.1\r\nHost: " + dut + "\r\nCustom: SomeValue\r\n\n",
"code": "200",
"header": "SomeValue"
'request': 'POST /echo HTTP/1.1\r\nHost: ' + dut + '\r\nCustom: SomeValue\r\n\n',
'code': '200',
'header': 'SomeValue'
},
{
"request": "POST /echo HTTP/1.1\nHost: " + dut + "\nCustom: SomeValue\n\n",
"code": "200",
"header": "SomeValue"
'request': 'POST /echo HTTP/1.1\nHost: ' + dut + '\nCustom: SomeValue\n\n',
'code': '200',
'header': 'SomeValue'
},
{
"request": "POST /echo HTTP/1.1\r\nHost: " + dut + "\r\nContent-Length: 5\n\r\nABCDE",
"code": "200",
"body": "ABCDE"
'request': 'POST /echo HTTP/1.1\r\nHost: ' + dut + '\r\nContent-Length: 5\n\r\nABCDE',
'code': '200',
'body': 'ABCDE'
},
{
"request": "POST /echo HTTP/1.1\r\nHost: " + dut + "\r\nContent-Length: 5\r\n\nABCDE",
"code": "200",
"body": "ABCDE"
'request': 'POST /echo HTTP/1.1\r\nHost: ' + dut + '\r\nContent-Length: 5\r\n\nABCDE',
'code': '200',
'body': 'ABCDE'
},
{
"request": "POST /echo HTTP/1.1\r\nHost: " + dut + "\r\nContent-Length: 5\n\nABCDE",
"code": "200",
"body": "ABCDE"
'request': 'POST /echo HTTP/1.1\r\nHost: ' + dut + '\r\nContent-Length: 5\n\nABCDE',
'code': '200',
'body': 'ABCDE'
},
{
"request": "POST /echo HTTP/1.1\r\nHost: " + dut + "\r\nContent-Length: 5\n\n\rABCD",
"code": "200",
"body": "\rABCD"
'request': 'POST /echo HTTP/1.1\r\nHost: ' + dut + '\r\nContent-Length: 5\n\n\rABCD',
'code': '200',
'body': '\rABCD'
},
{
"request": "POST /echo HTTP/1.1\r\nHost: " + dut + "\r\r\nCustom: SomeValue\r\r\n\r\r\n",
"code": "400"
'request': 'POST /echo HTTP/1.1\r\nHost: ' + dut + '\r\r\nCustom: SomeValue\r\r\n\r\r\n',
'code': '400'
},
{
"request": "POST /echo HTTP/1.1\r\r\nHost: " + dut + "\r\n\r\n",
"code": "400"
'request': 'POST /echo HTTP/1.1\r\r\nHost: ' + dut + '\r\n\r\n',
'code': '400'
},
{
"request": "POST /echo HTTP/1.1\r\n\rHost: " + dut + "\r\n\r\n",
"code": "400"
'request': 'POST /echo HTTP/1.1\r\n\rHost: ' + dut + '\r\n\r\n',
'code': '400'
},
{
"request": "POST /echo HTTP/1.1\r\nHost: " + dut + "\rCustom: SomeValue\r\n",
"code": "400"
'request': 'POST /echo HTTP/1.1\r\nHost: ' + dut + '\rCustom: SomeValue\r\n',
'code': '400'
},
{
"request": "POST /echo HTTP/1.1\r\nHost: " + dut + "\r\nCustom: Some\rValue\r\n",
"code": "400"
'request': 'POST /echo HTTP/1.1\r\nHost: ' + dut + '\r\nCustom: Some\rValue\r\n',
'code': '400'
},
{
"request": "POST /echo HTTP/1.1\r\nHost: " + dut + "\r\nCustom- SomeValue\r\n\r\n",
"code": "400"
'request': 'POST /echo HTTP/1.1\r\nHost: ' + dut + '\r\nCustom- SomeValue\r\n\r\n',
'code': '400'
}
]
for case in cases:
@@ -748,159 +746,159 @@ def arbitrary_termination_test(dut, port):
resp_hdrs = s.read_resp_hdrs()
resp_body = s.read_resp_data()
s.close()
if not test_val("Response Code", case["code"], s.status):
if not test_val('Response Code', case['code'], s.status):
return False
if "header" in case.keys():
if 'header' in case.keys():
resp_hdr_val = None
if "Custom" in resp_hdrs.keys():
resp_hdr_val = resp_hdrs["Custom"]
if not test_val("Response Header", case["header"], resp_hdr_val):
if 'Custom' in resp_hdrs.keys():
resp_hdr_val = resp_hdrs['Custom']
if not test_val('Response Header', case['header'], resp_hdr_val):
return False
if "body" in case.keys():
if not test_val("Response Body", case["body"], resp_body):
if 'body' in case.keys():
if not test_val('Response Body', case['body'], resp_body):
return False
Utility.console_log("Success")
Utility.console_log('Success')
return True
def code_500_server_error_test(dut, port):
Utility.console_log("[test] 500 Server Error test =>", end=' ')
Utility.console_log('[test] 500 Server Error test =>', end=' ')
s = Session(dut, port)
# Sending a very large content length will cause malloc to fail
content_len = 2**30
s.client.sendall(("POST /echo HTTP/1.1\r\nHost: " + dut + "\r\nContent-Length: " + str(content_len) + "\r\n\r\nABCD").encode())
s.client.sendall(('POST /echo HTTP/1.1\r\nHost: ' + dut + '\r\nContent-Length: ' + str(content_len) + '\r\n\r\nABCD').encode())
s.read_resp_hdrs()
s.read_resp_data()
if not test_val("Server Error", "500", s.status):
if not test_val('Server Error', '500', s.status):
s.close()
return False
s.close()
Utility.console_log("Success")
Utility.console_log('Success')
return True
def code_501_method_not_impl(dut, port):
Utility.console_log("[test] 501 Method Not Implemented =>", end=' ')
Utility.console_log('[test] 501 Method Not Implemented =>', end=' ')
s = Session(dut, port)
path = "/hello"
s.client.sendall(("ABC " + path + " HTTP/1.1\r\nHost: " + dut + "\r\n\r\n").encode())
path = '/hello'
s.client.sendall(('ABC ' + path + ' HTTP/1.1\r\nHost: ' + dut + '\r\n\r\n').encode())
s.read_resp_hdrs()
s.read_resp_data()
# Presently server sends back 400 Bad Request
# if not test_val("Server Error", "501", s.status):
# s.close()
# return False
if not test_val("Server Error", "400", s.status):
if not test_val('Server Error', '400', s.status):
s.close()
return False
s.close()
Utility.console_log("Success")
Utility.console_log('Success')
return True
def code_505_version_not_supported(dut, port):
Utility.console_log("[test] 505 Version Not Supported =>", end=' ')
Utility.console_log('[test] 505 Version Not Supported =>', end=' ')
s = Session(dut, port)
path = "/hello"
s.client.sendall(("GET " + path + " HTTP/2.0\r\nHost: " + dut + "\r\n\r\n").encode())
path = '/hello'
s.client.sendall(('GET ' + path + ' HTTP/2.0\r\nHost: ' + dut + '\r\n\r\n').encode())
s.read_resp_hdrs()
s.read_resp_data()
if not test_val("Server Error", "505", s.status):
if not test_val('Server Error', '505', s.status):
s.close()
return False
s.close()
Utility.console_log("Success")
Utility.console_log('Success')
return True
def code_400_bad_request(dut, port):
Utility.console_log("[test] 400 Bad Request =>", end=' ')
Utility.console_log('[test] 400 Bad Request =>', end=' ')
s = Session(dut, port)
path = "/hello"
s.client.sendall(("XYZ " + path + " HTTP/1.1\r\nHost: " + dut + "\r\n\r\n").encode())
path = '/hello'
s.client.sendall(('XYZ ' + path + ' HTTP/1.1\r\nHost: ' + dut + '\r\n\r\n').encode())
s.read_resp_hdrs()
s.read_resp_data()
if not test_val("Client Error", "400", s.status):
if not test_val('Client Error', '400', s.status):
s.close()
return False
s.close()
Utility.console_log("Success")
Utility.console_log('Success')
return True
def code_404_not_found(dut, port):
Utility.console_log("[test] 404 Not Found =>", end=' ')
Utility.console_log('[test] 404 Not Found =>', end=' ')
s = Session(dut, port)
path = "/dummy"
s.client.sendall(("GET " + path + " HTTP/1.1\r\nHost: " + dut + "\r\n\r\n").encode())
path = '/dummy'
s.client.sendall(('GET ' + path + ' HTTP/1.1\r\nHost: ' + dut + '\r\n\r\n').encode())
s.read_resp_hdrs()
s.read_resp_data()
if not test_val("Client Error", "404", s.status):
if not test_val('Client Error', '404', s.status):
s.close()
return False
s.close()
Utility.console_log("Success")
Utility.console_log('Success')
return True
def code_405_method_not_allowed(dut, port):
Utility.console_log("[test] 405 Method Not Allowed =>", end=' ')
Utility.console_log('[test] 405 Method Not Allowed =>', end=' ')
s = Session(dut, port)
path = "/hello"
s.client.sendall(("POST " + path + " HTTP/1.1\r\nHost: " + dut + "\r\n\r\n").encode())
path = '/hello'
s.client.sendall(('POST ' + path + ' HTTP/1.1\r\nHost: ' + dut + '\r\n\r\n').encode())
s.read_resp_hdrs()
s.read_resp_data()
if not test_val("Client Error", "405", s.status):
if not test_val('Client Error', '405', s.status):
s.close()
return False
s.close()
Utility.console_log("Success")
Utility.console_log('Success')
return True
def code_408_req_timeout(dut, port):
Utility.console_log("[test] 408 Request Timeout =>", end=' ')
Utility.console_log('[test] 408 Request Timeout =>', end=' ')
s = Session(dut, port)
s.client.sendall(("POST /echo HTTP/1.1\r\nHost: " + dut + "\r\nContent-Length: 10\r\n\r\nABCD").encode())
s.client.sendall(('POST /echo HTTP/1.1\r\nHost: ' + dut + '\r\nContent-Length: 10\r\n\r\nABCD').encode())
s.read_resp_hdrs()
s.read_resp_data()
if not test_val("Client Error", "408", s.status):
if not test_val('Client Error', '408', s.status):
s.close()
return False
s.close()
Utility.console_log("Success")
Utility.console_log('Success')
return True
def code_411_length_required(dut, port):
Utility.console_log("[test] 411 Length Required =>", end=' ')
Utility.console_log('[test] 411 Length Required =>', end=' ')
s = Session(dut, port)
path = "/echo"
s.client.sendall(("POST " + path + " HTTP/1.1\r\nHost: " + dut + "\r\nContent-Type: text/plain\r\nTransfer-Encoding: chunked\r\n\r\n").encode())
path = '/echo'
s.client.sendall(('POST ' + path + ' HTTP/1.1\r\nHost: ' + dut + '\r\nContent-Type: text/plain\r\nTransfer-Encoding: chunked\r\n\r\n').encode())
s.read_resp_hdrs()
s.read_resp_data()
# Presently server sends back 400 Bad Request
# if not test_val("Client Error", "411", s.status):
# s.close()
# return False
if not test_val("Client Error", "400", s.status):
if not test_val('Client Error', '400', s.status):
s.close()
return False
s.close()
Utility.console_log("Success")
Utility.console_log('Success')
return True
def send_getx_uri_len(dut, port, length):
s = Session(dut, port)
method = "GET "
version = " HTTP/1.1\r\n"
path = "/" + "x" * (length - len(method) - len(version) - len("/"))
method = 'GET '
version = ' HTTP/1.1\r\n'
path = '/' + 'x' * (length - len(method) - len(version) - len('/'))
s.client.sendall(method.encode())
time.sleep(1)
s.client.sendall(path.encode())
time.sleep(1)
s.client.sendall((version + "Host: " + dut + "\r\n\r\n").encode())
s.client.sendall((version + 'Host: ' + dut + '\r\n\r\n').encode())
s.read_resp_hdrs()
s.read_resp_data()
s.close()
@@ -908,59 +906,59 @@ def send_getx_uri_len(dut, port, length):
def code_414_uri_too_long(dut, port, max_uri_len):
Utility.console_log("[test] 414 URI Too Long =>", end=' ')
Utility.console_log('[test] 414 URI Too Long =>', end=' ')
status = send_getx_uri_len(dut, port, max_uri_len)
if not test_val("Client Error", "404", status):
if not test_val('Client Error', '404', status):
return False
status = send_getx_uri_len(dut, port, max_uri_len + 1)
if not test_val("Client Error", "414", status):
if not test_val('Client Error', '414', status):
return False
Utility.console_log("Success")
Utility.console_log('Success')
return True
def send_postx_hdr_len(dut, port, length):
s = Session(dut, port)
path = "/echo"
host = "Host: " + dut
custom_hdr_field = "\r\nCustom: "
custom_hdr_val = "x" * (length - len(host) - len(custom_hdr_field) - len("\r\n\r\n") + len("0"))
request = ("POST " + path + " HTTP/1.1\r\n" + host + custom_hdr_field + custom_hdr_val + "\r\n\r\n").encode()
path = '/echo'
host = 'Host: ' + dut
custom_hdr_field = '\r\nCustom: '
custom_hdr_val = 'x' * (length - len(host) - len(custom_hdr_field) - len('\r\n\r\n') + len('0'))
request = ('POST ' + path + ' HTTP/1.1\r\n' + host + custom_hdr_field + custom_hdr_val + '\r\n\r\n').encode()
s.client.sendall(request[:length // 2])
time.sleep(1)
s.client.sendall(request[length // 2:])
hdr = s.read_resp_hdrs()
resp = s.read_resp_data()
s.close()
if hdr and ("Custom" in hdr):
return (hdr["Custom"] == custom_hdr_val), resp
if hdr and ('Custom' in hdr):
return (hdr['Custom'] == custom_hdr_val), resp
return False, s.status
def code_431_hdr_too_long(dut, port, max_hdr_len):
Utility.console_log("[test] 431 Header Too Long =>", end=' ')
Utility.console_log('[test] 431 Header Too Long =>', end=' ')
res, status = send_postx_hdr_len(dut, port, max_hdr_len)
if not res:
return False
res, status = send_postx_hdr_len(dut, port, max_hdr_len + 1)
if not test_val("Client Error", "431", status):
if not test_val('Client Error', '431', status):
return False
Utility.console_log("Success")
Utility.console_log('Success')
return True
def test_upgrade_not_supported(dut, port):
Utility.console_log("[test] Upgrade Not Supported =>", end=' ')
Utility.console_log('[test] Upgrade Not Supported =>', end=' ')
s = Session(dut, port)
# path = "/hello"
s.client.sendall(("OPTIONS * HTTP/1.1\r\nHost:" + dut + "\r\nUpgrade: TLS/1.0\r\nConnection: Upgrade\r\n\r\n").encode())
s.client.sendall(('OPTIONS * HTTP/1.1\r\nHost:' + dut + '\r\nUpgrade: TLS/1.0\r\nConnection: Upgrade\r\n\r\n').encode())
s.read_resp_hdrs()
s.read_resp_data()
if not test_val("Client Error", "400", s.status):
if not test_val('Client Error', '400', s.status):
s.close()
return False
s.close()
Utility.console_log("Success")
Utility.console_log('Success')
return True
@@ -985,7 +983,7 @@ if __name__ == '__main__':
_verbose_ = True
Utility.console_log("### Basic HTTP Client Tests")
Utility.console_log('### Basic HTTP Client Tests')
get_hello(dut, port)
post_hello(dut, port)
put_hello(dut, port)
@@ -997,7 +995,7 @@ if __name__ == '__main__':
get_false_uri(dut, port)
get_test_headers(dut, port)
Utility.console_log("### Error code tests")
Utility.console_log('### Error code tests')
code_500_server_error_test(dut, port)
code_501_method_not_impl(dut, port)
code_505_version_not_supported(dut, port)
@@ -1012,7 +1010,7 @@ if __name__ == '__main__':
# Not supported yet (Error on chunked request)
# code_411_length_required(dut, port)
Utility.console_log("### Sessions and Context Tests")
Utility.console_log('### Sessions and Context Tests')
parallel_sessions_adder(dut, port, max_sessions)
leftover_data_test(dut, port)
async_response_test(dut, port)

17
tools/ci/python_packages/idf_iperf_test_util/Attenuator.py
View File

@@ -18,9 +18,10 @@ Internal use only.
This file provide method to control programmable attenuator.
"""
import time
import serial
import codecs
import time
import serial
def set_att(port, att, att_fix=False):
@@ -47,16 +48,16 @@ def set_att(port, att, att_fix=False):
serial_port = serial.Serial(port, baudrate=9600, rtscts=False, timeout=0.1)
if serial_port.isOpen() is False:
raise IOError("attenuator control, failed to open att port")
raise IOError('attenuator control, failed to open att port')
cmd_hex = "7e7e10{:02x}{:x}".format(att_t, 0x10 + att_t)
exp_res_hex = "7e7e20{:02x}00{:x}".format(att_t, 0x20 + att_t)
cmd_hex = '7e7e10{:02x}{:x}'.format(att_t, 0x10 + att_t)
exp_res_hex = '7e7e20{:02x}00{:x}'.format(att_t, 0x20 + att_t)
cmd = codecs.decode(cmd_hex, "hex")
exp_res = codecs.decode(exp_res_hex, "hex")
cmd = codecs.decode(cmd_hex, 'hex')
exp_res = codecs.decode(exp_res_hex, 'hex')
serial_port.write(cmd)
res = b""
res = b''
for i in range(5):
res += serial_port.read(20)

12
tools/ci/python_packages/idf_iperf_test_util/LineChart.py
View File

@@ -40,18 +40,18 @@ def draw_line_chart(file_name, title, x_label, y_label, data_series, range_list)
_data[str(key)] = data_series[item][key]
_data = list(_data.values())
try:
legend = item + " (max: {:.02f})".format(max([x for x in _data if x]))
legend = item + ' (max: {:.02f})'.format(max([x for x in _data if x]))
except TypeError:
legend = item
line.add_yaxis(legend, _data, is_smooth=True, is_connect_nones=True,
label_opts=opts.LabelOpts(is_show=False))
line.set_global_opts(
datazoom_opts=opts.DataZoomOpts(range_start=0, range_end=100),
title_opts=opts.TitleOpts(title=title, pos_left="center"),
legend_opts=opts.LegendOpts(pos_top="10%", pos_left="right", orient="vertical"),
tooltip_opts=opts.TooltipOpts(trigger="axis"),
xaxis_opts=opts.AxisOpts(type_="category", name=x_label, splitline_opts=opts.SplitLineOpts(is_show=True)),
yaxis_opts=opts.AxisOpts(type_="value", name=y_label,
title_opts=opts.TitleOpts(title=title, pos_left='center'),
legend_opts=opts.LegendOpts(pos_top='10%', pos_left='right', orient='vertical'),
tooltip_opts=opts.TooltipOpts(trigger='axis'),
xaxis_opts=opts.AxisOpts(type_='category', name=x_label, splitline_opts=opts.SplitLineOpts(is_show=True)),
yaxis_opts=opts.AxisOpts(type_='value', name=y_label,
axistick_opts=opts.AxisTickOpts(is_show=True),
splitline_opts=opts.SplitLineOpts(is_show=True)),
)

44
tools/ci/python_packages/idf_iperf_test_util/PowerControl.py
View File

@@ -27,15 +27,15 @@ class Control(object):
@classmethod
def apc_telnet_make_choice(cls, telnet, choice):
""" select a choice """
telnet.read_until(b"Event Log")
telnet.read_until(b">")
telnet.write(choice.encode() + b"\r\n")
telnet.read_until(b'Event Log')
telnet.read_until(b'>')
telnet.write(choice.encode() + b'\r\n')
@classmethod
def apc_telnet_common_action(cls, telnet, check_str, action):
""" wait until a pattern and then write a line """
telnet.read_until(check_str.encode())
telnet.write(action.encode() + b"\r\n")
telnet.write(action.encode() + b'\r\n')
@classmethod
def control(cls, apc_ip, control_dict):
@@ -48,45 +48,45 @@ class Control(object):
for _outlet in control_dict:
assert 0 < _outlet < 9
assert control_dict[_outlet] in ["ON", "OFF"]
assert control_dict[_outlet] in ['ON', 'OFF']
# telnet
# set timeout as 2s so that it won't waste time even can't access APC
tn = telnetlib.Telnet(host=apc_ip, timeout=5)
# log on
cls.apc_telnet_common_action(tn, "User Name :", "apc")
cls.apc_telnet_common_action(tn, "Password :", "apc")
cls.apc_telnet_common_action(tn, 'User Name :', 'apc')
cls.apc_telnet_common_action(tn, 'Password :', 'apc')
# go to Device Manager
cls.apc_telnet_make_choice(tn, "1")
cls.apc_telnet_make_choice(tn, '1')
# go to Outlet Management
cls.apc_telnet_make_choice(tn, "2")
cls.apc_telnet_make_choice(tn, '2')
# go to Outlet Control/Configuration
cls.apc_telnet_make_choice(tn, "1")
cls.apc_telnet_make_choice(tn, '1')
# do select Outlet and control
for _outlet in control_dict:
# choose Outlet
cls.apc_telnet_make_choice(tn, str(_outlet))
# choose Control Outlet
cls.apc_telnet_make_choice(tn, "1")
cls.apc_telnet_make_choice(tn, '1')
# choose action
_action = control_dict[_outlet]
if "ON" in _action:
cls.apc_telnet_make_choice(tn, "1")
if 'ON' in _action:
cls.apc_telnet_make_choice(tn, '1')
else:
cls.apc_telnet_make_choice(tn, "2")
cls.apc_telnet_make_choice(tn, '2')
# do confirm
cls.apc_telnet_common_action(tn, "cancel :", "YES")
cls.apc_telnet_common_action(tn, "continue...", "")
cls.apc_telnet_common_action(tn, 'cancel :', 'YES')
cls.apc_telnet_common_action(tn, 'continue...', '')
# return to Outlet Control/Configuration
cls.apc_telnet_make_choice(tn, "\033")
cls.apc_telnet_make_choice(tn, "\033")
cls.apc_telnet_make_choice(tn, '\033')
cls.apc_telnet_make_choice(tn, '\033')
# exit to main menu and logout
tn.write(b"\033\r\n")
tn.write(b"\033\r\n")
tn.write(b"\033\r\n")
tn.write(b"4\r\n")
tn.write(b'\033\r\n')
tn.write(b'\033\r\n')
tn.write(b'\033\r\n')
tn.write(b'4\r\n')
@classmethod
def control_rest(cls, apc_ip, outlet, action):

80
tools/ci/python_packages/idf_iperf_test_util/TestReport.py
View File

@@ -10,9 +10,9 @@ import os
class ThroughputForConfigsReport(object):
THROUGHPUT_TYPES = ["tcp_tx", "tcp_rx", "udp_tx", "udp_rx"]
THROUGHPUT_TYPES = ['tcp_tx', 'tcp_rx', 'udp_tx', 'udp_rx']
REPORT_FILE_NAME = "ThroughputForConfigs.md"
REPORT_FILE_NAME = 'ThroughputForConfigs.md'
def __init__(self, output_path, ap_ssid, throughput_results, sdkconfig_files):
"""
@@ -42,14 +42,14 @@ class ThroughputForConfigsReport(object):
@staticmethod
def _parse_config_file(config_file_path):
sdkconfig = {}
with open(config_file_path, "r") as f:
with open(config_file_path, 'r') as f:
for line in f:
if not line.isspace():
if line[0] == "#":
if line[0] == '#':
continue
name, value = line.split("=")
value = value.strip("\r\n")
sdkconfig[name] = value if value else "n"
name, value = line.split('=')
value = value.strip('\r\n')
sdkconfig[name] = value if value else 'n'
return sdkconfig
def _generate_the_difference_between_configs(self):
@@ -65,7 +65,7 @@ class ThroughputForConfigsReport(object):
"""
data = "## Config Definition:\r\n\r\n"
data = '## Config Definition:\r\n\r\n'
def find_difference(base, new):
_difference = {}
@@ -75,13 +75,13 @@ class ThroughputForConfigsReport(object):
try:
_base_value = base[_config]
except KeyError:
_base_value = "null"
_base_value = 'null'
try:
_new_value = new[_config]
except KeyError:
_new_value = "null"
_new_value = 'null'
if _base_value != _new_value:
_difference[_config] = "{} -> {}".format(_base_value, _new_value)
_difference[_config] = '{} -> {}'.format(_base_value, _new_value)
return _difference
for i, _config_name in enumerate(self.sort_order):
@@ -96,9 +96,9 @@ class ThroughputForConfigsReport(object):
if previous_config:
# log the difference
difference = find_difference(previous_config, current_config)
data += "* {} (compared to {}):\r\n".format(_config_name, previous_config_name)
data += '* {} (compared to {}):\r\n'.format(_config_name, previous_config_name)
for diff_name in difference:
data += " * `{}`: {}\r\n".format(diff_name, difference[diff_name])
data += ' * `{}`: {}\r\n'.format(diff_name, difference[diff_name])
return data
def _generate_report_for_one_type(self, throughput_type):
@@ -115,39 +115,39 @@ class ThroughputForConfigsReport(object):
"""
empty = True
ret = "\r\n### {} {}\r\n\r\n".format(*throughput_type.split("_"))
ret += "| config name | throughput (Mbps) | free heap size (bytes) |\r\n"
ret += "|-------------|-------------------|------------------------|\r\n"
ret = '\r\n### {} {}\r\n\r\n'.format(*throughput_type.split('_'))
ret += '| config name | throughput (Mbps) | free heap size (bytes) |\r\n'
ret += '|-------------|-------------------|------------------------|\r\n'
for config in self.sort_order:
try:
result = self.results[config][throughput_type]
throughput = "{:.02f}".format(max(result.throughput_by_att[self.ap_ssid].values()))
throughput = '{:.02f}'.format(max(result.throughput_by_att[self.ap_ssid].values()))
heap_size = str(result.heap_size)
# although markdown table will do alignment
# do align here for better text editor presentation
ret += "| {:<12}| {:<18}| {:<23}|\r\n".format(config, throughput, heap_size)
ret += '| {:<12}| {:<18}| {:<23}|\r\n'.format(config, throughput, heap_size)
empty = False
except KeyError:
pass
return ret if not empty else ""
return ret if not empty else ''
def generate_report(self):
data = "# Throughput for different configs\r\n"
data += "\r\nAP: {}\r\n".format(self.ap_ssid)
data = '# Throughput for different configs\r\n'
data += '\r\nAP: {}\r\n'.format(self.ap_ssid)
for throughput_type in self.THROUGHPUT_TYPES:
data += self._generate_report_for_one_type(throughput_type)
data += "\r\n------\r\n"
data += '\r\n------\r\n'
data += self._generate_the_difference_between_configs()
with open(os.path.join(self.output_path, self.REPORT_FILE_NAME), "w") as f:
with open(os.path.join(self.output_path, self.REPORT_FILE_NAME), 'w') as f:
f.write(data)
class ThroughputVsRssiReport(object):
REPORT_FILE_NAME = "ThroughputVsRssi.md"
REPORT_FILE_NAME = 'ThroughputVsRssi.md'
def __init__(self, output_path, throughput_results):
"""
@@ -160,7 +160,7 @@ class ThroughputVsRssiReport(object):
}
"""
self.output_path = output_path
self.raw_data_path = os.path.join(output_path, "raw_data")
self.raw_data_path = os.path.join(output_path, 'raw_data')
self.results = throughput_results
self.throughput_types = list(self.results.keys())
self.throughput_types.sort()
@@ -179,20 +179,20 @@ class ThroughputVsRssiReport(object):
| udp rx | Failed | 55.44 |
"""
ret = "\r\n### Summary\r\n\r\n"
ret += "| item | curve analysis | max throughput (Mbps) |\r\n"
ret += "|---------|----------------|-----------------------|\r\n"
ret = '\r\n### Summary\r\n\r\n'
ret += '| item | curve analysis | max throughput (Mbps) |\r\n'
ret += '|---------|----------------|-----------------------|\r\n'
for _type in self.throughput_types:
result = self.results[_type]
max_throughput = 0.0
curve_analysis = "Failed" if result.error_list else "Success"
curve_analysis = 'Failed' if result.error_list else 'Success'
for ap_ssid in result.throughput_by_att:
_max_for_ap = max(result.throughput_by_rssi[ap_ssid].values())
if _max_for_ap > max_throughput:
max_throughput = _max_for_ap
max_throughput = "{:.02f}".format(max_throughput)
ret += "| {:<8}| {:<15}| {:<22}|\r\n".format("{}_{}".format(result.proto, result.direction),
max_throughput = '{:.02f}'.format(max_throughput)
ret += '| {:<8}| {:<15}| {:<22}|\r\n'.format('{}_{}'.format(result.proto, result.direction),
curve_analysis, max_throughput)
return ret
@@ -217,29 +217,29 @@ class ThroughputVsRssiReport(object):
"""
result.post_analysis()
ret = "\r\n### {} {}\r\n".format(result.proto, result.direction)
ret = '\r\n### {} {}\r\n'.format(result.proto, result.direction)
if result.error_list:
ret += "\r\nErrors:\r\n\r\n"
ret += '\r\nErrors:\r\n\r\n'
for error in result.error_list:
ret += "* " + error + "\r\n"
ret += '* ' + error + '\r\n'
for ap_ssid in result.throughput_by_rssi:
ret += "\r\nAP: {}\r\n".format(ap_ssid)
ret += '\r\nAP: {}\r\n'.format(ap_ssid)
# draw figure
file_name = result.draw_throughput_figure(self.raw_data_path, ap_ssid, "rssi")
result.draw_throughput_figure(self.raw_data_path, ap_ssid, "att")
file_name = result.draw_throughput_figure(self.raw_data_path, ap_ssid, 'rssi')
result.draw_throughput_figure(self.raw_data_path, ap_ssid, 'att')
result.draw_rssi_vs_att_figure(self.raw_data_path, ap_ssid)
ret += "\r\n[throughput Vs RSSI]({})\r\n".format(os.path.join("raw_data", file_name))
ret += '\r\n[throughput Vs RSSI]({})\r\n'.format(os.path.join('raw_data', file_name))
return ret
def generate_report(self):
data = "# Throughput Vs RSSI\r\n"
data = '# Throughput Vs RSSI\r\n'
data += self._generate_summary()
for _type in self.throughput_types:
data += self._generate_report_for_one_type(self.results[_type])
with open(os.path.join(self.output_path, self.REPORT_FILE_NAME), "w") as f:
with open(os.path.join(self.output_path, self.REPORT_FILE_NAME), 'w') as f:
f.write(data)

4
tools/ci/python_packages/tiny_test_fw/App.py
View File

@@ -80,9 +80,9 @@ class BaseApp(object):
if not test_suite_name:
test_suite_name = os.path.splitext(os.path.basename(sys.modules['__main__'].__file__))[0]
sdk_path = cls.get_sdk_path()
log_folder = os.path.join(sdk_path, "TEST_LOGS",
log_folder = os.path.join(sdk_path, 'TEST_LOGS',
test_suite_name +
time.strftime("_%m%d_%H_%M_%S", time.localtime(LOG_FOLDER_TIMESTAMP)))
time.strftime('_%m%d_%H_%M_%S', time.localtime(LOG_FOLDER_TIMESTAMP)))
if not os.path.exists(log_folder):
os.makedirs(log_folder)
return log_folder

119
tools/ci/python_packages/tiny_test_fw/DUT.py
View File

@@ -38,12 +38,13 @@ If they using different port then need to implement their DUTPort class as well.
"""
from __future__ import print_function
import time
import copy
import functools
import re
import sys
import threading
import copy
import functools
import time
# python2 and python3 queue package name is different
try:
@@ -82,15 +83,15 @@ def _decode_data(data):
# convert bytes to string. This is a bit of a hack, we know that we want to log this
# later so encode to the stdout encoding with backslash escapes for anything non-encodable
try:
return data.decode(sys.stdout.encoding, "backslashreplace")
return data.decode(sys.stdout.encoding, 'backslashreplace')
except UnicodeDecodeError: # Python <3.5 doesn't support backslashreplace
return data.decode(sys.stdout.encoding, "replace")
return data.decode(sys.stdout.encoding, 'replace')
return data
def _pattern_to_string(pattern):
try:
ret = "RegEx: " + pattern.pattern
ret = 'RegEx: ' + pattern.pattern
except AttributeError:
ret = pattern
return ret
@@ -167,7 +168,7 @@ class _LogThread(threading.Thread, _queue.Queue):
Then data will be passed to ``expect`` as soon as received.
"""
def __init__(self):
threading.Thread.__init__(self, name="LogThread")
threading.Thread.__init__(self, name='LogThread')
_queue.Queue.__init__(self, maxsize=0)
self.setDaemon(True)
self.flush_lock = threading.Lock()
@@ -177,7 +178,7 @@ class _LogThread(threading.Thread, _queue.Queue):
:param filename: log file name
:param data: log data. Must be ``bytes``.
"""
self.put({"filename": filename, "data": data})
self.put({'filename': filename, 'data': data})
def flush_data(self):
with self.flush_lock:
@@ -187,14 +188,14 @@ class _LogThread(threading.Thread, _queue.Queue):
try:
log = self.get_nowait()
try:
data_cache[log["filename"]] += log["data"]
data_cache[log['filename']] += log['data']
except KeyError:
data_cache[log["filename"]] = log["data"]
data_cache[log['filename']] = log['data']
except _queue.Empty:
break
# flush data
for filename in data_cache:
with open(filename, "ab+") as f:
with open(filename, 'ab+') as f:
f.write(data_cache[filename])
def run(self):
@@ -231,7 +232,7 @@ class RecvThread(threading.Thread):
lines = decoded_data.splitlines(True)
last_line = lines[-1]
if last_line[-1] != "\n":
if last_line[-1] != '\n':
if len(lines) == 1:
# only one line and the line is not finished, then append this to cache
self._line_cache += lines[-1]
@@ -239,7 +240,7 @@ class RecvThread(threading.Thread):
else:
# more than one line and not finished, replace line cache
self._line_cache = lines[-1]
ret += "".join(lines[:-1])
ret += ''.join(lines[:-1])
else:
# line finishes, flush cache
self._line_cache = str()
@@ -302,7 +303,7 @@ class BaseDUT(object):
self.start_receive()
def __str__(self):
return "DUT({}: {})".format(self.name, str(self.port))
return 'DUT({}: {})'.format(self.name, str(self.port))
def _save_expect_failure(self, pattern, data, start_time):
"""
@@ -311,8 +312,8 @@ class BaseDUT(object):
The expect failures could be false alarm, and test case might generate a lot of such failures.
Therefore, we don't print the failure immediately and limit the max size of failure list.
"""
self.expect_failures.insert(0, {"pattern": pattern, "data": data,
"start": start_time, "end": time.time()})
self.expect_failures.insert(0, {'pattern': pattern, 'data': data,
'start': start_time, 'end': time.time()})
self.expect_failures = self.expect_failures[:self.MAX_EXPECT_FAILURES_TO_SAVED]
def _save_dut_log(self, data):
@@ -444,7 +445,7 @@ class BaseDUT(object):
raise e
return data
def write(self, data, eol="\r\n", flush=True):
def write(self, data, eol='\r\n', flush=True):
"""
:param data: data
:param eol: end of line pattern.
@@ -474,7 +475,7 @@ class BaseDUT(object):
self.data_cache.flush(size)
return data
def start_capture_raw_data(self, capture_id="default"):
def start_capture_raw_data(self, capture_id='default'):
"""
Sometime application want to get DUT raw data and use ``expect`` method at the same time.
Capture methods provides a way to get raw data without affecting ``expect`` or ``read`` method.
@@ -491,7 +492,7 @@ class BaseDUT(object):
# otherwise, create new data cache
self.recorded_data[capture_id] = _DataCache()
def stop_capture_raw_data(self, capture_id="default"):
def stop_capture_raw_data(self, capture_id='default'):
"""
Stop capture and get raw data.
This method should be used after ``start_capture_raw_data`` on the same capture ID.
@@ -504,9 +505,9 @@ class BaseDUT(object):
ret = self.recorded_data[capture_id].get_data()
self.recorded_data.pop(capture_id)
except KeyError as e:
e.message = "capture_id does not exist. " \
"You should call start_capture_raw_data with same ID " \
"before calling stop_capture_raw_data"
e.message = 'capture_id does not exist. ' \
'You should call start_capture_raw_data with same ID ' \
'before calling stop_capture_raw_data'
raise e
return ret
@@ -552,9 +553,9 @@ class BaseDUT(object):
return ret, index
EXPECT_METHOD = [
[type(re.compile("")), "_expect_re"],
[type(b''), "_expect_str"], # Python 2 & 3 hook to work without 'from builtins import str' from future
[type(u''), "_expect_str"],
[type(re.compile('')), '_expect_re'],
[type(b''), '_expect_str'], # Python 2 & 3 hook to work without 'from builtins import str' from future
[type(u''), '_expect_str'],
]
def _get_expect_method(self, pattern):
@@ -607,7 +608,7 @@ class BaseDUT(object):
if ret is None:
pattern = _pattern_to_string(pattern)
self._save_expect_failure(pattern, data, start_time)
raise ExpectTimeout(self.name + ": " + pattern)
raise ExpectTimeout(self.name + ': ' + pattern)
return stdout if full_stdout else ret
def _expect_multi(self, expect_all, expect_item_list, timeout):
@@ -622,12 +623,12 @@ class BaseDUT(object):
def process_expected_item(item_raw):
# convert item raw data to standard dict
item = {
"pattern": item_raw[0] if isinstance(item_raw, tuple) else item_raw,
"method": self._get_expect_method(item_raw[0] if isinstance(item_raw, tuple)
'pattern': item_raw[0] if isinstance(item_raw, tuple) else item_raw,
'method': self._get_expect_method(item_raw[0] if isinstance(item_raw, tuple)
else item_raw),
"callback": item_raw[1] if isinstance(item_raw, tuple) else None,
"index": -1,
"ret": None,
'callback': item_raw[1] if isinstance(item_raw, tuple) else None,
'index': -1,
'ret': None,
}
return item
@@ -642,9 +643,9 @@ class BaseDUT(object):
for expect_item in expect_items:
if expect_item not in matched_expect_items:
# exclude those already matched
expect_item["ret"], expect_item["index"] = \
expect_item["method"](data, expect_item["pattern"])
if expect_item["ret"] is not None:
expect_item['ret'], expect_item['index'] = \
expect_item['method'](data, expect_item['pattern'])
if expect_item['ret'] is not None:
# match succeed for one item
matched_expect_items.append(expect_item)
@@ -664,20 +665,20 @@ class BaseDUT(object):
if match_succeed:
# sort matched items according to order of appearance in the input data,
# so that the callbacks are invoked in correct order
matched_expect_items = sorted(matched_expect_items, key=lambda it: it["index"])
matched_expect_items = sorted(matched_expect_items, key=lambda it: it['index'])
# invoke callbacks and flush matched data cache
slice_index = -1
for expect_item in matched_expect_items:
# trigger callback
if expect_item["callback"]:
expect_item["callback"](expect_item["ret"])
slice_index = max(slice_index, expect_item["index"])
if expect_item['callback']:
expect_item['callback'](expect_item['ret'])
slice_index = max(slice_index, expect_item['index'])
# flush already matched data
self.data_cache.flush(slice_index)
else:
pattern = str([_pattern_to_string(x["pattern"]) for x in expect_items])
pattern = str([_pattern_to_string(x['pattern']) for x in expect_items])
self._save_expect_failure(pattern, data, start_time)
raise ExpectTimeout(self.name + ": " + pattern)
raise ExpectTimeout(self.name + ': ' + pattern)
@_expect_lock
def expect_any(self, *expect_items, **timeout):
@@ -697,8 +698,8 @@ class BaseDUT(object):
"""
# to be compatible with python2
# in python3 we can write f(self, *expect_items, timeout=DEFAULT_TIMEOUT)
if "timeout" not in timeout:
timeout["timeout"] = self.DEFAULT_EXPECT_TIMEOUT
if 'timeout' not in timeout:
timeout['timeout'] = self.DEFAULT_EXPECT_TIMEOUT
return self._expect_multi(False, expect_items, **timeout)
@_expect_lock
@@ -719,38 +720,38 @@ class BaseDUT(object):
"""
# to be compatible with python2
# in python3 we can write f(self, *expect_items, timeout=DEFAULT_TIMEOUT)
if "timeout" not in timeout:
timeout["timeout"] = self.DEFAULT_EXPECT_TIMEOUT
if 'timeout' not in timeout:
timeout['timeout'] = self.DEFAULT_EXPECT_TIMEOUT
return self._expect_multi(True, expect_items, **timeout)
@staticmethod
def _format_ts(ts):
return "{}:{}".format(time.strftime("%m-%d %H:%M:%S", time.localtime(ts)), str(ts % 1)[2:5])
return '{}:{}'.format(time.strftime('%m-%d %H:%M:%S', time.localtime(ts)), str(ts % 1)[2:5])
def print_debug_info(self):
"""
Print debug info of current DUT. Currently we will print debug info for expect failures.
"""
Utility.console_log("DUT debug info for DUT: {}:".format(self.name), color="orange")
Utility.console_log('DUT debug info for DUT: {}:'.format(self.name), color='orange')
for failure in self.expect_failures:
Utility.console_log(u"\t[pattern]: {}\r\n\t[data]: {}\r\n\t[time]: {} - {}\r\n"
.format(failure["pattern"], failure["data"],
self._format_ts(failure["start"]), self._format_ts(failure["end"])),
color="orange")
Utility.console_log(u'\t[pattern]: {}\r\n\t[data]: {}\r\n\t[time]: {} - {}\r\n'
.format(failure['pattern'], failure['data'],
self._format_ts(failure['start']), self._format_ts(failure['end'])),
color='orange')
class SerialDUT(BaseDUT):
""" serial with logging received data feature """
DEFAULT_UART_CONFIG = {
"baudrate": 115200,
"bytesize": serial.EIGHTBITS,
"parity": serial.PARITY_NONE,
"stopbits": serial.STOPBITS_ONE,
"timeout": 0.05,
"xonxoff": False,
"rtscts": False,
'baudrate': 115200,
'bytesize': serial.EIGHTBITS,
'parity': serial.PARITY_NONE,
'stopbits': serial.STOPBITS_ONE,
'timeout': 0.05,
'xonxoff': False,
'rtscts': False,
}
def __init__(self, name, port, log_file, app, **kwargs):
@@ -768,8 +769,8 @@ class SerialDUT(BaseDUT):
:param data: raw data from read
:return: formatted data (str)
"""
timestamp = "[{}]".format(self._format_ts(time.time()))
formatted_data = timestamp.encode() + b"\r\n" + data + b"\r\n"
timestamp = '[{}]'.format(self._format_ts(time.time()))
formatted_data = timestamp.encode() + b'\r\n' + data + b'\r\n'
return formatted_data
def _port_open(self):

30
tools/ci/python_packages/tiny_test_fw/Env.py
View File

@@ -13,12 +13,12 @@
# limitations under the License.
""" Test Env, manages DUT, App and EnvConfig, interface for test cases to access these components """
import functools
import os
import threading
import functools
import traceback
import netifaces
import traceback
from . import EnvConfig
@@ -44,7 +44,7 @@ class Env(object):
:keyword env_config_file: test env config file path
:keyword test_name: test suite name, used when generate log folder name
"""
CURRENT_LOG_FOLDER = ""
CURRENT_LOG_FOLDER = ''
def __init__(self,
app=None,
@@ -79,7 +79,7 @@ class Env(object):
:return: dut instance
"""
if dut_name in self.allocated_duts:
dut = self.allocated_duts[dut_name]["dut"]
dut = self.allocated_duts[dut_name]['dut']
else:
if dut_class is None:
dut_class = self.default_dut_cls
@@ -95,7 +95,7 @@ class Env(object):
result, detected_target = dut_class.confirm_dut(port)
except ValueError:
# try to auto detect ports
allocated_ports = [self.allocated_duts[x]["port"] for x in self.allocated_duts]
allocated_ports = [self.allocated_duts[x]['port'] for x in self.allocated_duts]
available_ports = dut_class.list_available_ports()
for port in available_ports:
if port not in allocated_ports:
@@ -113,17 +113,17 @@ class Env(object):
if port:
try:
dut_config = self.get_variable(dut_name + "_port_config")
dut_config = self.get_variable(dut_name + '_port_config')
except ValueError:
dut_config = dict()
dut_config.update(dut_init_args)
dut = dut_class(dut_name, port,
os.path.join(self.log_path, dut_name + ".log"),
os.path.join(self.log_path, dut_name + '.log'),
app_inst,
**dut_config)
self.allocated_duts[dut_name] = {"port": port, "dut": dut}
self.allocated_duts[dut_name] = {'port': port, 'dut': dut}
else:
raise ValueError("Failed to get DUT")
raise ValueError('Failed to get DUT')
return dut
@_synced
@@ -136,7 +136,7 @@ class Env(object):
:return: None
"""
try:
dut = self.allocated_duts.pop(dut_name)["dut"]
dut = self.allocated_duts.pop(dut_name)['dut']
dut.close()
except KeyError:
pass
@@ -153,13 +153,13 @@ class Env(object):
return self.config.get_variable(variable_name)
PROTO_MAP = {
"ipv4": netifaces.AF_INET,
"ipv6": netifaces.AF_INET6,
"mac": netifaces.AF_LINK,
'ipv4': netifaces.AF_INET,
'ipv6': netifaces.AF_INET6,
'mac': netifaces.AF_LINK,
}
@_synced
def get_pc_nic_info(self, nic_name="pc_nic", proto="ipv4"):
def get_pc_nic_info(self, nic_name='pc_nic', proto='ipv4'):
"""
get_pc_nic_info(nic_name="pc_nic")
try to get info of a specified NIC and protocol.
@@ -192,7 +192,7 @@ class Env(object):
"""
dut_close_errors = []
for dut_name in self.allocated_duts:
dut = self.allocated_duts[dut_name]["dut"]
dut = self.allocated_duts[dut_name]['dut']
try:
if dut_debug:
dut.print_debug_info()

2
tools/ci/python_packages/tiny_test_fw/EnvConfig.py
View File

@@ -79,5 +79,5 @@ class Config(object):
# TODO: to support auto get variable here
value = None
if value is None:
raise ValueError("Failed to get variable")
raise ValueError('Failed to get variable')
return value

55
tools/ci/python_packages/tiny_test_fw/TinyFW.py
View File

@@ -13,18 +13,15 @@
# limitations under the License.
""" Interface for test cases. """
import os
import time
import functools
import os
import socket
import time
from datetime import datetime
import junit_xml
from . import Env
from . import DUT
from . import App
from . import Utility
from . import DUT, App, Env, Utility
class TestCaseFailed(AssertionError):
@@ -37,7 +34,7 @@ class TestCaseFailed(AssertionError):
'cases' argument is the names of one or more test cases
"""
message = "Test case{} failed: {}".format("s" if len(cases) > 1 else "", ", ".join(str(c) for c in cases))
message = 'Test case{} failed: {}'.format('s' if len(cases) > 1 else '', ', '.join(str(c) for c in cases))
super(TestCaseFailed, self).__init__(self, message)
@@ -50,11 +47,11 @@ class DefaultEnvConfig(object):
3. default env config get from this class
"""
DEFAULT_CONFIG = {
"app": App.BaseApp,
"dut": DUT.BaseDUT,
"env_tag": "default",
"env_config_file": None,
"test_suite_name": None,
'app': App.BaseApp,
'dut': DUT.BaseDUT,
'env_tag': 'default',
'env_config_file': None,
'test_suite_name': None,
}
@classmethod
@@ -78,10 +75,10 @@ get_default_config = DefaultEnvConfig.get_default_config
MANDATORY_INFO = {
"execution_time": 1,
"env_tag": "default",
"category": "function",
"ignore": False,
'execution_time': 1,
'env_tag': 'default',
'category': 'function',
'ignore': False,
}
@@ -89,8 +86,8 @@ class JunitReport(object):
# wrapper for junit test report
# TODO: JunitReport methods are not thread safe (although not likely to be used this way).
JUNIT_FILE_NAME = "XUNIT_RESULT.xml"
JUNIT_DEFAULT_TEST_SUITE = "test-suite"
JUNIT_FILE_NAME = 'XUNIT_RESULT.xml'
JUNIT_DEFAULT_TEST_SUITE = 'test-suite'
JUNIT_TEST_SUITE = junit_xml.TestSuite(JUNIT_DEFAULT_TEST_SUITE,
hostname=socket.gethostname(),
timestamp=datetime.utcnow().isoformat())
@@ -100,7 +97,7 @@ class JunitReport(object):
@classmethod
def output_report(cls, junit_file_path):
""" Output current test result to file. """
with open(os.path.join(junit_file_path, cls.JUNIT_FILE_NAME), "w") as f:
with open(os.path.join(junit_file_path, cls.JUNIT_FILE_NAME), 'w') as f:
cls.JUNIT_TEST_SUITE.to_file(f, [cls.JUNIT_TEST_SUITE], prettyprint=False)
@classmethod
@@ -136,7 +133,7 @@ class JunitReport(object):
"""
# set stdout to empty string, so we can always append string to stdout.
# It won't affect output logic. If stdout is empty, it won't be put to report.
test_case = junit_xml.TestCase(name, stdout="")
test_case = junit_xml.TestCase(name, stdout='')
cls.JUNIT_CURRENT_TEST_CASE = test_case
cls._TEST_CASE_CREATED_TS = time.time()
return test_case
@@ -151,7 +148,7 @@ class JunitReport(object):
assert cls.JUNIT_CURRENT_TEST_CASE
for item in performance_items:
cls.JUNIT_CURRENT_TEST_CASE.stdout += "[{}]: {}\n".format(item[0], item[1])
cls.JUNIT_CURRENT_TEST_CASE.stdout += '[{}]: {}\n'.format(item[0], item[1])
def test_method(**kwargs):
@@ -174,8 +171,8 @@ def test_method(**kwargs):
def test(test_func):
case_info = MANDATORY_INFO.copy()
case_info["name"] = case_info["ID"] = test_func.__name__
case_info["junit_report_by_case"] = False
case_info['name'] = case_info['ID'] = test_func.__name__
case_info['junit_report_by_case'] = False
case_info.update(kwargs)
@functools.wraps(test_func)
@@ -197,12 +194,12 @@ def test_method(**kwargs):
env_inst = Env.Env(**env_config)
# prepare for xunit test results
junit_file_path = env_inst.app_cls.get_log_folder(env_config["test_suite_name"])
junit_test_case = JunitReport.create_test_case(case_info["ID"])
junit_file_path = env_inst.app_cls.get_log_folder(env_config['test_suite_name'])
junit_test_case = JunitReport.create_test_case(case_info['ID'])
result = False
try:
Utility.console_log("starting running test: " + test_func.__name__, color="green")
Utility.console_log('starting running test: ' + test_func.__name__, color='green')
# execute test function
test_func(env_inst, extra_data)
# if finish without exception, test result is True
@@ -224,16 +221,16 @@ def test_method(**kwargs):
for error in close_errors:
junit_test_case.add_failure_info(str(error))
result = False
if not case_info["junit_report_by_case"]:
if not case_info['junit_report_by_case']:
JunitReport.test_case_finish(junit_test_case)
# end case and output result
JunitReport.output_report(junit_file_path)
if result:
Utility.console_log("Test Succeed: " + test_func.__name__, color="green")
Utility.console_log('Test Succeed: ' + test_func.__name__, color='green')
else:
Utility.console_log(("Test Fail: " + test_func.__name__), color="red")
Utility.console_log(('Test Fail: ' + test_func.__name__), color='red')
return result
handle_test.case_info = case_info

54
tools/ci/python_packages/tiny_test_fw/Utility/CIAssignTest.py
View File

@@ -39,9 +39,9 @@ The Basic logic to assign test cases is as follow:
"""
import json
import os
import re
import json
import yaml
@@ -50,13 +50,13 @@ try:
except ImportError:
from yaml import Loader as Loader
from . import (CaseConfig, SearchCases, GitlabCIJob, console_log)
from . import CaseConfig, GitlabCIJob, SearchCases, console_log
class Group(object):
MAX_EXECUTION_TIME = 30
MAX_CASE = 15
SORT_KEYS = ["env_tag"]
SORT_KEYS = ['env_tag']
# Matching CI job rules could be different from the way we want to group test cases.
# For example, when assign unit test cases, different test cases need to use different test functions.
# We need to put them into different groups.
@@ -92,7 +92,7 @@ class Group(object):
:return: True or False
"""
max_time = (sum([self._get_case_attr(x, "execution_time") for x in self.case_list])
max_time = (sum([self._get_case_attr(x, 'execution_time') for x in self.case_list])
< self.MAX_EXECUTION_TIME)
max_case = (len(self.case_list) < self.MAX_CASE)
return max_time and max_case
@@ -135,8 +135,8 @@ class Group(object):
:return: {"Filter": case filter, "CaseConfig": list of case configs for cases in this group}
"""
output_data = {
"Filter": self.filters,
"CaseConfig": [{"name": self._get_case_attr(x, "name")} for x in self.case_list],
'Filter': self.filters,
'CaseConfig': [{'name': self._get_case_attr(x, 'name')} for x in self.case_list],
}
return output_data
@@ -149,12 +149,12 @@ class AssignTest(object):
:param ci_config_file: path of ``.gitlab-ci.yml``
"""
# subclass need to rewrite CI test job pattern, to filter all test jobs
CI_TEST_JOB_PATTERN = re.compile(r"^test_.+")
CI_TEST_JOB_PATTERN = re.compile(r'^test_.+')
# by default we only run function in CI, as other tests could take long time
DEFAULT_FILTER = {
"category": "function",
"ignore": False,
"supported_in_ci": True,
'category': 'function',
'ignore': False,
'supported_in_ci': True,
}
def __init__(self, test_case_paths, ci_config_file, case_group=Group):
@@ -168,25 +168,25 @@ class AssignTest(object):
def _handle_parallel_attribute(job_name, job):
jobs_out = []
try:
for i in range(job["parallel"]):
jobs_out.append(GitlabCIJob.Job(job, job_name + "_{}".format(i + 1)))
for i in range(job['parallel']):
jobs_out.append(GitlabCIJob.Job(job, job_name + '_{}'.format(i + 1)))
except KeyError:
# Gitlab don't allow to set parallel to 1.
# to make test job name same ($CI_JOB_NAME_$CI_NODE_INDEX),
# we append "_" to jobs don't have parallel attribute
jobs_out.append(GitlabCIJob.Job(job, job_name + "_"))
jobs_out.append(GitlabCIJob.Job(job, job_name + '_'))
return jobs_out
def _parse_gitlab_ci_config(self, ci_config_file):
with open(ci_config_file, "r") as f:
with open(ci_config_file, 'r') as f:
ci_config = yaml.load(f, Loader=Loader)
job_list = list()
for job_name in ci_config:
if self.CI_TEST_JOB_PATTERN.search(job_name) is not None:
job_list.extend(self._handle_parallel_attribute(job_name, ci_config[job_name]))
job_list.sort(key=lambda x: x["name"])
job_list.sort(key=lambda x: x['name'])
return job_list
def search_cases(self, case_filter=None):
@@ -256,7 +256,7 @@ class AssignTest(object):
Bot could also pass test count.
If filtered cases need to be tested for several times, then we do duplicate them here.
"""
test_count = os.getenv("BOT_TEST_COUNT")
test_count = os.getenv('BOT_TEST_COUNT')
if test_count:
test_count = int(test_count)
self.test_cases *= test_count
@@ -269,7 +269,7 @@ class AssignTest(object):
"""
group_count = dict()
for group in test_groups:
key = ",".join(group.ci_job_match_keys)
key = ','.join(group.ci_job_match_keys)
try:
group_count[key] += 1
except KeyError:
@@ -305,26 +305,26 @@ class AssignTest(object):
# print debug info
# total requirement of current pipeline
required_group_count = self._count_groups_by_keys(test_groups)
console_log("Required job count by tags:")
console_log('Required job count by tags:')
for tags in required_group_count:
console_log("\t{}: {}".format(tags, required_group_count[tags]))
console_log('\t{}: {}'.format(tags, required_group_count[tags]))
# number of unused jobs
not_used_jobs = [job for job in self.jobs if "case group" not in job]
not_used_jobs = [job for job in self.jobs if 'case group' not in job]
if not_used_jobs:
console_log("{} jobs not used. Please check if you define too much jobs".format(len(not_used_jobs)), "O")
console_log('{} jobs not used. Please check if you define too much jobs'.format(len(not_used_jobs)), 'O')
for job in not_used_jobs:
console_log("\t{}".format(job["name"]), "O")
console_log('\t{}'.format(job['name']), 'O')
# failures
if failed_to_assign:
console_log("Too many test cases vs jobs to run. "
"Please increase parallel count in tools/ci/config/target-test.yml "
"for jobs with specific tags:", "R")
console_log('Too many test cases vs jobs to run. '
'Please increase parallel count in tools/ci/config/target-test.yml '
'for jobs with specific tags:', 'R')
failed_group_count = self._count_groups_by_keys(failed_to_assign)
for tags in failed_group_count:
console_log("\t{}: {}".format(tags, failed_group_count[tags]), "R")
raise RuntimeError("Failed to assign test case to CI jobs")
console_log('\t{}: {}'.format(tags, failed_group_count[tags]), 'R')
raise RuntimeError('Failed to assign test case to CI jobs')
def output_configs(self, output_path):
"""

30
tools/ci/python_packages/tiny_test_fw/Utility/CaseConfig.py
View File

@@ -141,9 +141,9 @@ def filter_test_cases(test_methods, case_filter):
class Parser(object):
DEFAULT_CONFIG = {
"TestConfig": dict(),
"Filter": dict(),
"CaseConfig": [{"extra_data": None}],
'TestConfig': dict(),
'Filter': dict(),
'CaseConfig': [{'extra_data': None}],
}
@classmethod
@@ -156,7 +156,7 @@ class Parser(object):
"""
configs = cls.DEFAULT_CONFIG.copy()
if config_file:
with open(config_file, "r") as f:
with open(config_file, 'r') as f:
configs.update(yaml.load(f, Loader=Loader))
return configs
@@ -170,8 +170,8 @@ class Parser(object):
"""
output = dict()
for key in overwrite:
module = importlib.import_module(overwrite[key]["package"])
output[key] = module.__getattribute__(overwrite[key]["class"])
module = importlib.import_module(overwrite[key]['package'])
output[key] = module.__getattribute__(overwrite[key]['class'])
return output
@classmethod
@@ -185,10 +185,10 @@ class Parser(object):
"""
configs = cls.parse_config_file(config_file)
test_case_list = []
for _config in configs["CaseConfig"]:
_filter = configs["Filter"].copy()
_overwrite = cls.handle_overwrite_args(_config.pop("overwrite", dict()))
_extra_data = _config.pop("extra_data", None)
for _config in configs['CaseConfig']:
_filter = configs['Filter'].copy()
_overwrite = cls.handle_overwrite_args(_config.pop('overwrite', dict()))
_extra_data = _config.pop('extra_data', None)
_filter.update(_config)
# Try get target from yml
@@ -222,8 +222,8 @@ class Generator(object):
def __init__(self):
self.default_config = {
"TestConfig": dict(),
"Filter": dict(),
'TestConfig': dict(),
'Filter': dict(),
}
def set_default_configs(self, test_config, case_filter):
@@ -232,7 +232,7 @@ class Generator(object):
:param case_filter: "Filter" value
:return: None
"""
self.default_config = {"TestConfig": test_config, "Filter": case_filter}
self.default_config = {'TestConfig': test_config, 'Filter': case_filter}
def generate_config(self, case_configs, output_file):
"""
@@ -241,6 +241,6 @@ class Generator(object):
:return: None
"""
config = self.default_config.copy()
config.update({"CaseConfig": case_configs})
with open(output_file, "w") as f:
config.update({'CaseConfig': case_configs})
with open(output_file, 'w') as f:
yaml.dump(config, f)

16
tools/ci/python_packages/tiny_test_fw/Utility/GitlabCIJob.py
View File

@@ -26,8 +26,8 @@ class Job(dict):
"""
def __init__(self, job, job_name):
super(Job, self).__init__(job)
self["name"] = job_name
self.tags = set(self["tags"])
self['name'] = job_name
self.tags = set(self['tags'])
def match_group(self, group):
"""
@@ -38,7 +38,7 @@ class Job(dict):
:return: True or False
"""
match_result = False
if "case group" not in self and group.ci_job_match_keys == self.tags:
if 'case group' not in self and group.ci_job_match_keys == self.tags:
# group not assigned and all tags match
match_result = True
return match_result
@@ -49,7 +49,7 @@ class Job(dict):
:param group: the case group to assign
"""
self["case group"] = group
self['case group'] = group
def output_config(self, file_path):
"""
@@ -59,7 +59,7 @@ class Job(dict):
:param file_path: output file path
:return: None
"""
file_name = os.path.join(file_path, self["name"] + ".yml")
if "case group" in self:
with open(file_name, "w") as f:
yaml.safe_dump(self["case group"].output(), f, encoding='utf-8', default_flow_style=False)
file_name = os.path.join(file_path, self['name'] + '.yml')
if 'case group' in self:
with open(file_name, 'w') as f:
yaml.safe_dump(self['case group'].output(), f, encoding='utf-8', default_flow_style=False)

16
tools/ci/python_packages/tiny_test_fw/Utility/SearchCases.py
View File

@@ -13,23 +13,23 @@
# limitations under the License.
""" search test cases from a given file or path """
import os
import fnmatch
import types
import copy
import fnmatch
import os
import types
from . import load_source
class Search(object):
TEST_CASE_FILE_PATTERN = "*_test.py"
TEST_CASE_FILE_PATTERN = '*_test.py'
SUPPORT_REPLICATE_CASES_KEY = ['target']
@classmethod
def _search_cases_from_file(cls, file_name):
""" get test cases from test case .py file """
print("Try to get cases from: " + file_name)
print('Try to get cases from: ' + file_name)
test_functions = []
try:
mod = load_source(file_name)
@@ -42,14 +42,14 @@ class Search(object):
except AttributeError:
continue
except ImportError as e:
print("ImportError: \r\n\tFile:" + file_name + "\r\n\tError:" + str(e))
print('ImportError: \r\n\tFile:' + file_name + '\r\n\tError:' + str(e))
test_functions_out = []
for case in test_functions:
test_functions_out += cls.replicate_case(case)
for i, test_function in enumerate(test_functions_out):
print("\t{}. {} <{}>".format(i + 1, test_function.case_info["name"], test_function.case_info["target"]))
print('\t{}. {} <{}>'.format(i + 1, test_function.case_info['name'], test_function.case_info['target']))
test_function.case_info['app_dir'] = os.path.dirname(file_name)
return test_functions_out
@@ -58,7 +58,7 @@ class Search(object):
""" search all test case files recursively of a path """
if not os.path.exists(test_case):
raise OSError("test case path not exist")
raise OSError('test case path not exist')
if os.path.isdir(test_case):
test_case_files = []
for root, _, file_names in os.walk(test_case):

45
tools/ci/python_packages/tiny_test_fw/Utility/__init__.py
View File

@@ -1,4 +1,5 @@
from __future__ import print_function
import os.path
import sys
import time
@@ -7,35 +8,35 @@ import traceback
from .. import Env
_COLOR_CODES = {
"white": u'\033[0m',
"red": u'\033[31m',
"green": u'\033[32m',
"orange": u'\033[33m',
"blue": u'\033[34m',
"purple": u'\033[35m',
"W": u'\033[0m',
"R": u'\033[31m',
"G": u'\033[32m',
"O": u'\033[33m',
"B": u'\033[34m',
"P": u'\033[35m'
'white': u'\033[0m',
'red': u'\033[31m',
'green': u'\033[32m',
'orange': u'\033[33m',
'blue': u'\033[34m',
'purple': u'\033[35m',
'W': u'\033[0m',
'R': u'\033[31m',
'G': u'\033[32m',
'O': u'\033[33m',
'B': u'\033[34m',
'P': u'\033[35m'
}
def _get_log_file_name():
if Env.Env.CURRENT_LOG_FOLDER:
file_name = os.path.join(Env.Env.CURRENT_LOG_FOLDER, "console.log")
file_name = os.path.join(Env.Env.CURRENT_LOG_FOLDER, 'console.log')
else:
raise OSError("env log folder does not exist, will not save to log file")
raise OSError('env log folder does not exist, will not save to log file')
return file_name
def format_timestamp():
ts = time.time()
return "{}:{}".format(time.strftime("%m-%d %H:%M:%S", time.localtime(ts)), str(ts % 1)[2:5])
return '{}:{}'.format(time.strftime('%m-%d %H:%M:%S', time.localtime(ts)), str(ts % 1)[2:5])
def console_log(data, color="white", end="\n"):
def console_log(data, color='white', end='\n'):
"""
log data to console.
(if not flush console log, Gitlab-CI won't update logs during job execution)
@@ -44,19 +45,19 @@ def console_log(data, color="white", end="\n"):
:param color: color
"""
if color not in _COLOR_CODES:
color = "white"
color = 'white'
color_codes = _COLOR_CODES[color]
if isinstance(data, type(b'')):
data = data.decode('utf-8', 'replace')
print(color_codes + data, end=end)
if color not in ["white", "W"]:
if color not in ['white', 'W']:
# reset color to white for later logs
print(_COLOR_CODES["white"] + u"\r")
print(_COLOR_CODES['white'] + u'\r')
sys.stdout.flush()
log_data = "[{}] ".format(format_timestamp()) + data
log_data = '[{}] '.format(format_timestamp()) + data
try:
log_file = _get_log_file_name()
with open(log_file, "a+") as f:
with open(log_file, 'a+') as f:
f.write(log_data + end)
except OSError:
pass
@@ -108,4 +109,4 @@ def handle_unexpected_exception(junit_test_case, exception):
traceback.print_exc()
# AssertionError caused by an 'assert' statement has an empty string as its 'str' form
e_str = str(exception) if str(exception) else repr(exception)
junit_test_case.add_failure_info("Unexpected exception: {}\n{}".format(e_str, traceback.format_exc()))
junit_test_case.add_failure_info('Unexpected exception: {}\n{}'.format(e_str, traceback.format_exc()))

20
tools/ci/python_packages/tiny_test_fw/bin/Runner.py
View File

@@ -21,13 +21,13 @@ Command line interface to run test cases from a given path.
Use ``python Runner.py test_case_path -c config_file -e env_config_file`` to run test cases.
"""
import argparse
import os
import sys
import argparse
import threading
from tiny_test_fw import TinyFW
from tiny_test_fw.Utility import SearchCases, CaseConfig
from tiny_test_fw.Utility import CaseConfig, SearchCases
class Runner(threading.Thread):
@@ -43,7 +43,7 @@ class Runner(threading.Thread):
if case_config:
test_suite_name = os.path.splitext(os.path.basename(case_config))[0]
else:
test_suite_name = "TestRunner"
test_suite_name = 'TestRunner'
TinyFW.set_default_config(env_config_file=env_config_file, test_suite_name=test_suite_name)
test_methods = SearchCases.Search.search_test_cases(test_case_paths)
self.test_cases = CaseConfig.Parser.apply_config(test_methods, case_config)
@@ -60,12 +60,12 @@ class Runner(threading.Thread):
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument("test_cases", nargs='+',
help="test case folders or files")
parser.add_argument("--case_config", "-c", default=None,
help="case filter/config file")
parser.add_argument("--env_config_file", "-e", default=None,
help="test env config file")
parser.add_argument('test_cases', nargs='+',
help='test case folders or files')
parser.add_argument('--case_config', '-c', default=None,
help='case filter/config file')
parser.add_argument('--env_config_file', '-e', default=None,
help='test env config file')
args = parser.parse_args()
test_cases = [os.path.join(os.getenv('IDF_PATH'), path) if not os.path.isabs(path) else path for path in args.test_cases]
@@ -78,7 +78,7 @@ if __name__ == '__main__':
if not runner.is_alive():
break
except KeyboardInterrupt:
print("exit by Ctrl-C")
print('exit by Ctrl-C')
break
if not runner.get_test_result():
sys.exit(1)

18
tools/ci/python_packages/tiny_test_fw/bin/example.py
View File

@@ -19,7 +19,7 @@ import ttfw_idf
from tiny_test_fw import TinyFW
@ttfw_idf.idf_example_test(env_tag="Example_WIFI")
@ttfw_idf.idf_example_test(env_tag='Example_WIFI')
def test_examples_protocol_https_request(env, extra_data):
"""
steps: |
@@ -27,17 +27,17 @@ def test_examples_protocol_https_request(env, extra_data):
2. connect to www.howsmyssl.com:443
3. send http request
"""
dut1 = env.get_dut("https_request", "examples/protocols/https_request", dut_class=ttfw_idf.ESP32DUT)
dut1 = env.get_dut('https_request', 'examples/protocols/https_request', dut_class=ttfw_idf.ESP32DUT)
dut1.start_app()
dut1.expect(re.compile(r"Connecting to www.howsmyssl.com:443"), timeout=30)
dut1.expect("Performing the SSL/TLS handshake")
dut1.expect("Certificate verified.", timeout=15)
dut1.expect_all(re.compile(r"Cipher suite is TLS-ECDHE-RSA-WITH-AES-128-GCM-SHA256"),
"Reading HTTP response",
dut1.expect(re.compile(r'Connecting to www.howsmyssl.com:443'), timeout=30)
dut1.expect('Performing the SSL/TLS handshake')
dut1.expect('Certificate verified.', timeout=15)
dut1.expect_all(re.compile(r'Cipher suite is TLS-ECDHE-RSA-WITH-AES-128-GCM-SHA256'),
'Reading HTTP response',
timeout=20)
dut1.expect(re.compile(r"Completed (\d) requests"))
dut1.expect(re.compile(r'Completed (\d) requests'))
if __name__ == '__main__':
TinyFW.set_default_config(env_config_file="EnvConfigTemplate.yml", dut=ttfw_idf.IDFDUT)
TinyFW.set_default_config(env_config_file='EnvConfigTemplate.yml', dut=ttfw_idf.IDFDUT)
test_examples_protocol_https_request()

1
tools/ci/python_packages/tiny_test_fw/docs/conf.py
View File

@@ -18,6 +18,7 @@
#
import os
import sys
sys.path.insert(0, os.path.abspath('..'))
# import sphinx_rtd_theme

14
tools/ci/python_packages/ttfw_idf/CIScanTests.py
View File

@@ -7,9 +7,9 @@ from collections import defaultdict
from copy import deepcopy
from find_apps import find_apps
from find_build_apps import BUILD_SYSTEMS, BUILD_SYSTEM_CMAKE
from find_build_apps import BUILD_SYSTEM_CMAKE, BUILD_SYSTEMS
from idf_py_actions.constants import PREVIEW_TARGETS, SUPPORTED_TARGETS
from ttfw_idf.IDFAssignTest import ExampleAssignTest, TestAppsAssignTest
from idf_py_actions.constants import SUPPORTED_TARGETS, PREVIEW_TARGETS
TEST_LABELS = {
'example_test': 'BOT_LABEL_EXAMPLE_TEST',
@@ -73,15 +73,15 @@ def main():
default=BUILD_SYSTEM_CMAKE)
parser.add_argument('-c', '--ci-config-file',
required=True,
help="gitlab ci config target-test file")
help='gitlab ci config target-test file')
parser.add_argument('-o', '--output-path',
required=True,
help="output path of the scan result")
parser.add_argument("--exclude", nargs="*",
help='output path of the scan result')
parser.add_argument('--exclude', nargs='*',
help='Ignore specified directory. Can be used multiple times.')
parser.add_argument('--preserve', action="store_true",
parser.add_argument('--preserve', action='store_true',
help='add this flag to preserve artifacts for all apps')
parser.add_argument('--build-all', action="store_true",
parser.add_argument('--build-all', action='store_true',
help='add this flag to build all apps')
args = parser.parse_args()

4
tools/ci/python_packages/ttfw_idf/DebugUtils.py
View File

@@ -13,12 +13,12 @@
# limitations under the License.
from __future__ import unicode_literals
from io import open
import logging
from io import open
import pexpect
import pygdbmi.gdbcontroller
from tiny_test_fw import Utility
try:

125
tools/ci/python_packages/ttfw_idf/IDFApp.py
View File

@@ -22,7 +22,8 @@ import sys
from abc import abstractmethod
from tiny_test_fw import App
from .IDFAssignTest import ExampleGroup, TestAppsGroup, UnitTestGroup, IDFCaseGroup, ComponentUTGroup
from .IDFAssignTest import ComponentUTGroup, ExampleGroup, IDFCaseGroup, TestAppsGroup, UnitTestGroup
try:
import gitlab_api
@@ -36,8 +37,8 @@ def parse_encrypted_flag(args, offs, binary):
# If the current entry is a partition, we have to check whether it is
# the one we are looking for or not
try:
if (entry["offset"], entry["file"]) == (offs, binary):
return entry["encrypted"] == "true"
if (entry['offset'], entry['file']) == (offs, binary):
return entry['encrypted'] == 'true'
except (TypeError, KeyError):
# TypeError occurs if the entry is a list, which is possible in JSON
# data structure.
@@ -58,12 +59,12 @@ def parse_flash_settings(path, default_encryption=False):
# The following list only contains the files that need encryption
encrypt_files = []
if file_name == "flasher_args.json":
if file_name == 'flasher_args.json':
# CMake version using build metadata file
with open(path, "r") as f:
with open(path, 'r') as f:
args = json.load(f)
for (offs, binary) in args["flash_files"].items():
for (offs, binary) in args['flash_files'].items():
if offs:
flash_files.append((offs, binary))
encrypted = parse_encrypted_flag(args, offs, binary)
@@ -73,15 +74,15 @@ def parse_flash_settings(path, default_encryption=False):
if (encrypted is None and default_encryption) or encrypted:
encrypt_files.append((offs, binary))
flash_settings = args["flash_settings"]
app_name = os.path.splitext(args["app"]["file"])[0]
flash_settings = args['flash_settings']
app_name = os.path.splitext(args['app']['file'])[0]
else:
# GNU Make version uses download.config arguments file
with open(path, "r") as f:
args = f.readlines()[-1].split(" ")
with open(path, 'r') as f:
args = f.readlines()[-1].split(' ')
flash_settings = {}
for idx in range(0, len(args), 2): # process arguments in pairs
if args[idx].startswith("--"):
if args[idx].startswith('--'):
# strip the -- from the command line argument
flash_settings[args[idx][2:]] = args[idx + 1]
else:
@@ -92,7 +93,7 @@ def parse_flash_settings(path, default_encryption=False):
encrypt_files = flash_files
# we can only guess app name in download.config.
for p in flash_files:
if not os.path.dirname(p[1]) and "partition" not in p[1]:
if not os.path.dirname(p[1]) and 'partition' not in p[1]:
# app bin usually in the same dir with download.config and it's not partition table
app_name = os.path.splitext(p[1])[0]
break
@@ -107,9 +108,9 @@ class Artifacts(object):
# at least one of app_path or config_name is not None. otherwise we can't match artifact
assert app_path or config_name
assert os.path.exists(artifact_index_file)
self.gitlab_inst = gitlab_api.Gitlab(os.getenv("CI_PROJECT_ID"))
self.gitlab_inst = gitlab_api.Gitlab(os.getenv('CI_PROJECT_ID'))
self.dest_root_path = dest_root_path
with open(artifact_index_file, "r") as f:
with open(artifact_index_file, 'r') as f:
artifact_index = json.load(f)
self.artifact_info = self._find_artifact(artifact_index, app_path, config_name, target)
@@ -120,11 +121,11 @@ class Artifacts(object):
if app_path:
# We use endswith here to avoid issue like:
# examples_protocols_mqtt_ws but return a examples_protocols_mqtt_wss failure
match_result = artifact_info["app_dir"].endswith(app_path)
match_result = artifact_info['app_dir'].endswith(app_path)
if config_name:
match_result = match_result and config_name == artifact_info["config"]
match_result = match_result and config_name == artifact_info['config']
if target:
match_result = match_result and target == artifact_info["target"]
match_result = match_result and target == artifact_info['target']
if match_result:
ret = artifact_info
break
@@ -134,15 +135,15 @@ class Artifacts(object):
def _get_app_base_path(self):
if self.artifact_info:
return os.path.join(self.artifact_info["work_dir"], self.artifact_info["build_dir"])
return os.path.join(self.artifact_info['work_dir'], self.artifact_info['build_dir'])
else:
return None
def _get_flash_arg_file(self, base_path, job_id):
if self.artifact_info["build_system"] == "cmake":
flash_arg_file = os.path.join(base_path, "flasher_args.json")
if self.artifact_info['build_system'] == 'cmake':
flash_arg_file = os.path.join(base_path, 'flasher_args.json')
else:
flash_arg_file = os.path.join(base_path, "download.config")
flash_arg_file = os.path.join(base_path, 'download.config')
self.gitlab_inst.download_artifact(job_id, [flash_arg_file], self.dest_root_path)
return flash_arg_file
@@ -152,19 +153,19 @@ class Artifacts(object):
# files also appear in the first list
flash_files, _, _, app_name = parse_flash_settings(os.path.join(self.dest_root_path, flash_arg_file))
artifact_files = [os.path.join(base_path, p[1]) for p in flash_files]
artifact_files.append(os.path.join(base_path, app_name + ".elf"))
artifact_files.append(os.path.join(base_path, app_name + '.elf'))
self.gitlab_inst.download_artifact(job_id, artifact_files, self.dest_root_path)
def _download_sdkconfig_file(self, base_path, job_id):
self.gitlab_inst.download_artifact(job_id, [os.path.join(os.path.dirname(base_path), "sdkconfig")],
self.gitlab_inst.download_artifact(job_id, [os.path.join(os.path.dirname(base_path), 'sdkconfig')],
self.dest_root_path)
def download_artifacts(self):
if not self.artifact_info:
return None
base_path = self._get_app_base_path()
job_id = self.artifact_info["ci_job_id"]
job_id = self.artifact_info['ci_job_id']
# 1. download flash args file
flash_arg_file = self._get_flash_arg_file(base_path, job_id)
@@ -177,15 +178,15 @@ class Artifacts(object):
def download_artifact_files(self, file_names):
if self.artifact_info:
base_path = os.path.join(self.artifact_info["work_dir"], self.artifact_info["build_dir"])
job_id = self.artifact_info["ci_job_id"]
base_path = os.path.join(self.artifact_info['work_dir'], self.artifact_info['build_dir'])
job_id = self.artifact_info['ci_job_id']
# download all binary files
artifact_files = [os.path.join(base_path, fn) for fn in file_names]
self.gitlab_inst.download_artifact(job_id, artifact_files, self.dest_root_path)
# download sdkconfig file
self.gitlab_inst.download_artifact(job_id, [os.path.join(os.path.dirname(base_path), "sdkconfig")],
self.gitlab_inst.download_artifact(job_id, [os.path.join(os.path.dirname(base_path), 'sdkconfig')],
self.dest_root_path)
else:
base_path = None
@@ -197,13 +198,13 @@ class UnitTestArtifacts(Artifacts):
def _get_app_base_path(self):
if self.artifact_info:
output_dir = self.BUILDS_DIR_RE.sub('output/', self.artifact_info["build_dir"])
return os.path.join(self.artifact_info["app_dir"], output_dir)
output_dir = self.BUILDS_DIR_RE.sub('output/', self.artifact_info['build_dir'])
return os.path.join(self.artifact_info['app_dir'], output_dir)
else:
return None
def _download_sdkconfig_file(self, base_path, job_id):
self.gitlab_inst.download_artifact(job_id, [os.path.join(base_path, "sdkconfig")], self.dest_root_path)
self.gitlab_inst.download_artifact(job_id, [os.path.join(base_path, 'sdkconfig')], self.dest_root_path)
class IDFApp(App.BaseApp):
@@ -212,8 +213,8 @@ class IDFApp(App.BaseApp):
idf applications should inherent from this class and overwrite method get_binary_path.
"""
IDF_DOWNLOAD_CONFIG_FILE = "download.config"
IDF_FLASH_ARGS_FILE = "flasher_args.json"
IDF_DOWNLOAD_CONFIG_FILE = 'download.config'
IDF_FLASH_ARGS_FILE = 'flasher_args.json'
def __init__(self, app_path, config_name=None, target=None, case_group=IDFCaseGroup, artifact_cls=Artifacts):
super(IDFApp, self).__init__(app_path)
@@ -229,11 +230,11 @@ class IDFApp(App.BaseApp):
assert os.path.exists(self.binary_path)
if self.IDF_DOWNLOAD_CONFIG_FILE not in os.listdir(self.binary_path):
if self.IDF_FLASH_ARGS_FILE not in os.listdir(self.binary_path):
msg = ("Neither {} nor {} exists. "
msg = ('Neither {} nor {} exists. '
"Try to run 'make print_flash_cmd | tail -n 1 > {}/{}' "
"or 'idf.py build' "
"for resolving the issue."
"").format(self.IDF_DOWNLOAD_CONFIG_FILE, self.IDF_FLASH_ARGS_FILE,
'for resolving the issue.'
'').format(self.IDF_DOWNLOAD_CONFIG_FILE, self.IDF_FLASH_ARGS_FILE,
self.binary_path, self.IDF_DOWNLOAD_CONFIG_FILE)
raise AssertionError(msg)
@@ -252,7 +253,7 @@ class IDFApp(App.BaseApp):
@classmethod
def get_sdk_path(cls): # type: () -> str
idf_path = os.getenv("IDF_PATH")
idf_path = os.getenv('IDF_PATH')
assert idf_path
assert os.path.exists(idf_path)
return idf_path
@@ -263,7 +264,7 @@ class IDFApp(App.BaseApp):
Note: could be overwritten by a derived class to provide other locations or order
"""
return [os.path.join(self.binary_path, "sdkconfig"), os.path.join(self.binary_path, "..", "sdkconfig")]
return [os.path.join(self.binary_path, 'sdkconfig'), os.path.join(self.binary_path, '..', 'sdkconfig')]
def get_sdkconfig(self):
"""
@@ -306,13 +307,13 @@ class IDFApp(App.BaseApp):
if path:
return os.path.join(self.idf_path, path)
else:
raise OSError("Failed to get binary for {}".format(self))
raise OSError('Failed to get binary for {}'.format(self))
def _get_elf_file_path(self):
ret = ""
ret = ''
file_names = os.listdir(self.binary_path)
for fn in file_names:
if os.path.splitext(fn)[1] == ".elf":
if os.path.splitext(fn)[1] == '.elf':
ret = os.path.join(self.binary_path, fn)
return ret
@@ -343,14 +344,14 @@ class IDFApp(App.BaseApp):
# a default encrpytion flag: the macro
# CONFIG_SECURE_FLASH_ENCRYPTION_MODE_DEVELOPMENT
sdkconfig_dict = self.get_sdkconfig()
default_encryption = "CONFIG_SECURE_FLASH_ENCRYPTION_MODE_DEVELOPMENT" in sdkconfig_dict
default_encryption = 'CONFIG_SECURE_FLASH_ENCRYPTION_MODE_DEVELOPMENT' in sdkconfig_dict
flash_files, encrypt_files, flash_settings, _ = parse_flash_settings(path, default_encryption)
# Flash setting "encrypt" only and only if all the files to flash
# must be encrypted. Else, this parameter should be False.
# All files must be encrypted is both file lists are the same
flash_settings["encrypt"] = sorted(flash_files) == sorted(encrypt_files)
flash_settings['encrypt'] = sorted(flash_files) == sorted(encrypt_files)
return self._int_offs_abs_paths(flash_files), self._int_offs_abs_paths(encrypt_files), flash_settings
@@ -363,9 +364,9 @@ class IDFApp(App.BaseApp):
(Called from constructor)
"""
partition_tool = os.path.join(self.idf_path,
"components",
"partition_table",
"gen_esp32part.py")
'components',
'partition_table',
'gen_esp32part.py')
assert os.path.exists(partition_tool)
errors = []
@@ -393,18 +394,18 @@ class IDFApp(App.BaseApp):
p,
os.linesep,
msg) for p, msg in errors])
raise ValueError("No partition table found for IDF binary path: {}{}{}".format(self.binary_path,
raise ValueError('No partition table found for IDF binary path: {}{}{}'.format(self.binary_path,
os.linesep,
traceback_msg))
partition_table = dict()
for line in raw_data.splitlines():
if line[0] != "#":
if line[0] != '#':
try:
_name, _type, _subtype, _offset, _size, _flags = line.split(",")
if _size[-1] == "K":
_name, _type, _subtype, _offset, _size, _flags = line.split(',')
if _size[-1] == 'K':
_size = int(_size[:-1]) * 1024
elif _size[-1] == "M":
elif _size[-1] == 'M':
_size = int(_size[:-1]) * 1024 * 1024
else:
_size = int(_size)
@@ -412,11 +413,11 @@ class IDFApp(App.BaseApp):
except ValueError:
continue
partition_table[_name] = {
"type": _type,
"subtype": _subtype,
"offset": _offset,
"size": _size,
"flags": _flags
'type': _type,
'subtype': _subtype,
'offset': _offset,
'size': _size,
'flags': _flags
}
return partition_table
@@ -444,11 +445,11 @@ class Example(IDFApp):
"""
overrides the parent method to provide exact path of sdkconfig for example tests
"""
return [os.path.join(self.binary_path, "..", "sdkconfig")]
return [os.path.join(self.binary_path, '..', 'sdkconfig')]
def _try_get_binary_from_local_fs(self):
# build folder of example path
path = os.path.join(self.idf_path, self.app_path, "build")
path = os.path.join(self.idf_path, self.app_path, 'build')
if os.path.exists(path):
return path
@@ -456,11 +457,11 @@ class Example(IDFApp):
# Path format: $IDF_PATH/build_examples/app_path_with_underscores/config/target
# (see tools/ci/build_examples.sh)
# For example: $IDF_PATH/build_examples/examples_get-started_blink/default/esp32
app_path_underscored = self.app_path.replace(os.path.sep, "_")
app_path_underscored = self.app_path.replace(os.path.sep, '_')
example_path = os.path.join(self.idf_path, self.case_group.LOCAL_BUILD_DIR)
for dirpath in os.listdir(example_path):
if os.path.basename(dirpath) == app_path_underscored:
path = os.path.join(example_path, dirpath, self.config_name, self.target, "build")
path = os.path.join(example_path, dirpath, self.config_name, self.target, 'build')
if os.path.exists(path):
return path
else:
@@ -476,18 +477,18 @@ class UT(IDFApp):
super(UT, self).__init__(app_path, config_name, target, case_group, artifacts_cls)
def _try_get_binary_from_local_fs(self):
path = os.path.join(self.idf_path, self.app_path, "build")
path = os.path.join(self.idf_path, self.app_path, 'build')
if os.path.exists(path):
return path
# first try to get from build folder of unit-test-app
path = os.path.join(self.idf_path, "tools", "unit-test-app", "build")
path = os.path.join(self.idf_path, 'tools', 'unit-test-app', 'build')
if os.path.exists(path):
# found, use bin in build path
return path
# ``build_unit_test.sh`` will copy binary to output folder
path = os.path.join(self.idf_path, "tools", "unit-test-app", "output", self.target, self.config_name)
path = os.path.join(self.idf_path, 'tools', 'unit-test-app', 'output', self.target, self.config_name)
if os.path.exists(path):
return path

2
tools/ci/python_packages/ttfw_idf/IDFAssignTest.py
View File

@@ -18,7 +18,7 @@ import gitlab_api
from tiny_test_fw.Utility import CIAssignTest
try:
from idf_py_actions.constants import SUPPORTED_TARGETS, PREVIEW_TARGETS
from idf_py_actions.constants import PREVIEW_TARGETS, SUPPORTED_TARGETS
except ImportError:
SUPPORTED_TARGETS = []
PREVIEW_TARGETS = []

167
tools/ci/python_packages/ttfw_idf/IDFDUT.py
View File

@@ -13,14 +13,15 @@
# limitations under the License.
""" DUT for IDF applications """
import functools
import os
import os.path
import sys
import re
import functools
import tempfile
import subprocess
import sys
import tempfile
import time
import pexpect
# python2 and python3 queue package name is different
@@ -29,18 +30,16 @@ try:
except ImportError:
import queue as _queue
from serial.tools import list_ports
from tiny_test_fw import DUT, Utility
try:
import esptool
except ImportError: # cheat and use IDF's copy of esptool if available
idf_path = os.getenv("IDF_PATH")
idf_path = os.getenv('IDF_PATH')
if not idf_path or not os.path.exists(idf_path):
raise
sys.path.insert(0, os.path.join(idf_path, "components", "esptool_py", "esptool"))
sys.path.insert(0, os.path.join(idf_path, 'components', 'esptool_py', 'esptool'))
import esptool
@@ -54,14 +53,14 @@ class IDFDUTException(RuntimeError):
class IDFRecvThread(DUT.RecvThread):
PERFORMANCE_PATTERN = re.compile(r"\[Performance]\[(\w+)]: ([^\r\n]+)\r?\n")
PERFORMANCE_PATTERN = re.compile(r'\[Performance]\[(\w+)]: ([^\r\n]+)\r?\n')
EXCEPTION_PATTERNS = [
re.compile(r"(Guru Meditation Error: Core\s+\d panic'ed \([\w].*?\))"),
re.compile(r"(abort\(\) was called at PC 0x[a-fA-F\d]{8} on core \d)"),
re.compile(r"(rst 0x\d+ \(TG\dWDT_SYS_RESET|TGWDT_CPU_RESET\))")
re.compile(r'(abort\(\) was called at PC 0x[a-fA-F\d]{8} on core \d)'),
re.compile(r'(rst 0x\d+ \(TG\dWDT_SYS_RESET|TGWDT_CPU_RESET\))')
]
BACKTRACE_PATTERN = re.compile(r"Backtrace:((\s(0x[0-9a-f]{8}):0x[0-9a-f]{8})+)")
BACKTRACE_ADDRESS_PATTERN = re.compile(r"(0x[0-9a-f]{8}):0x[0-9a-f]{8}")
BACKTRACE_PATTERN = re.compile(r'Backtrace:((\s(0x[0-9a-f]{8}):0x[0-9a-f]{8})+)')
BACKTRACE_ADDRESS_PATTERN = re.compile(r'(0x[0-9a-f]{8}):0x[0-9a-f]{8}')
def __init__(self, read, dut):
super(IDFRecvThread, self).__init__(read, dut)
@@ -71,8 +70,8 @@ class IDFRecvThread(DUT.RecvThread):
def collect_performance(self, comp_data):
matches = self.PERFORMANCE_PATTERN.findall(comp_data)
for match in matches:
Utility.console_log("[Performance][{}]: {}".format(match[0], match[1]),
color="orange")
Utility.console_log('[Performance][{}]: {}'.format(match[0], match[1]),
color='orange')
self.performance_items.put((match[0], match[1]))
def detect_exception(self, comp_data):
@@ -83,7 +82,7 @@ class IDFRecvThread(DUT.RecvThread):
if match:
start = match.end()
self.exceptions.put(match.group(0))
Utility.console_log("[Exception]: {}".format(match.group(0)), color="red")
Utility.console_log('[Exception]: {}'.format(match.group(0)), color='red')
else:
break
@@ -93,18 +92,18 @@ class IDFRecvThread(DUT.RecvThread):
match = self.BACKTRACE_PATTERN.search(comp_data, pos=start)
if match:
start = match.end()
Utility.console_log("[Backtrace]:{}".format(match.group(1)), color="red")
Utility.console_log('[Backtrace]:{}'.format(match.group(1)), color='red')
# translate backtrace
addresses = self.BACKTRACE_ADDRESS_PATTERN.findall(match.group(1))
translated_backtrace = ""
translated_backtrace = ''
for addr in addresses:
ret = self.dut.lookup_pc_address(addr)
if ret:
translated_backtrace += ret + "\n"
translated_backtrace += ret + '\n'
if translated_backtrace:
Utility.console_log("Translated backtrace\n:" + translated_backtrace, color="yellow")
Utility.console_log('Translated backtrace\n:' + translated_backtrace, color='yellow')
else:
Utility.console_log("Failed to translate backtrace", color="yellow")
Utility.console_log('Failed to translate backtrace', color='yellow')
else:
break
@@ -149,7 +148,7 @@ class IDFDUT(DUT.SerialDUT):
# /dev/ttyAMA0 port is listed in Raspberry Pi
# /dev/tty.Bluetooth-Incoming-Port port is listed in Mac
INVALID_PORT_PATTERN = re.compile(r"AMA|Bluetooth")
INVALID_PORT_PATTERN = re.compile(r'AMA|Bluetooth')
# if need to erase NVS partition in start app
ERASE_NVS = True
RECV_THREAD_CLS = IDFRecvThread
@@ -163,7 +162,7 @@ class IDFDUT(DUT.SerialDUT):
@classmethod
def _get_rom(cls):
raise NotImplementedError("This is an abstraction class, method not defined.")
raise NotImplementedError('This is an abstraction class, method not defined.')
@classmethod
def get_mac(cls, app, port):
@@ -200,7 +199,7 @@ class IDFDUT(DUT.SerialDUT):
# Otherwise overwrite it in ESP8266DUT
inst = esptool.ESPLoader.detect_chip(port)
if expected_rom_class and type(inst) != expected_rom_class:
raise RuntimeError("Target not expected")
raise RuntimeError('Target not expected')
return inst.read_mac() is not None, get_target_by_rom_class(type(inst))
except(esptool.FatalError, RuntimeError):
return False, None
@@ -227,7 +226,7 @@ class IDFDUT(DUT.SerialDUT):
# and encrypt_files contains the ones to flash encrypted.
flash_files = self.app.flash_files
encrypt_files = self.app.encrypt_files
encrypt = self.app.flash_settings.get("encrypt", False)
encrypt = self.app.flash_settings.get('encrypt', False)
if encrypt:
flash_files = encrypt_files
encrypt_files = []
@@ -236,12 +235,12 @@ class IDFDUT(DUT.SerialDUT):
for entry in flash_files
if entry not in encrypt_files]
flash_files = [(offs, open(path, "rb")) for (offs, path) in flash_files]
encrypt_files = [(offs, open(path, "rb")) for (offs, path) in encrypt_files]
flash_files = [(offs, open(path, 'rb')) for (offs, path) in flash_files]
encrypt_files = [(offs, open(path, 'rb')) for (offs, path) in encrypt_files]
if erase_nvs:
address = self.app.partition_table["nvs"]["offset"]
size = self.app.partition_table["nvs"]["size"]
address = self.app.partition_table['nvs']['offset']
size = self.app.partition_table['nvs']['size']
nvs_file = tempfile.TemporaryFile()
nvs_file.write(b'\xff' * size)
nvs_file.seek(0)
@@ -252,7 +251,7 @@ class IDFDUT(DUT.SerialDUT):
# Get the CONFIG_SECURE_FLASH_ENCRYPTION_MODE_DEVELOPMENT macro
# value. If it is set to True, then NVS is always encrypted.
sdkconfig_dict = self.app.get_sdkconfig()
macro_encryption = "CONFIG_SECURE_FLASH_ENCRYPTION_MODE_DEVELOPMENT" in sdkconfig_dict
macro_encryption = 'CONFIG_SECURE_FLASH_ENCRYPTION_MODE_DEVELOPMENT' in sdkconfig_dict
# If the macro is not enabled (plain text flash) or all files
# must be encrypted, add NVS to flash_files.
if not macro_encryption or encrypt:
@@ -270,9 +269,9 @@ class IDFDUT(DUT.SerialDUT):
# write_flash expects the parameter encrypt_files to be None and not
# an empty list, so perform the check here
flash_args = FlashArgs({
'flash_size': self.app.flash_settings["flash_size"],
'flash_mode': self.app.flash_settings["flash_mode"],
'flash_freq': self.app.flash_settings["flash_freq"],
'flash_size': self.app.flash_settings['flash_size'],
'flash_mode': self.app.flash_settings['flash_mode'],
'flash_freq': self.app.flash_settings['flash_freq'],
'addr_filename': flash_files,
'encrypt_files': encrypt_files or None,
'no_stub': False,
@@ -328,9 +327,9 @@ class IDFDUT(DUT.SerialDUT):
"""
raise NotImplementedError() # TODO: implement this
# address = self.app.partition_table[partition]["offset"]
size = self.app.partition_table[partition]["size"]
size = self.app.partition_table[partition]['size']
# TODO can use esp.erase_region() instead of this, I think
with open(".erase_partition.tmp", "wb") as f:
with open('.erase_partition.tmp', 'wb') as f:
f.write(chr(0xFF) * size)
@_uses_esptool
@@ -356,18 +355,18 @@ class IDFDUT(DUT.SerialDUT):
"""
if os.path.isabs(output_file) is False:
output_file = os.path.relpath(output_file, self.app.get_log_folder())
if "partition" in kwargs:
partition = self.app.partition_table[kwargs["partition"]]
_address = partition["offset"]
_size = partition["size"]
elif "address" in kwargs and "size" in kwargs:
_address = kwargs["address"]
_size = kwargs["size"]
if 'partition' in kwargs:
partition = self.app.partition_table[kwargs['partition']]
_address = partition['offset']
_size = partition['size']
elif 'address' in kwargs and 'size' in kwargs:
_address = kwargs['address']
_size = kwargs['size']
else:
raise IDFToolError("You must specify 'partition' or ('address' and 'size') to dump flash")
content = esp.read_flash(_address, _size)
with open(output_file, "wb") as f:
with open(output_file, 'wb') as f:
f.write(content)
@classmethod
@@ -405,9 +404,9 @@ class IDFDUT(DUT.SerialDUT):
return ports
def lookup_pc_address(self, pc_addr):
cmd = ["%saddr2line" % self.TOOLCHAIN_PREFIX,
"-pfiaC", "-e", self.app.elf_file, pc_addr]
ret = ""
cmd = ['%saddr2line' % self.TOOLCHAIN_PREFIX,
'-pfiaC', '-e', self.app.elf_file, pc_addr]
ret = ''
try:
translation = subprocess.check_output(cmd)
ret = translation.decode()
@@ -439,7 +438,7 @@ class IDFDUT(DUT.SerialDUT):
def stop_receive(self):
if self.receive_thread:
for name in ["performance_items", "exceptions"]:
for name in ['performance_items', 'exceptions']:
source_queue = getattr(self.receive_thread, name)
dest_queue = getattr(self, name)
self._queue_copy(source_queue, dest_queue)
@@ -447,7 +446,7 @@ class IDFDUT(DUT.SerialDUT):
def get_exceptions(self):
""" Get exceptions detected by DUT receive thread. """
return self._get_from_queue("exceptions")
return self._get_from_queue('exceptions')
def get_performance_items(self):
"""
@@ -456,18 +455,18 @@ class IDFDUT(DUT.SerialDUT):
:return: a list of performance items.
"""
return self._get_from_queue("performance_items")
return self._get_from_queue('performance_items')
def close(self):
super(IDFDUT, self).close()
if not self.allow_dut_exception and self.get_exceptions():
Utility.console_log("DUT exception detected on {}".format(self), color="red")
Utility.console_log('DUT exception detected on {}'.format(self), color='red')
raise IDFDUTException()
class ESP32DUT(IDFDUT):
TARGET = "esp32"
TOOLCHAIN_PREFIX = "xtensa-esp32-elf-"
TARGET = 'esp32'
TOOLCHAIN_PREFIX = 'xtensa-esp32-elf-'
@classmethod
def _get_rom(cls):
@@ -478,8 +477,8 @@ class ESP32DUT(IDFDUT):
class ESP32S2DUT(IDFDUT):
TARGET = "esp32s2"
TOOLCHAIN_PREFIX = "xtensa-esp32s2-elf-"
TARGET = 'esp32s2'
TOOLCHAIN_PREFIX = 'xtensa-esp32s2-elf-'
@classmethod
def _get_rom(cls):
@@ -490,8 +489,8 @@ class ESP32S2DUT(IDFDUT):
class ESP32C3DUT(IDFDUT):
TARGET = "esp32c3"
TOOLCHAIN_PREFIX = "riscv32-esp-elf-"
TARGET = 'esp32c3'
TOOLCHAIN_PREFIX = 'riscv32-esp-elf-'
@classmethod
def _get_rom(cls):
@@ -502,8 +501,8 @@ class ESP32C3DUT(IDFDUT):
class ESP8266DUT(IDFDUT):
TARGET = "esp8266"
TOOLCHAIN_PREFIX = "xtensa-lx106-elf-"
TARGET = 'esp8266'
TOOLCHAIN_PREFIX = 'xtensa-lx106-elf-'
@classmethod
def _get_rom(cls):
@@ -528,34 +527,34 @@ class IDFQEMUDUT(IDFDUT):
QEMU_SERIAL_PORT = 3334
def __init__(self, name, port, log_file, app, allow_dut_exception=False, **kwargs):
self.flash_image = tempfile.NamedTemporaryFile('rb+', suffix=".bin", prefix="qemu_flash_img")
self.flash_image = tempfile.NamedTemporaryFile('rb+', suffix='.bin', prefix='qemu_flash_img')
self.app = app
self.flash_size = 4 * 1024 * 1024
self._write_flash_img()
args = [
"qemu-system-xtensa",
"-nographic",
"-machine", self.TARGET,
"-drive", "file={},if=mtd,format=raw".format(self.flash_image.name),
"-nic", "user,model=open_eth",
"-serial", "tcp::{},server,nowait".format(self.QEMU_SERIAL_PORT),
"-S",
"-global driver=timer.esp32.timg,property=wdt_disable,value=true"]
'qemu-system-xtensa',
'-nographic',
'-machine', self.TARGET,
'-drive', 'file={},if=mtd,format=raw'.format(self.flash_image.name),
'-nic', 'user,model=open_eth',
'-serial', 'tcp::{},server,nowait'.format(self.QEMU_SERIAL_PORT),
'-S',
'-global driver=timer.esp32.timg,property=wdt_disable,value=true']
# TODO(IDF-1242): generate a temporary efuse binary, pass it to QEMU
if "QEMU_BIOS_PATH" in os.environ:
args += ["-L", os.environ["QEMU_BIOS_PATH"]]
if 'QEMU_BIOS_PATH' in os.environ:
args += ['-L', os.environ['QEMU_BIOS_PATH']]
self.qemu = pexpect.spawn(" ".join(args), timeout=self.DEFAULT_EXPECT_TIMEOUT)
self.qemu.expect_exact(b"(qemu)")
self.qemu = pexpect.spawn(' '.join(args), timeout=self.DEFAULT_EXPECT_TIMEOUT)
self.qemu.expect_exact(b'(qemu)')
super(IDFQEMUDUT, self).__init__(name, port, log_file, app, allow_dut_exception=allow_dut_exception, **kwargs)
def _write_flash_img(self):
self.flash_image.seek(0)
self.flash_image.write(b'\x00' * self.flash_size)
for offs, path in self.app.flash_files:
with open(path, "rb") as flash_file:
with open(path, 'rb') as flash_file:
contents = flash_file.read()
self.flash_image.seek(offs)
self.flash_image.write(contents)
@@ -568,7 +567,7 @@ class IDFQEMUDUT(IDFDUT):
@classmethod
def get_mac(cls, app, port):
# TODO(IDF-1242): get this from QEMU/efuse binary
return "11:22:33:44:55:66"
return '11:22:33:44:55:66'
@classmethod
def confirm_dut(cls, port, **kwargs):
@@ -577,30 +576,30 @@ class IDFQEMUDUT(IDFDUT):
def start_app(self, erase_nvs=ERASE_NVS):
# TODO: implement erase_nvs
# since the flash image is generated every time in the constructor, maybe this isn't needed...
self.qemu.sendline(b"cont\n")
self.qemu.expect_exact(b"(qemu)")
self.qemu.sendline(b'cont\n')
self.qemu.expect_exact(b'(qemu)')
def reset(self):
self.qemu.sendline(b"system_reset\n")
self.qemu.expect_exact(b"(qemu)")
self.qemu.sendline(b'system_reset\n')
self.qemu.expect_exact(b'(qemu)')
def erase_partition(self, partition):
raise NotImplementedError("method erase_partition not implemented")
raise NotImplementedError('method erase_partition not implemented')
def erase_flash(self):
raise NotImplementedError("method erase_flash not implemented")
raise NotImplementedError('method erase_flash not implemented')
def dump_flash(self, output_file, **kwargs):
raise NotImplementedError("method dump_flash not implemented")
raise NotImplementedError('method dump_flash not implemented')
@classmethod
def list_available_ports(cls):
return ["socket://localhost:{}".format(cls.QEMU_SERIAL_PORT)]
return ['socket://localhost:{}'.format(cls.QEMU_SERIAL_PORT)]
def close(self):
super(IDFQEMUDUT, self).close()
self.qemu.sendline(b"q\n")
self.qemu.expect_exact(b"(qemu)")
self.qemu.sendline(b'q\n')
self.qemu.expect_exact(b'(qemu)')
for _ in range(self.DEFAULT_EXPECT_TIMEOUT):
if not self.qemu.isalive():
break
@@ -610,5 +609,5 @@ class IDFQEMUDUT(IDFDUT):
class ESP32QEMUDUT(IDFQEMUDUT):
TARGET = "esp32"
TOOLCHAIN_PREFIX = "xtensa-esp32-elf-"
TARGET = 'esp32'
TOOLCHAIN_PREFIX = 'xtensa-esp32-elf-'

38
tools/ci/python_packages/ttfw_idf/__init__.py
View File

@@ -19,12 +19,12 @@ import re
from copy import deepcopy
import junit_xml
from tiny_test_fw import TinyFW, Utility
from .DebugUtils import OCDBackend, GDBBackend, CustomProcess # noqa: export DebugUtils for users
from .IDFApp import IDFApp, Example, LoadableElfTestApp, UT, TestApp, ComponentUTApp # noqa: export all Apps for users
from .IDFDUT import IDFDUT, ESP32DUT, ESP32S2DUT, ESP32C3DUT, ESP8266DUT, ESP32QEMUDUT # noqa: export DUTs for users
from .unity_test_parser import TestResults, TestFormat
from .DebugUtils import CustomProcess, GDBBackend, OCDBackend # noqa: export DebugUtils for users
from .IDFApp import UT, ComponentUTApp, Example, IDFApp, LoadableElfTestApp, TestApp # noqa: export all Apps for users
from .IDFDUT import ESP32C3DUT, ESP32DUT, ESP32QEMUDUT, ESP32S2DUT, ESP8266DUT, IDFDUT # noqa: export DUTs for users
from .unity_test_parser import TestFormat, TestResults
# pass TARGET_DUT_CLS_DICT to Env.py to avoid circular dependency issue.
TARGET_DUT_CLS_DICT = {
@@ -35,7 +35,7 @@ TARGET_DUT_CLS_DICT = {
def format_case_id(target, case_name):
return "{}.{}".format(target, case_name)
return '{}.{}'.format(target, case_name)
try:
@@ -128,13 +128,13 @@ def test_func_generator(func, app, target, ci_target, module, execution_time, le
dut_dict=dut_classes, **kwargs
)
test_func = original_method(func)
test_func.case_info["ID"] = format_case_id(target, test_func.case_info["name"])
test_func.case_info['ID'] = format_case_id(target, test_func.case_info['name'])
return test_func
@ci_target_check
def idf_example_test(app=Example, target="ESP32", ci_target=None, module="examples", execution_time=1,
level="example", erase_nvs=True, config_name=None, **kwargs):
def idf_example_test(app=Example, target='ESP32', ci_target=None, module='examples', execution_time=1,
level='example', erase_nvs=True, config_name=None, **kwargs):
"""
decorator for testing idf examples (with default values for some keyword args).
@@ -155,8 +155,8 @@ def idf_example_test(app=Example, target="ESP32", ci_target=None, module="exampl
@ci_target_check
def idf_unit_test(app=UT, target="ESP32", ci_target=None, module="unit-test", execution_time=1,
level="unit", erase_nvs=True, **kwargs):
def idf_unit_test(app=UT, target='ESP32', ci_target=None, module='unit-test', execution_time=1,
level='unit', erase_nvs=True, **kwargs):
"""
decorator for testing idf unit tests (with default values for some keyword args).
@@ -176,8 +176,8 @@ def idf_unit_test(app=UT, target="ESP32", ci_target=None, module="unit-test", ex
@ci_target_check
def idf_custom_test(app=TestApp, target="ESP32", ci_target=None, module="misc", execution_time=1,
level="integration", erase_nvs=True, config_name=None, **kwargs):
def idf_custom_test(app=TestApp, target='ESP32', ci_target=None, module='misc', execution_time=1,
level='integration', erase_nvs=True, config_name=None, **kwargs):
"""
decorator for idf custom tests (with default values for some keyword args).
@@ -198,8 +198,8 @@ def idf_custom_test(app=TestApp, target="ESP32", ci_target=None, module="misc",
@ci_target_check
def idf_component_unit_test(app=ComponentUTApp, target="ESP32", ci_target=None, module="misc", execution_time=1,
level="integration", erase_nvs=True, config_name=None, **kwargs):
def idf_component_unit_test(app=ComponentUTApp, target='ESP32', ci_target=None, module='misc', execution_time=1,
level='integration', erase_nvs=True, config_name=None, **kwargs):
"""
decorator for idf custom tests (with default values for some keyword args).
@@ -253,11 +253,11 @@ def log_performance(item, value):
:param item: performance item name
:param value: performance value
"""
performance_msg = "[Performance][{}]: {}".format(item, value)
Utility.console_log(performance_msg, "orange")
performance_msg = '[Performance][{}]: {}'.format(item, value)
Utility.console_log(performance_msg, 'orange')
# update to junit test report
current_junit_case = TinyFW.JunitReport.get_current_test_case()
current_junit_case.stdout += performance_msg + "\r\n"
current_junit_case.stdout += performance_msg + '\r\n'
def check_performance(item, value, target):
@@ -300,7 +300,7 @@ def check_performance(item, value, target):
# if no exception was thrown then the performance is met and no need to continue
break
else:
raise AssertionError("Failed to get performance standard for {}".format(item))
raise AssertionError('Failed to get performance standard for {}'.format(item))
MINIMUM_FREE_HEAP_SIZE_RE = re.compile(r'Minimum free heap size: (\d+) bytes')

98
tools/ci/python_packages/ttfw_idf/unity_test_parser.py
View File

@@ -13,13 +13,13 @@ import re
import junit_xml
_NORMAL_TEST_REGEX = re.compile(r"(?P<file>.+):(?P<line>\d+):(?P<test_name>[^\s:]+):(?P<result>PASS|FAIL|IGNORE)(?:: (?P<message>.+))?")
_UNITY_FIXTURE_VERBOSE_PREFIX_REGEX = re.compile(r"(?P<prefix>TEST\((?P<test_group>[^\s,]+), (?P<test_name>[^\s\)]+)\))(?P<remainder>.+)?$")
_UNITY_FIXTURE_REMAINDER_REGEX = re.compile(r"^(?P<file>.+):(?P<line>\d+)::(?P<result>PASS|FAIL|IGNORE)(?:: (?P<message>.+))?")
_NORMAL_TEST_REGEX = re.compile(r'(?P<file>.+):(?P<line>\d+):(?P<test_name>[^\s:]+):(?P<result>PASS|FAIL|IGNORE)(?:: (?P<message>.+))?')
_UNITY_FIXTURE_VERBOSE_PREFIX_REGEX = re.compile(r'(?P<prefix>TEST\((?P<test_group>[^\s,]+), (?P<test_name>[^\s\)]+)\))(?P<remainder>.+)?$')
_UNITY_FIXTURE_REMAINDER_REGEX = re.compile(r'^(?P<file>.+):(?P<line>\d+)::(?P<result>PASS|FAIL|IGNORE)(?:: (?P<message>.+))?')
_TEST_SUMMARY_BLOCK_REGEX = re.compile(
r"^(?P<num_tests>\d+) Tests (?P<num_failures>\d+) Failures (?P<num_ignored>\d+) Ignored\s*\r?\n(?P<overall_result>OK|FAIL)(?:ED)?", re.MULTILINE
r'^(?P<num_tests>\d+) Tests (?P<num_failures>\d+) Failures (?P<num_ignored>\d+) Ignored\s*\r?\n(?P<overall_result>OK|FAIL)(?:ED)?', re.MULTILINE
)
_TEST_RESULT_ENUM = ["PASS", "FAIL", "IGNORE"]
_TEST_RESULT_ENUM = ['PASS', 'FAIL', 'IGNORE']
class TestFormat(enum.Enum):
@@ -63,14 +63,14 @@ class TestResult:
self,
test_name,
result,
group="default",
file="",
group='default',
file='',
line=0,
message="",
full_line="",
message='',
full_line='',
):
if result not in _TEST_RESULT_ENUM:
raise ValueError("result must be one of {}.".format(_TEST_RESULT_ENUM))
raise ValueError('result must be one of {}.'.format(_TEST_RESULT_ENUM))
self._test_name = test_name
self._result = result
@@ -78,11 +78,11 @@ class TestResult:
self._message = message
self._full_line = full_line
if result != "PASS":
if result != 'PASS':
self._file = file
self._line = line
else:
self._file = ""
self._file = ''
self._line = 0
def file(self):
@@ -150,7 +150,7 @@ class TestResults:
self._parse_unity_fixture_verbose(test_output)
else:
raise ValueError(
"test_format must be one of UNITY_BASIC or UNITY_FIXTURE_VERBOSE."
'test_format must be one of UNITY_BASIC or UNITY_FIXTURE_VERBOSE.'
)
def num_tests(self):
@@ -185,7 +185,7 @@ class TestResults:
return self._tests
def to_junit(
self, suite_name="all_tests",
self, suite_name='all_tests',
):
"""
Convert the tests to JUnit XML.
@@ -207,7 +207,7 @@ class TestResults:
test_case_list = []
for test in self._tests:
if test.result() == "PASS":
if test.result() == 'PASS':
test_case_list.append(
junit_xml.TestCase(name=test.name(), classname=test.group())
)
@@ -218,11 +218,11 @@ class TestResults:
file=test.file(),
line=test.line(),
)
if test.result() == "FAIL":
if test.result() == 'FAIL':
junit_tc.add_failure_info(
message=test.message(), output=test.full_line()
)
elif test.result() == "IGNORE":
elif test.result() == 'IGNORE':
junit_tc.add_skipped_info(
message=test.message(), output=test.full_line()
)
@@ -245,17 +245,17 @@ class TestResults:
"""
match = _TEST_SUMMARY_BLOCK_REGEX.search(unity_output)
if not match:
raise ValueError("A Unity test summary block was not found.")
raise ValueError('A Unity test summary block was not found.')
try:
stats = TestStats()
stats.total = int(match.group("num_tests"))
stats.failed = int(match.group("num_failures"))
stats.ignored = int(match.group("num_ignored"))
stats.total = int(match.group('num_tests'))
stats.failed = int(match.group('num_failures'))
stats.ignored = int(match.group('num_ignored'))
stats.passed = stats.total - stats.failed - stats.ignored
return stats
except ValueError:
raise ValueError("The Unity test summary block was not valid.")
raise ValueError('The Unity test summary block was not valid.')
def _parse_unity_basic(self, unity_output):
"""
@@ -268,13 +268,13 @@ class TestResults:
for test in _NORMAL_TEST_REGEX.finditer(unity_output):
try:
new_test = TestResult(
test.group("test_name"),
test.group("result"),
file=test.group("file"),
line=int(test.group("line")),
message=test.group("message")
if test.group("message") is not None
else "",
test.group('test_name'),
test.group('result'),
file=test.group('file'),
line=int(test.group('line')),
message=test.group('message')
if test.group('message') is not None
else '',
full_line=test.group(0),
)
except ValueError:
@@ -283,10 +283,10 @@ class TestResults:
self._add_new_test(new_test, found_test_stats)
if len(self._tests) == 0:
raise ValueError("No tests were found.")
raise ValueError('No tests were found.')
if found_test_stats != self._test_stats:
raise ValueError("Test output does not match summary block.")
raise ValueError('Test output does not match summary block.')
def _parse_unity_fixture_verbose(self, unity_output):
"""
@@ -309,7 +309,7 @@ class TestResults:
if prefix_match:
# Handle the remaining portion of a test case line after the unity_fixture
# prefix.
remainder = prefix_match.group("remainder")
remainder = prefix_match.group('remainder')
if remainder:
self._parse_unity_fixture_remainder(
prefix_match, remainder, found_test_stats
@@ -324,10 +324,10 @@ class TestResults:
pass
if len(self._tests) == 0:
raise ValueError("No tests were found.")
raise ValueError('No tests were found.')
if found_test_stats != self._test_stats:
raise ValueError("Test output does not match summary block.")
raise ValueError('Test output does not match summary block.')
def _parse_unity_fixture_remainder(self, prefix_match, remainder, test_stats):
"""
@@ -337,26 +337,26 @@ class TestResults:
"""
new_test = None
if remainder == " PASS":
if remainder == ' PASS':
new_test = TestResult(
prefix_match.group("test_name"),
"PASS",
group=prefix_match.group("test_group"),
prefix_match.group('test_name'),
'PASS',
group=prefix_match.group('test_group'),
full_line=prefix_match.group(0),
)
else:
remainder_match = _UNITY_FIXTURE_REMAINDER_REGEX.match(remainder)
if remainder_match:
new_test = TestResult(
prefix_match.group("test_name"),
remainder_match.group("result"),
group=prefix_match.group("test_group"),
file=remainder_match.group("file"),
line=int(remainder_match.group("line")),
message=remainder_match.group("message")
if remainder_match.group("message") is not None
else "",
full_line=prefix_match.group("prefix") + remainder_match.group(0),
prefix_match.group('test_name'),
remainder_match.group('result'),
group=prefix_match.group('test_group'),
file=remainder_match.group('file'),
line=int(remainder_match.group('line')),
message=remainder_match.group('message')
if remainder_match.group('message') is not None
else '',
full_line=prefix_match.group('prefix') + remainder_match.group(0),
)
if new_test is not None:
@@ -365,9 +365,9 @@ class TestResults:
def _add_new_test(self, new_test, test_stats):
"""Add a new test and increment the proper members of test_stats."""
test_stats.total += 1
if new_test.result() == "PASS":
if new_test.result() == 'PASS':
test_stats.passed += 1
elif new_test.result() == "FAIL":
elif new_test.result() == 'FAIL':
test_stats.failed += 1
else:
test_stats.ignored += 1

29
tools/ci/python_packages/wifi_tools.py
View File

@@ -13,10 +13,11 @@
# limitations under the License.
#
import time
import dbus
import dbus.mainloop.glib
import netifaces
import time
def get_wiface_name():
@@ -46,17 +47,17 @@ class wpa_cli:
dbus.mainloop.glib.DBusGMainLoop(set_as_default=True)
bus = dbus.SystemBus()
service = dbus.Interface(bus.get_object("fi.w1.wpa_supplicant1", "/fi/w1/wpa_supplicant1"),
"fi.w1.wpa_supplicant1")
service = dbus.Interface(bus.get_object('fi.w1.wpa_supplicant1', '/fi/w1/wpa_supplicant1'),
'fi.w1.wpa_supplicant1')
iface_path = service.GetInterface(self.iface_name)
self.iface_obj = bus.get_object("fi.w1.wpa_supplicant1", iface_path)
self.iface_ifc = dbus.Interface(self.iface_obj, "fi.w1.wpa_supplicant1.Interface")
self.iface_obj = bus.get_object('fi.w1.wpa_supplicant1', iface_path)
self.iface_ifc = dbus.Interface(self.iface_obj, 'fi.w1.wpa_supplicant1.Interface')
self.iface_props = dbus.Interface(self.iface_obj, 'org.freedesktop.DBus.Properties')
if self.iface_ifc is None:
raise RuntimeError('supplicant : Failed to fetch interface')
self.old_network = self._get_iface_property("CurrentNetwork")
print("Old network is %s" % self.old_network)
self.old_network = self._get_iface_property('CurrentNetwork')
print('Old network is %s' % self.old_network)
if self.old_network == '/':
self.old_network = None
@@ -69,7 +70,7 @@ class wpa_cli:
Note: The result is a dbus wrapped type, so should usually convert it to the corresponding native
Python type
"""
return self.iface_props.Get("fi.w1.wpa_supplicant1.Interface", name)
return self.iface_props.Get('fi.w1.wpa_supplicant1.Interface', name)
def connect(self, ssid, password):
if self.connected is True:
@@ -79,9 +80,9 @@ class wpa_cli:
if self.new_network is not None:
self.iface_ifc.RemoveNetwork(self.new_network)
print("Pre-connect state is %s, IP is %s" % (self._get_iface_property("State"), get_wiface_IPv4(self.iface_name)))
print('Pre-connect state is %s, IP is %s' % (self._get_iface_property('State'), get_wiface_IPv4(self.iface_name)))
self.new_network = self.iface_ifc.AddNetwork({"ssid": ssid, "psk": password})
self.new_network = self.iface_ifc.AddNetwork({'ssid': ssid, 'psk': password})
self.iface_ifc.SelectNetwork(self.new_network)
time.sleep(10)
@@ -89,12 +90,12 @@ class wpa_cli:
retry = 10
while retry > 0:
time.sleep(5)
state = str(self._get_iface_property("State"))
print("wpa iface state %s (scanning %s)" % (state, bool(self._get_iface_property("Scanning"))))
if state in ["disconnected", "inactive"]:
state = str(self._get_iface_property('State'))
print('wpa iface state %s (scanning %s)' % (state, bool(self._get_iface_property('Scanning'))))
if state in ['disconnected', 'inactive']:
self.iface_ifc.Reconnect()
ip = get_wiface_IPv4(self.iface_name)
print("wpa iface %s IP %s" % (self.iface_name, ip))
print('wpa iface %s IP %s' % (self.iface_name, ip))
if ip is not None:
self.connected = True
return ip

49
tools/ci/test_autocomplete.py
View File

@@ -1,55 +1,56 @@
#!/usr/bin/env python
import os
import sys
import pexpect
import unittest
import pexpect
class Test(unittest.TestCase):
def test_fish(self):
os.environ["TERM"] = "vt100"
child = pexpect.spawn("fish -i")
with open(os.environ["IDF_PATH"] + "/fish" + str(sys.version_info.major) + ".out", "wb") as output:
os.environ['TERM'] = 'vt100'
child = pexpect.spawn('fish -i')
with open(os.environ['IDF_PATH'] + '/fish' + str(sys.version_info.major) + '.out', 'wb') as output:
child.logfile = output
child.sendline('. ./export.fish')
result = child.expect(
["Go to the project directory and run.*idf\\.py build", pexpect.EOF,
['Go to the project directory and run.*idf\\.py build', pexpect.EOF,
pexpect.TIMEOUT], timeout=40)
self.assertEqual(result, 0, "Export was not successful!")
child.send("idf.py \t\t")
result = child.expect(["all.*app.*app-flash.*bootloader.*", pexpect.EOF, pexpect.TIMEOUT], timeout=40)
self.assertEqual(result, 0, "Autocompletion for idf.py failed in fish!")
self.assertEqual(result, 0, 'Export was not successful!')
child.send('idf.py \t\t')
result = child.expect(['all.*app.*app-flash.*bootloader.*', pexpect.EOF, pexpect.TIMEOUT], timeout=40)
self.assertEqual(result, 0, 'Autocompletion for idf.py failed in fish!')
def test_bash(self):
os.environ["TERM"] = "xterm-256color"
child = pexpect.spawn("bash -i")
with open(os.environ["IDF_PATH"] + "/bash" + str(sys.version_info.major) + ".out", "wb") as output:
os.environ['TERM'] = 'xterm-256color'
child = pexpect.spawn('bash -i')
with open(os.environ['IDF_PATH'] + '/bash' + str(sys.version_info.major) + '.out', 'wb') as output:
child.logfile = output
child.sendline('. ./export.sh')
child.send("idf.py \t\t")
child.send('idf.py \t\t')
result = child.expect(
["Go to the project directory and run.*idf\\.py build", pexpect.EOF,
['Go to the project directory and run.*idf\\.py build', pexpect.EOF,
pexpect.TIMEOUT], timeout=40)
self.assertEqual(result, 0, "Export was not successful!")
self.assertEqual(result, 0, 'Export was not successful!')
result = child.expect(
["all.*app.*app-flash.*bootloader.*bootloader-flash.*build-system-targets.*clean.*", pexpect.EOF,
['all.*app.*app-flash.*bootloader.*bootloader-flash.*build-system-targets.*clean.*', pexpect.EOF,
pexpect.TIMEOUT], timeout=40)
self.assertEqual(result, 0, "Autocompletion for idf.py failed in bash!")
self.assertEqual(result, 0, 'Autocompletion for idf.py failed in bash!')
def test_zsh(self):
child = pexpect.spawn("zsh -i")
with open(os.environ["IDF_PATH"] + "/zsh" + str(sys.version_info.major) + ".out", "wb") as output:
child = pexpect.spawn('zsh -i')
with open(os.environ['IDF_PATH'] + '/zsh' + str(sys.version_info.major) + '.out', 'wb') as output:
child.logfile = output
child.sendline('. ./export.sh')
result = child.expect(
["Go to the project directory and run.*idf\\.py build", pexpect.EOF,
['Go to the project directory and run.*idf\\.py build', pexpect.EOF,
pexpect.TIMEOUT], timeout=40)
self.assertEqual(result, 0, "Export was not successful!")
child.send("idf.py \t\t")
self.assertEqual(result, 0, 'Export was not successful!')
child.send('idf.py \t\t')
result = child.expect(
["all.*app.*app-flash.*bootloader.*bootloader-flash.*build-system-targets.*clean.*", pexpect.EOF,
['all.*app.*app-flash.*bootloader.*bootloader-flash.*build-system-targets.*clean.*', pexpect.EOF,
pexpect.TIMEOUT], timeout=40)
self.assertEqual(result, 0, "Autocompletion for idf.py failed in zsh!")
self.assertEqual(result, 0, 'Autocompletion for idf.py failed in zsh!')
if __name__ == '__main__':

7
tools/ci/test_check_kconfigs.py
View File

@@ -15,11 +15,8 @@
# limitations under the License.
import unittest
from check_kconfigs import LineRuleChecker
from check_kconfigs import SourceChecker
from check_kconfigs import InputError
from check_kconfigs import IndentAndNameChecker
from check_kconfigs import CONFIG_NAME_MAX_LENGTH
from check_kconfigs import CONFIG_NAME_MAX_LENGTH, IndentAndNameChecker, InputError, LineRuleChecker, SourceChecker
class ApplyLine(object):

132
tools/cmake/convert_to_cmake.py
View File

@@ -4,15 +4,15 @@
# CMakeLists.txt files
#
import argparse
import subprocess
import re
import os.path
import glob
import os.path
import re
import subprocess
debug = False
def get_make_variables(path, makefile="Makefile", expected_failure=False, variables={}):
def get_make_variables(path, makefile='Makefile', expected_failure=False, variables={}):
"""
Given the path to a Makefile of some kind, return a dictionary of all variables defined in this Makefile
@@ -20,82 +20,82 @@ def get_make_variables(path, makefile="Makefile", expected_failure=False, variab
Overrides IDF_PATH= to avoid recursively evaluating the entire project Makefile structure.
"""
variable_setters = [("%s=%s" % (k,v)) for (k,v) in variables.items()]
variable_setters = [('%s=%s' % (k,v)) for (k,v) in variables.items()]
cmdline = ["make", "-rpn", "-C", path, "-f", makefile] + variable_setters
cmdline = ['make', '-rpn', '-C', path, '-f', makefile] + variable_setters
if debug:
print("Running %s..." % (" ".join(cmdline)))
print('Running %s...' % (' '.join(cmdline)))
p = subprocess.Popen(cmdline, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
(output, stderr) = p.communicate("\n")
(output, stderr) = p.communicate('\n')
if (not expected_failure) and p.returncode != 0:
raise RuntimeError("Unexpected make failure, result %d" % p.returncode)
raise RuntimeError('Unexpected make failure, result %d' % p.returncode)
if debug:
print("Make stdout:")
print('Make stdout:')
print(output)
print("Make stderr:")
print('Make stderr:')
print(stderr)
next_is_makefile = False # is the next line a makefile variable?
result = {}
BUILT_IN_VARS = set(["MAKEFILE_LIST", "SHELL", "CURDIR", "MAKEFLAGS"])
BUILT_IN_VARS = set(['MAKEFILE_LIST', 'SHELL', 'CURDIR', 'MAKEFLAGS'])
for line in output.decode('utf-8').split("\n"):
if line.startswith("# makefile"): # this line appears before any variable defined in the makefile itself
for line in output.decode('utf-8').split('\n'):
if line.startswith('# makefile'): # this line appears before any variable defined in the makefile itself
next_is_makefile = True
elif next_is_makefile:
next_is_makefile = False
m = re.match(r"(?P<var>[^ ]+) :?= (?P<val>.+)", line)
m = re.match(r'(?P<var>[^ ]+) :?= (?P<val>.+)', line)
if m is not None:
if not m.group("var") in BUILT_IN_VARS:
result[m.group("var")] = m.group("val").strip()
if not m.group('var') in BUILT_IN_VARS:
result[m.group('var')] = m.group('val').strip()
return result
def get_component_variables(project_path, component_path):
make_vars = get_make_variables(component_path,
os.path.join(os.environ["IDF_PATH"],
"make",
"component_wrapper.mk"),
os.path.join(os.environ['IDF_PATH'],
'make',
'component_wrapper.mk'),
expected_failure=True,
variables={
"COMPONENT_MAKEFILE": os.path.join(component_path, "component.mk"),
"COMPONENT_NAME": os.path.basename(component_path),
"PROJECT_PATH": project_path,
'COMPONENT_MAKEFILE': os.path.join(component_path, 'component.mk'),
'COMPONENT_NAME': os.path.basename(component_path),
'PROJECT_PATH': project_path,
})
if "COMPONENT_OBJS" in make_vars: # component.mk specifies list of object files
if 'COMPONENT_OBJS' in make_vars: # component.mk specifies list of object files
# Convert to sources
def find_src(obj):
obj = os.path.splitext(obj)[0]
for ext in ["c", "cpp", "S"]:
if os.path.exists(os.path.join(component_path, obj) + "." + ext):
return obj + "." + ext
for ext in ['c', 'cpp', 'S']:
if os.path.exists(os.path.join(component_path, obj) + '.' + ext):
return obj + '.' + ext
print("WARNING: Can't find source file for component %s COMPONENT_OBJS %s" % (component_path, obj))
return None
srcs = []
for obj in make_vars["COMPONENT_OBJS"].split():
for obj in make_vars['COMPONENT_OBJS'].split():
src = find_src(obj)
if src is not None:
srcs.append(src)
make_vars["COMPONENT_SRCS"] = " ".join(srcs)
make_vars['COMPONENT_SRCS'] = ' '.join(srcs)
else:
component_srcs = list()
for component_srcdir in make_vars.get("COMPONENT_SRCDIRS", ".").split():
for component_srcdir in make_vars.get('COMPONENT_SRCDIRS', '.').split():
component_srcdir_path = os.path.abspath(os.path.join(component_path, component_srcdir))
srcs = list()
srcs += glob.glob(os.path.join(component_srcdir_path, "*.[cS]"))
srcs += glob.glob(os.path.join(component_srcdir_path, "*.cpp"))
srcs += glob.glob(os.path.join(component_srcdir_path, '*.[cS]'))
srcs += glob.glob(os.path.join(component_srcdir_path, '*.cpp'))
srcs = [('"%s"' % str(os.path.relpath(s, component_path))) for s in srcs]
make_vars["COMPONENT_ADD_INCLUDEDIRS"] = make_vars.get("COMPONENT_ADD_INCLUDEDIRS", "include")
make_vars['COMPONENT_ADD_INCLUDEDIRS'] = make_vars.get('COMPONENT_ADD_INCLUDEDIRS', 'include')
component_srcs += srcs
make_vars["COMPONENT_SRCS"] = " ".join(component_srcs)
make_vars['COMPONENT_SRCS'] = ' '.join(component_srcs)
return make_vars
@@ -103,33 +103,33 @@ def get_component_variables(project_path, component_path):
def convert_project(project_path):
if not os.path.exists(project_path):
raise RuntimeError("Project directory '%s' not found" % project_path)
if not os.path.exists(os.path.join(project_path, "Makefile")):
if not os.path.exists(os.path.join(project_path, 'Makefile')):
raise RuntimeError("Directory '%s' doesn't contain a project Makefile" % project_path)
project_cmakelists = os.path.join(project_path, "CMakeLists.txt")
project_cmakelists = os.path.join(project_path, 'CMakeLists.txt')
if os.path.exists(project_cmakelists):
raise RuntimeError("This project already has a CMakeLists.txt file")
raise RuntimeError('This project already has a CMakeLists.txt file')
project_vars = get_make_variables(project_path, expected_failure=True)
if "PROJECT_NAME" not in project_vars:
raise RuntimeError("PROJECT_NAME does not appear to be defined in IDF project Makefile at %s" % project_path)
if 'PROJECT_NAME' not in project_vars:
raise RuntimeError('PROJECT_NAME does not appear to be defined in IDF project Makefile at %s' % project_path)
component_paths = project_vars["COMPONENT_PATHS"].split()
component_paths = project_vars['COMPONENT_PATHS'].split()
converted_components = 0
# Convert components as needed
for p in component_paths:
if "MSYSTEM" in os.environ:
cmd = ["cygpath", "-w", p]
if 'MSYSTEM' in os.environ:
cmd = ['cygpath', '-w', p]
p = subprocess.check_output(cmd).decode('utf-8').strip()
converted_components += convert_component(project_path, p)
project_name = project_vars["PROJECT_NAME"]
project_name = project_vars['PROJECT_NAME']
# Generate the project CMakeLists.txt file
with open(project_cmakelists, "w") as f:
with open(project_cmakelists, 'w') as f:
f.write("""
# (Automatically converted from project Makefile by convert_to_cmake.py.)
@@ -141,47 +141,47 @@ cmake_minimum_required(VERSION 3.5)
f.write("""
include($ENV{IDF_PATH}/tools/cmake/project.cmake)
""")
f.write("project(%s)\n" % project_name)
f.write('project(%s)\n' % project_name)
print("Converted project %s" % project_cmakelists)
print('Converted project %s' % project_cmakelists)
if converted_components > 0:
print("Note: Newly created component CMakeLists.txt do not have any REQUIRES or PRIV_REQUIRES "
"lists to declare their component requirements. Builds may fail to include other "
print('Note: Newly created component CMakeLists.txt do not have any REQUIRES or PRIV_REQUIRES '
'lists to declare their component requirements. Builds may fail to include other '
"components' header files. If so requirements need to be added to the components' "
"CMakeLists.txt files. See the 'Component Requirements' section of the "
"Build System docs for more details.")
'Build System docs for more details.')
def convert_component(project_path, component_path):
if debug:
print("Converting %s..." % (component_path))
cmakelists_path = os.path.join(component_path, "CMakeLists.txt")
print('Converting %s...' % (component_path))
cmakelists_path = os.path.join(component_path, 'CMakeLists.txt')
if os.path.exists(cmakelists_path):
print("Skipping already-converted component %s..." % cmakelists_path)
print('Skipping already-converted component %s...' % cmakelists_path)
return 0
v = get_component_variables(project_path, component_path)
# Look up all the variables before we start writing the file, so it's not
# created if there's an erro
component_srcs = v.get("COMPONENT_SRCS", None)
component_srcs = v.get('COMPONENT_SRCS', None)
component_add_includedirs = v["COMPONENT_ADD_INCLUDEDIRS"]
cflags = v.get("CFLAGS", None)
component_add_includedirs = v['COMPONENT_ADD_INCLUDEDIRS']
cflags = v.get('CFLAGS', None)
with open(cmakelists_path, "w") as f:
with open(cmakelists_path, 'w') as f:
if component_srcs is not None:
f.write("idf_component_register(SRCS %s)\n" % component_srcs)
f.write(" INCLUDE_DIRS %s" % component_add_includedirs)
f.write(" # Edit following two lines to set component requirements (see docs)\n")
f.write(" REQUIRES "")\n")
f.write(" PRIV_REQUIRES "")\n\n")
f.write('idf_component_register(SRCS %s)\n' % component_srcs)
f.write(' INCLUDE_DIRS %s' % component_add_includedirs)
f.write(' # Edit following two lines to set component requirements (see docs)\n')
f.write(' REQUIRES '')\n')
f.write(' PRIV_REQUIRES '')\n\n')
else:
f.write("idf_component_register()\n")
f.write('idf_component_register()\n')
if cflags is not None:
f.write("target_compile_options(${COMPONENT_LIB} PRIVATE %s)\n" % cflags)
f.write('target_compile_options(${COMPONENT_LIB} PRIVATE %s)\n' % cflags)
print("Converted %s" % cmakelists_path)
print('Converted %s' % cmakelists_path)
return 1
@@ -197,9 +197,9 @@ def main():
args = parser.parse_args()
debug = args.debug
print("Converting %s..." % args.project)
print('Converting %s...' % args.project)
convert_project(args.project)
if __name__ == "__main__":
if __name__ == '__main__':
main()

42
tools/esp_app_trace/espytrace/apptrace.py
View File

@@ -1,6 +1,8 @@
from __future__ import print_function
import sys
import os
import sys
try:
from urlparse import urlparse
except ImportError:
@@ -9,13 +11,15 @@ try:
import SocketServer
except ImportError:
import socketserver as SocketServer
import threading
import tempfile
import time
import subprocess
import os.path
import elftools.elf.elffile as elffile
import subprocess
import tempfile
import threading
import time
import elftools.elf.constants as elfconst
import elftools.elf.elffile as elffile
def clock():
@@ -44,7 +48,7 @@ def addr2line(toolchain, elf_path, addr):
source line location string
"""
try:
return subprocess.check_output(['%saddr2line' % toolchain, '-e', elf_path, '0x%x' % addr]).decode("utf-8")
return subprocess.check_output(['%saddr2line' % toolchain, '-e', elf_path, '0x%x' % addr]).decode('utf-8')
except subprocess.CalledProcessError:
return ''
@@ -208,7 +212,7 @@ class FileReader(Reader):
line = ''
start_tm = clock()
while not self.need_stop:
line += self.trace_file.readline().decode("utf-8")
line += self.trace_file.readline().decode('utf-8')
if line.endswith(linesep):
break
if self.timeout != -1 and clock() >= start_tm + self.timeout:
@@ -421,13 +425,13 @@ class TraceDataProcessor:
event : object
Event object
"""
print("EVENT[{:d}]: {}".format(self.total_events, event))
print('EVENT[{:d}]: {}'.format(self.total_events, event))
def print_report(self):
"""
Base method to print report.
"""
print("Processed {:d} events".format(self.total_events))
print('Processed {:d} events'.format(self.total_events))
def cleanup(self):
"""
@@ -579,8 +583,8 @@ class BaseLogTraceDataProcessorImpl:
"""
Prints log report
"""
print("=============== LOG TRACE REPORT ===============")
print("Processed {:d} log messages.".format(len(self.messages)))
print('=============== LOG TRACE REPORT ===============')
print('Processed {:d} log messages.'.format(len(self.messages)))
def on_new_event(self, event):
"""
@@ -678,13 +682,13 @@ class HeapTraceEvent:
callers += ':'
callers += '0x{:x}'.format(addr)
if self.alloc:
return "[{:.9f}] HEAP: Allocated {:d} bytes @ 0x{:x} from {} on core {:d} by: {}".format(self.trace_event.ts,
return '[{:.9f}] HEAP: Allocated {:d} bytes @ 0x{:x} from {} on core {:d} by: {}'.format(self.trace_event.ts,
self.size, self.addr,
self.trace_event.ctx_desc,
self.trace_event.core_id,
callers)
else:
return "[{:.9f}] HEAP: Freed bytes @ 0x{:x} from {} on core {:d} by: {}".format(self.trace_event.ts,
return '[{:.9f}] HEAP: Freed bytes @ 0x{:x} from {} on core {:d} by: {}'.format(self.trace_event.ts,
self.addr, self.trace_event.ctx_desc,
self.trace_event.core_id, callers)
@@ -738,10 +742,10 @@ class BaseHeapTraceDataProcessorImpl:
"""
Prints heap report
"""
print("=============== HEAP TRACE REPORT ===============")
print("Processed {:d} heap events.".format(self.heap_events_count))
print('=============== HEAP TRACE REPORT ===============')
print('Processed {:d} heap events.'.format(self.heap_events_count))
if len(self.allocs) == 0:
print("OK - Heap errors was not found.")
print('OK - Heap errors was not found.')
return
leaked_bytes = 0
for alloc in self.allocs:
@@ -749,6 +753,6 @@ class BaseHeapTraceDataProcessorImpl:
print(alloc)
for free in self.frees:
if free.addr > alloc.addr and free.addr <= alloc.addr + alloc.size:
print("Possible wrong free operation found")
print('Possible wrong free operation found')
print(free)
print("Found {:d} leaked bytes in {:d} blocks.".format(leaked_bytes, len(self.allocs)))
print('Found {:d} leaked bytes in {:d} blocks.'.format(leaked_bytes, len(self.allocs)))

108
tools/esp_app_trace/espytrace/sysview.py
View File

@@ -1,9 +1,9 @@
import re
import struct
import copy
import json
import espytrace.apptrace as apptrace
import re
import struct
import espytrace.apptrace as apptrace
SYSVIEW_EVTID_NOP = 0 # Dummy packet.
SYSVIEW_EVTID_OVERFLOW = 1
@@ -42,33 +42,33 @@ SYSVIEW_MODULE_EVENT_OFFSET = 512
SYSVIEW_SYNC_LEN = 10
_sysview_events_map = {
"SYS_NOP": SYSVIEW_EVTID_NOP,
"SYS_OVERFLOW": SYSVIEW_EVTID_OVERFLOW,
"SYS_ISR_ENTER": SYSVIEW_EVTID_ISR_ENTER,
"SYS_ISR_EXIT": SYSVIEW_EVTID_ISR_EXIT,
"SYS_TASK_START_EXEC": SYSVIEW_EVTID_TASK_START_EXEC,
"SYS_TASK_STOP_EXEC": SYSVIEW_EVTID_TASK_STOP_EXEC,
"SYS_TASK_START_READY": SYSVIEW_EVTID_TASK_START_READY,
"SYS_TASK_STOP_READY": SYSVIEW_EVTID_TASK_STOP_READY,
"SYS_TASK_CREATE": SYSVIEW_EVTID_TASK_CREATE,
"SYS_TASK_INFO": SYSVIEW_EVTID_TASK_INFO,
"SYS_TRACE_START": SYSVIEW_EVTID_TRACE_START,
"SYS_TRACE_STOP": SYSVIEW_EVTID_TRACE_STOP,
"SYS_SYSTIME_CYCLES": SYSVIEW_EVTID_SYSTIME_CYCLES,
"SYS_SYSTIME_US": SYSVIEW_EVTID_SYSTIME_US,
"SYS_SYSDESC": SYSVIEW_EVTID_SYSDESC,
"SYS_USER_START": SYSVIEW_EVTID_USER_START,
"SYS_USER_STOP": SYSVIEW_EVTID_USER_STOP,
"SYS_IDLE": SYSVIEW_EVTID_IDLE,
"SYS_ISR_TO_SCHEDULER": SYSVIEW_EVTID_ISR_TO_SCHEDULER,
"SYS_TIMER_ENTER": SYSVIEW_EVTID_TIMER_ENTER,
"SYS_TIMER_EXIT": SYSVIEW_EVTID_TIMER_EXIT,
"SYS_STACK_INFO": SYSVIEW_EVTID_STACK_INFO,
"SYS_MODULEDESC": SYSVIEW_EVTID_INIT,
"SYS_INIT": SYSVIEW_EVTID_INIT,
"SYS_NAME_RESOURCE": SYSVIEW_EVTID_NAME_RESOURCE,
"SYS_PRINT_FORMATTED": SYSVIEW_EVTID_PRINT_FORMATTED,
"SYS_NUMMODULES": SYSVIEW_EVTID_NUMMODULES
'SYS_NOP': SYSVIEW_EVTID_NOP,
'SYS_OVERFLOW': SYSVIEW_EVTID_OVERFLOW,
'SYS_ISR_ENTER': SYSVIEW_EVTID_ISR_ENTER,
'SYS_ISR_EXIT': SYSVIEW_EVTID_ISR_EXIT,
'SYS_TASK_START_EXEC': SYSVIEW_EVTID_TASK_START_EXEC,
'SYS_TASK_STOP_EXEC': SYSVIEW_EVTID_TASK_STOP_EXEC,
'SYS_TASK_START_READY': SYSVIEW_EVTID_TASK_START_READY,
'SYS_TASK_STOP_READY': SYSVIEW_EVTID_TASK_STOP_READY,
'SYS_TASK_CREATE': SYSVIEW_EVTID_TASK_CREATE,
'SYS_TASK_INFO': SYSVIEW_EVTID_TASK_INFO,
'SYS_TRACE_START': SYSVIEW_EVTID_TRACE_START,
'SYS_TRACE_STOP': SYSVIEW_EVTID_TRACE_STOP,
'SYS_SYSTIME_CYCLES': SYSVIEW_EVTID_SYSTIME_CYCLES,
'SYS_SYSTIME_US': SYSVIEW_EVTID_SYSTIME_US,
'SYS_SYSDESC': SYSVIEW_EVTID_SYSDESC,
'SYS_USER_START': SYSVIEW_EVTID_USER_START,
'SYS_USER_STOP': SYSVIEW_EVTID_USER_STOP,
'SYS_IDLE': SYSVIEW_EVTID_IDLE,
'SYS_ISR_TO_SCHEDULER': SYSVIEW_EVTID_ISR_TO_SCHEDULER,
'SYS_TIMER_ENTER': SYSVIEW_EVTID_TIMER_ENTER,
'SYS_TIMER_EXIT': SYSVIEW_EVTID_TIMER_EXIT,
'SYS_STACK_INFO': SYSVIEW_EVTID_STACK_INFO,
'SYS_MODULEDESC': SYSVIEW_EVTID_INIT,
'SYS_INIT': SYSVIEW_EVTID_INIT,
'SYS_NAME_RESOURCE': SYSVIEW_EVTID_NAME_RESOURCE,
'SYS_PRINT_FORMATTED': SYSVIEW_EVTID_PRINT_FORMATTED,
'SYS_NUMMODULES': SYSVIEW_EVTID_NUMMODULES
}
_os_events_map = {}
@@ -175,7 +175,7 @@ def _read_init_seq(reader):
sync_bytes = struct.unpack(SYNC_SEQ_FMT, reader.read(struct.calcsize(SYNC_SEQ_FMT)))
for b in sync_bytes:
if b != 0:
raise SysViewTraceParseError("Invalid sync sequense!")
raise SysViewTraceParseError('Invalid sync sequense!')
def _decode_u32(reader):
@@ -263,7 +263,7 @@ def _decode_str(reader):
buf = struct.unpack('<2B', reader.read(2))
sz = (buf[1] << 8) | buf[0]
val, = struct.unpack('<%ds' % sz, reader.read(sz))
val = val.decode("utf-8")
val = val.decode('utf-8')
if sz < 0xFF:
return (sz + 1,val) # one extra byte for length
return (sz + 3,val) # 3 extra bytes for length
@@ -347,7 +347,7 @@ class SysViewEvent(apptrace.TraceEvent):
if event has unknown or invalid format.
"""
if self.id not in events_fmt_map:
raise SysViewTraceParseError("Unknown event ID %d!" % self.id)
raise SysViewTraceParseError('Unknown event ID %d!' % self.id)
self.name = events_fmt_map[self.id][0]
evt_params_templates = events_fmt_map[self.id][1]
params_len = 0
@@ -357,13 +357,13 @@ class SysViewEvent(apptrace.TraceEvent):
cur_pos = reader.get_pos()
sz,param_val = event_param.decode(reader, self.plen - params_len)
except Exception as e:
raise SysViewTraceParseError("Failed to decode event {}({:d}) {:d} param @ 0x{:x}! {}".format(self.name, self.id, self.plen, cur_pos, e))
raise SysViewTraceParseError('Failed to decode event {}({:d}) {:d} param @ 0x{:x}! {}'.format(self.name, self.id, self.plen, cur_pos, e))
event_param.idx = i
event_param.value = param_val
self.params[event_param.name] = event_param
params_len += sz
if self.id >= SYSVIEW_EVENT_ID_PREDEF_LEN_MAX and self.plen != params_len:
raise SysViewTraceParseError("Invalid event {}({:d}) payload len {:d}! Must be {:d}.".format(self.name, self.id, self.plen, params_len))
raise SysViewTraceParseError('Invalid event {}({:d}) payload len {:d}! Must be {:d}.'.format(self.name, self.id, self.plen, params_len))
def __str__(self):
params = ''
@@ -886,7 +886,7 @@ class SysViewTraceDataProcessor(apptrace.TraceDataProcessor):
"""
apptrace.TraceDataProcessor.__init__(self, print_events, keep_all_events)
self.event_ids = {}
self.name = ""
self.name = ''
self.root_proc = root_proc if root_proc else self
self.traces = {}
self.ctx_stack = {}
@@ -1003,12 +1003,12 @@ class SysViewTraceDataProcessor(apptrace.TraceDataProcessor):
if event.core_id not in self.prev_ctx:
self.prev_ctx[event.core_id] = None
else:
raise SysViewTraceParseError("Event for unknown core %d" % event.core_id)
raise SysViewTraceParseError('Event for unknown core %d' % event.core_id)
else:
trace = self.traces[event.core_id]
if event.id == SYSVIEW_EVTID_ISR_ENTER:
if event.params['irq_num'].value not in trace.irqs_info:
raise SysViewTraceParseError("Enter unknown ISR %d" % event.params['irq_num'].value)
raise SysViewTraceParseError('Enter unknown ISR %d' % event.params['irq_num'].value)
if len(self.ctx_stack[event.core_id]):
self.prev_ctx[event.core_id] = self.ctx_stack[event.core_id][-1]
else:
@@ -1026,7 +1026,7 @@ class SysViewTraceDataProcessor(apptrace.TraceDataProcessor):
self.prev_ctx[event.core_id] = SysViewEventContext(None, True, 'IRQ_oncore%d' % event.core_id)
elif event.id == SYSVIEW_EVTID_TASK_START_EXEC:
if event.params['tid'].value not in trace.tasks_info:
raise SysViewTraceParseError("Start exec unknown task 0x%x" % event.params['tid'].value)
raise SysViewTraceParseError('Start exec unknown task 0x%x' % event.params['tid'].value)
if len(self.ctx_stack[event.core_id]):
# return to the previous context (the last in the list)
self.prev_ctx[event.core_id] = self.ctx_stack[event.core_id][-1]
@@ -1046,7 +1046,7 @@ class SysViewTraceDataProcessor(apptrace.TraceDataProcessor):
break
elif event.id == SYSVIEW_EVTID_TASK_STOP_READY:
if event.params['tid'].value not in trace.tasks_info:
raise SysViewTraceParseError("Stop ready unknown task 0x%x" % event.params['tid'].value)
raise SysViewTraceParseError('Stop ready unknown task 0x%x' % event.params['tid'].value)
if len(self.ctx_stack[event.core_id]):
if (not self.ctx_stack[event.core_id][-1].irq and event.params['tid'].value == self.ctx_stack[event.core_id][-1].handle):
# return to the previous context (the last in the list)
@@ -1187,13 +1187,13 @@ class SysViewMultiStreamTraceDataProcessor(SysViewTraceDataProcessor):
class SysViewTraceDataJsonEncoder(json.JSONEncoder):
JSON_TRACE_VER = "1.0"
JSON_TRACE_VER = '1.0'
def default(self, obj):
global _sysview_events_map
global _os_events_map
if isinstance(obj, SysViewMultiStreamTraceDataProcessor):
json_event_ids = {"system": _sysview_events_map, "os": {}}
json_event_ids = {'system': _sysview_events_map, 'os': {}}
for eid in _os_events_map:
ename = _os_events_map[eid][0]
json_event_ids['os'][ename] = eid
@@ -1208,20 +1208,20 @@ class SysViewTraceDataJsonEncoder(json.JSONEncoder):
# include also OS and pre-defined events
if isinstance(e, SysViewPredefinedEvent) or isinstance(e, SysViewOSEvent):
json_events.append(e)
return {"version": self.JSON_TRACE_VER, "streams": json_event_ids, "events": json_events}
return {'version': self.JSON_TRACE_VER, 'streams': json_event_ids, 'events': json_events}
if isinstance(obj, SysViewHeapEvent):
blk_size = 0
if "size" in obj.params:
blk_size = obj.params["size"].value
blk_addr = "0x{:x}".format(obj.params["addr"].value)
if 'size' in obj.params:
blk_size = obj.params['size'].value
blk_addr = '0x{:x}'.format(obj.params['addr'].value)
callers = []
for addr in obj.params['callers'].value:
callers.append('0x{:x}'.format(addr))
return {"ctx_name": obj.ctx_name, "in_irq": obj.in_irq, "id": obj.id, "core_id": obj.core_id,
"ts": obj.ts, "addr": blk_addr, "size": blk_size, "callers": callers}
return {'ctx_name': obj.ctx_name, 'in_irq': obj.in_irq, 'id': obj.id, 'core_id': obj.core_id,
'ts': obj.ts, 'addr': blk_addr, 'size': blk_size, 'callers': callers}
if isinstance(obj, SysViewPredefinedEvent) and obj.id == SYSVIEW_EVTID_PRINT_FORMATTED:
return {"ctx_name": obj.ctx_name, "in_irq": obj.in_irq, "id": obj.id, "core_id": obj.core_id,
"ts": obj.ts, "msg": obj.params["msg"].value, "lvl": obj.params["lvl"].value}
return {'ctx_name': obj.ctx_name, 'in_irq': obj.in_irq, 'id': obj.id, 'core_id': obj.core_id,
'ts': obj.ts, 'msg': obj.params['msg'].value, 'lvl': obj.params['lvl'].value}
if isinstance(obj, SysViewEvent):
jobj = obj.to_jsonable()
# remove unused fields
@@ -1280,9 +1280,9 @@ class SysViewHeapTraceDataProcessor(SysViewTraceDataProcessor, apptrace.BaseHeap
self.toolchain = toolchain_pref
self.elf_path = elf_path
# self.no_ctx_events = []
self.name = "heap"
self.name = 'heap'
stream = self.root_proc.get_trace_stream(0, SysViewTraceDataParser.STREAMID_HEAP)
self.event_ids = {"alloc": stream.events_off, "free": stream.events_off + 1}
self.event_ids = {'alloc': stream.events_off, 'free': stream.events_off + 1}
def event_supported(self, event):
heap_stream = self.root_proc.get_trace_stream(event.core_id, SysViewTraceDataParser.STREAMID_HEAP)
@@ -1362,8 +1362,8 @@ class SysViewLogTraceDataProcessor(SysViewTraceDataProcessor, apptrace.BaseLogTr
"""
SysViewTraceDataProcessor.__init__(self, traces, root_proc=root_proc, print_events=print_events)
apptrace.BaseLogTraceDataProcessorImpl.__init__(self, print_log_events)
self.name = "log"
self.event_ids = {"print": SYSVIEW_EVTID_PRINT_FORMATTED}
self.name = 'log'
self.event_ids = {'print': SYSVIEW_EVTID_PRINT_FORMATTED}
def event_supported(self, event):
return event.id == SYSVIEW_EVTID_PRINT_FORMATTED

36
tools/esp_app_trace/logtrace_proc.py
View File

@@ -2,9 +2,11 @@
#
from __future__ import print_function
import argparse
import struct
import sys
import elftools.elf.elffile as elffile
import espytrace.apptrace as apptrace
@@ -21,7 +23,7 @@ class ESPLogTraceRecord(object):
self.args = log_args
def __repr__(self):
return "fmt_addr = 0x%x, args = %d/%s" % (self.fmt_addr, len(self.args), self.args)
return 'fmt_addr = 0x%x, args = %d/%s' % (self.fmt_addr, len(self.args), self.args)
def logtrace_parse(fname):
@@ -32,40 +34,40 @@ def logtrace_parse(fname):
try:
ftrc = open(fname, 'rb')
except OSError as e:
raise ESPLogTraceParserError("Failed to open trace file (%s)!" % e)
raise ESPLogTraceParserError('Failed to open trace file (%s)!' % e)
# data_ok = True
while True:
# read args num and format str addr
try:
trc_buf = ftrc.read(ESP32_LOGTRACE_HDR_SZ)
except IOError as e:
raise ESPLogTraceParserError("Failed to read log record header (%s)!" % e)
raise ESPLogTraceParserError('Failed to read log record header (%s)!' % e)
if len(trc_buf) < ESP32_LOGTRACE_HDR_SZ:
# print "EOF"
if len(trc_buf) > 0:
print("Unprocessed %d bytes of log record header!" % len(trc_buf))
print('Unprocessed %d bytes of log record header!' % len(trc_buf))
# data_ok = False
break
try:
nargs,fmt_addr = struct.unpack(ESP32_LOGTRACE_HDR_FMT, trc_buf)
except struct.error as e:
raise ESPLogTraceParserError("Failed to unpack log record header (%s)!" % e)
raise ESPLogTraceParserError('Failed to unpack log record header (%s)!' % e)
# read args
args_sz = struct.calcsize('<%sL' % nargs)
try:
trc_buf = ftrc.read(args_sz)
except IOError as e:
raise ESPLogTraceParserError("Failed to read log record args (%s)!" % e)
raise ESPLogTraceParserError('Failed to read log record args (%s)!' % e)
if len(trc_buf) < args_sz:
# print("EOF")
if len(trc_buf) > 0:
print("Unprocessed %d bytes of log record args!" % len(trc_buf))
print('Unprocessed %d bytes of log record args!' % len(trc_buf))
# data_ok = False
break
try:
log_args = struct.unpack('<%sL' % nargs, trc_buf)
except struct.error as e:
raise ESPLogTraceParserError("Failed to unpack log record args (%s)!" % e)
raise ESPLogTraceParserError('Failed to unpack log record args (%s)!' % e)
# print(log_args)
recs.append(ESPLogTraceRecord(fmt_addr, list(log_args)))
@@ -78,7 +80,7 @@ def logtrace_formated_print(recs, elfname, no_err):
try:
felf = elffile.ELFFile(open(elfname, 'rb'))
except OSError as e:
raise ESPLogTraceParserError("Failed to open ELF file (%s)!" % e)
raise ESPLogTraceParserError('Failed to open ELF file (%s)!' % e)
for lrec in recs:
fmt_str = apptrace.get_str_from_elf(felf, lrec.fmt_addr)
@@ -104,8 +106,8 @@ def logtrace_formated_print(recs, elfname, no_err):
pass
except Exception as e:
if not no_err:
print("Print error (%s)" % e)
print("\nFmt = {%s}, args = %d/%s" % (fmt_str, len(lrec.args), lrec.args))
print('Print error (%s)' % e)
print('\nFmt = {%s}, args = %d/%s' % (fmt_str, len(lrec.args), lrec.args))
felf.stream.close()
@@ -123,21 +125,21 @@ def main():
try:
print("Parse trace file '%s'..." % args.trace_file)
lrecs = logtrace_parse(args.trace_file)
print("Parsing completed.")
print('Parsing completed.')
except ESPLogTraceParserError as e:
print("Failed to parse log trace (%s)!" % e)
print('Failed to parse log trace (%s)!' % e)
sys.exit(2)
# print recs
# get format strings and print info
print("====================================================================")
print('====================================================================')
try:
logtrace_formated_print(lrecs, args.elf_file, args.no_errors)
except ESPLogTraceParserError as e:
print("Failed to print log trace (%s)!" % e)
print('Failed to print log trace (%s)!' % e)
sys.exit(2)
print("\n====================================================================\n")
print('\n====================================================================\n')
print("Log records count: %d" % len(lrecs))
print('Log records count: %d' % len(lrecs))
if __name__ == '__main__':

23
tools/esp_app_trace/sysviewtrace_proc.py
View File

@@ -20,12 +20,13 @@
#
import argparse
import sys
import json
import logging
import os.path
import signal
import sys
import traceback
import logging
import json
import espytrace.apptrace as apptrace
import espytrace.sysview as sysview
@@ -83,22 +84,22 @@ def main():
sysview.SysViewLogTraceDataParser(print_events=False, core_id=i))
parsers.append(parser)
except Exception as e:
logging.error("Failed to create data parser (%s)!", e)
logging.error('Failed to create data parser (%s)!', e)
traceback.print_exc()
sys.exit(2)
reader = apptrace.reader_create(trace_source, args.tmo)
if not reader:
logging.error("Failed to create trace reader!")
logging.error('Failed to create trace reader!')
sys.exit(2)
try:
# logging.info("Parse trace from '{}'...".format(trace_source))
logging.info("Parse trace from '%s'...", trace_source)
sysview.parse_trace(reader, parser, args.events_map)
logging.info("Parsing completed.")
logging.info('Parsing completed.')
except (apptrace.ReaderTimeoutError, apptrace.ReaderShutdownRequest) as e:
logging.info("Stop parsing trace. (%s)", e)
logging.info('Stop parsing trace. (%s)', e)
except Exception as e:
logging.error("Failed to parse trace (%s)!", e)
logging.error('Failed to parse trace (%s)!', e)
parser.cleanup()
traceback.print_exc()
sys.exit(2)
@@ -115,16 +116,16 @@ def main():
proc.add_stream_processor(sysview.SysViewTraceDataParser.STREAMID_LOG,
sysview.SysViewLogTraceDataProcessor(root_proc=proc, print_log_events=args.print_events))
except Exception as e:
logging.error("Failed to create data processor (%s)!", e)
logging.error('Failed to create data processor (%s)!', e)
traceback.print_exc()
sys.exit(2)
try:
logging.info("Process events from '%s'...", args.trace_sources)
proc.merge_and_process()
logging.info("Processing completed.")
logging.info('Processing completed.')
except Exception as e:
logging.error("Failed to process trace (%s)!", e)
logging.error('Failed to process trace (%s)!', e)
traceback.print_exc()
sys.exit(2)
finally:

147
tools/esp_prov/esp_prov.py
View File

@@ -16,28 +16,29 @@
#
from __future__ import print_function
from builtins import input as binput
import argparse
import textwrap
import time
import json
import os
import sys
import json
import textwrap
import time
from builtins import input as binput
from getpass import getpass
try:
import prov
import security
import transport
import prov
except ImportError:
idf_path = os.environ['IDF_PATH']
sys.path.insert(0, idf_path + "/components/protocomm/python")
sys.path.insert(1, idf_path + "/tools/esp_prov")
sys.path.insert(0, idf_path + '/components/protocomm/python')
sys.path.insert(1, idf_path + '/tools/esp_prov')
import prov
import security
import transport
import prov
# Set this to true to allow exceptions to be thrown
config_throw_except = False
@@ -92,7 +93,7 @@ def version_match(tp, protover, verbose=False):
response = tp.send_data('proto-ver', protover)
if verbose:
print("proto-ver response : ", response)
print('proto-ver response : ', response)
# First assume this to be a simple version string
if response.lower() == protover.lower():
@@ -123,7 +124,7 @@ def has_capability(tp, capability='none', verbose=False):
response = tp.send_data('proto-ver', capability)
if verbose:
print("proto-ver response : ", response)
print('proto-ver response : ', response)
try:
# Interpret this as JSON structure containing
@@ -221,16 +222,16 @@ def scan_wifi_APs(sel_transport, tp, sec):
start_time = time.time()
response = tp.send_data('prov-scan', message)
stop_time = time.time()
print("++++ Scan process executed in " + str(stop_time - start_time) + " sec")
print('++++ Scan process executed in ' + str(stop_time - start_time) + ' sec')
prov.scan_start_response(sec, response)
message = prov.scan_status_request(sec)
response = tp.send_data('prov-scan', message)
result = prov.scan_status_response(sec, response)
print("++++ Scan results : " + str(result["count"]))
if result["count"] != 0:
print('++++ Scan results : ' + str(result['count']))
if result['count'] != 0:
index = 0
remaining = result["count"]
remaining = result['count']
while remaining:
count = [remaining, readlen][remaining > readlen]
message = prov.scan_result_request(sec, index, count)
@@ -287,25 +288,25 @@ def wait_wifi_connected(tp, sec):
while True:
time.sleep(TIME_PER_POLL)
print("\n==== Wi-Fi connection state ====")
print('\n==== Wi-Fi connection state ====')
ret = get_wifi_config(tp, sec)
if ret == "connecting":
if ret == 'connecting':
continue
elif ret == "connected":
print("==== Provisioning was successful ====")
elif ret == 'connected':
print('==== Provisioning was successful ====')
return True
elif retry > 0:
retry -= 1
print("Waiting to poll status again (status %s, %d tries left)..." % (ret, retry))
print('Waiting to poll status again (status %s, %d tries left)...' % (ret, retry))
else:
print("---- Provisioning failed ----")
print('---- Provisioning failed ----')
return False
def desc_format(*args):
desc = ''
for arg in args:
desc += textwrap.fill(replace_whitespace=False, text=arg) + "\n"
desc += textwrap.fill(replace_whitespace=False, text=arg) + '\n'
return desc
@@ -316,12 +317,12 @@ if __name__ == '__main__':
'See esp-idf/examples/provisioning for sample applications'),
formatter_class=argparse.RawTextHelpFormatter)
parser.add_argument("--transport", required=True, dest='mode', type=str,
parser.add_argument('--transport', required=True, dest='mode', type=str,
help=desc_format(
'Mode of transport over which provisioning is to be performed.',
'This should be one of "softap", "ble" or "console"'))
parser.add_argument("--service_name", dest='name', type=str,
parser.add_argument('--service_name', dest='name', type=str,
help=desc_format(
'This specifies the name of the provisioning service to connect to, '
'depending upon the mode of transport :',
@@ -329,12 +330,12 @@ if __name__ == '__main__':
'\t- transport "softap" : HTTP Server hostname or IP',
'\t (default "192.168.4.1:80")'))
parser.add_argument("--proto_ver", dest='version', type=str, default='',
parser.add_argument('--proto_ver', dest='version', type=str, default='',
help=desc_format(
'This checks the protocol version of the provisioning service running '
'on the device before initiating Wi-Fi configuration'))
parser.add_argument("--sec_ver", dest='secver', type=int, default=None,
parser.add_argument('--sec_ver', dest='secver', type=int, default=None,
help=desc_format(
'Protocomm security scheme used by the provisioning service for secure '
'session establishment. Accepted values are :',
@@ -345,48 +346,48 @@ if __name__ == '__main__':
'the compatible security version is automatically determined from '
'capabilities retrieved via the version endpoint'))
parser.add_argument("--pop", dest='pop', type=str, default='',
parser.add_argument('--pop', dest='pop', type=str, default='',
help=desc_format(
'This specifies the Proof of possession (PoP) when security scheme 1 '
'is used'))
parser.add_argument("--ssid", dest='ssid', type=str, default='',
parser.add_argument('--ssid', dest='ssid', type=str, default='',
help=desc_format(
'This configures the device to use SSID of the Wi-Fi network to which '
'we would like it to connect to permanently, once provisioning is complete. '
'If Wi-Fi scanning is supported by the provisioning service, this need not '
'be specified'))
parser.add_argument("--passphrase", dest='passphrase', type=str, default='',
parser.add_argument('--passphrase', dest='passphrase', type=str, default='',
help=desc_format(
'This configures the device to use Passphrase for the Wi-Fi network to which '
'we would like it to connect to permanently, once provisioning is complete. '
'If Wi-Fi scanning is supported by the provisioning service, this need not '
'be specified'))
parser.add_argument("--custom_data", dest='custom_data', type=str, default='',
parser.add_argument('--custom_data', dest='custom_data', type=str, default='',
help=desc_format(
'This is an optional parameter, only intended for use with '
'"examples/provisioning/wifi_prov_mgr_custom_data"'))
parser.add_argument("--custom_config", action="store_true",
parser.add_argument('--custom_config', action='store_true',
help=desc_format(
'This is an optional parameter, only intended for use with '
'"examples/provisioning/custom_config"'))
parser.add_argument("--custom_info", dest='custom_info', type=str, default='<some custom info string>',
parser.add_argument('--custom_info', dest='custom_info', type=str, default='<some custom info string>',
help=desc_format(
'Custom Config Info String. "--custom_config" must be specified for using this'))
parser.add_argument("--custom_ver", dest='custom_ver', type=int, default=2,
parser.add_argument('--custom_ver', dest='custom_ver', type=int, default=2,
help=desc_format(
'Custom Config Version Number. "--custom_config" must be specified for using this'))
parser.add_argument("-v","--verbose", help="Increase output verbosity", action="store_true")
parser.add_argument('-v','--verbose', help='Increase output verbosity', action='store_true')
args = parser.parse_args()
obj_transport = get_transport(args.mode.lower(), args.name)
if obj_transport is None:
print("---- Failed to establish connection ----")
print('---- Failed to establish connection ----')
exit(1)
# If security version not specified check in capabilities
@@ -394,107 +395,107 @@ if __name__ == '__main__':
# First check if capabilities are supported or not
if not has_capability(obj_transport):
print('Security capabilities could not be determined. Please specify "--sec_ver" explicitly')
print("---- Invalid Security Version ----")
print('---- Invalid Security Version ----')
exit(2)
# When no_sec is present, use security 0, else security 1
args.secver = int(not has_capability(obj_transport, 'no_sec'))
print("Security scheme determined to be :", args.secver)
print('Security scheme determined to be :', args.secver)
if (args.secver != 0) and not has_capability(obj_transport, 'no_pop'):
if len(args.pop) == 0:
print("---- Proof of Possession argument not provided ----")
print('---- Proof of Possession argument not provided ----')
exit(2)
elif len(args.pop) != 0:
print("---- Proof of Possession will be ignored ----")
print('---- Proof of Possession will be ignored ----')
args.pop = ''
obj_security = get_security(args.secver, args.pop, args.verbose)
if obj_security is None:
print("---- Invalid Security Version ----")
print('---- Invalid Security Version ----')
exit(2)
if args.version != '':
print("\n==== Verifying protocol version ====")
print('\n==== Verifying protocol version ====')
if not version_match(obj_transport, args.version, args.verbose):
print("---- Error in protocol version matching ----")
print('---- Error in protocol version matching ----')
exit(3)
print("==== Verified protocol version successfully ====")
print('==== Verified protocol version successfully ====')
print("\n==== Starting Session ====")
print('\n==== Starting Session ====')
if not establish_session(obj_transport, obj_security):
print("Failed to establish session. Ensure that security scheme and proof of possession are correct")
print("---- Error in establishing session ----")
print('Failed to establish session. Ensure that security scheme and proof of possession are correct')
print('---- Error in establishing session ----')
exit(4)
print("==== Session Established ====")
print('==== Session Established ====')
if args.custom_config:
print("\n==== Sending Custom config to esp32 ====")
print('\n==== Sending Custom config to esp32 ====')
if not custom_config(obj_transport, obj_security, args.custom_info, args.custom_ver):
print("---- Error in custom config ----")
print('---- Error in custom config ----')
exit(5)
print("==== Custom config sent successfully ====")
print('==== Custom config sent successfully ====')
if args.custom_data != '':
print("\n==== Sending Custom data to esp32 ====")
print('\n==== Sending Custom data to esp32 ====')
if not custom_data(obj_transport, obj_security, args.custom_data):
print("---- Error in custom data ----")
print('---- Error in custom data ----')
exit(5)
print("==== Custom data sent successfully ====")
print('==== Custom data sent successfully ====')
if args.ssid == '':
if not has_capability(obj_transport, 'wifi_scan'):
print("---- Wi-Fi Scan List is not supported by provisioning service ----")
print("---- Rerun esp_prov with SSID and Passphrase as argument ----")
print('---- Wi-Fi Scan List is not supported by provisioning service ----')
print('---- Rerun esp_prov with SSID and Passphrase as argument ----')
exit(3)
while True:
print("\n==== Scanning Wi-Fi APs ====")
print('\n==== Scanning Wi-Fi APs ====')
start_time = time.time()
APs = scan_wifi_APs(args.mode.lower(), obj_transport, obj_security)
end_time = time.time()
print("\n++++ Scan finished in " + str(end_time - start_time) + " sec")
print('\n++++ Scan finished in ' + str(end_time - start_time) + ' sec')
if APs is None:
print("---- Error in scanning Wi-Fi APs ----")
print('---- Error in scanning Wi-Fi APs ----')
exit(8)
if len(APs) == 0:
print("No APs found!")
print('No APs found!')
exit(9)
print("==== Wi-Fi Scan results ====")
print("{0: >4} {1: <33} {2: <12} {3: >4} {4: <4} {5: <16}".format(
"S.N.", "SSID", "BSSID", "CHN", "RSSI", "AUTH"))
print('==== Wi-Fi Scan results ====')
print('{0: >4} {1: <33} {2: <12} {3: >4} {4: <4} {5: <16}'.format(
'S.N.', 'SSID', 'BSSID', 'CHN', 'RSSI', 'AUTH'))
for i in range(len(APs)):
print("[{0: >2}] {1: <33} {2: <12} {3: >4} {4: <4} {5: <16}".format(
i + 1, APs[i]["ssid"], APs[i]["bssid"], APs[i]["channel"], APs[i]["rssi"], APs[i]["auth"]))
print('[{0: >2}] {1: <33} {2: <12} {3: >4} {4: <4} {5: <16}'.format(
i + 1, APs[i]['ssid'], APs[i]['bssid'], APs[i]['channel'], APs[i]['rssi'], APs[i]['auth']))
while True:
try:
select = int(binput("Select AP by number (0 to rescan) : "))
select = int(binput('Select AP by number (0 to rescan) : '))
if select < 0 or select > len(APs):
raise ValueError
break
except ValueError:
print("Invalid input! Retry")
print('Invalid input! Retry')
if select != 0:
break
args.ssid = APs[select - 1]["ssid"]
prompt_str = "Enter passphrase for {0} : ".format(args.ssid)
args.ssid = APs[select - 1]['ssid']
prompt_str = 'Enter passphrase for {0} : '.format(args.ssid)
args.passphrase = getpass(prompt_str)
print("\n==== Sending Wi-Fi credential to esp32 ====")
print('\n==== Sending Wi-Fi credential to esp32 ====')
if not send_wifi_config(obj_transport, obj_security, args.ssid, args.passphrase):
print("---- Error in send Wi-Fi config ----")
print('---- Error in send Wi-Fi config ----')
exit(6)
print("==== Wi-Fi Credentials sent successfully ====")
print('==== Wi-Fi Credentials sent successfully ====')
print("\n==== Applying config to esp32 ====")
print('\n==== Applying config to esp32 ====')
if not apply_wifi_config(obj_transport, obj_security):
print("---- Error in apply Wi-Fi config ----")
print('---- Error in apply Wi-Fi config ----')
exit(7)
print("==== Apply config sent successfully ====")
print('==== Apply config sent successfully ====')
wait_wifi_connected(obj_transport, obj_security)

18
tools/esp_prov/proto/__init__.py
View File

@@ -29,16 +29,16 @@ def _load_source(name, path):
idf_path = os.environ['IDF_PATH']
# protocomm component related python files generated from .proto files
constants_pb2 = _load_source("constants_pb2", idf_path + "/components/protocomm/python/constants_pb2.py")
sec0_pb2 = _load_source("sec0_pb2", idf_path + "/components/protocomm/python/sec0_pb2.py")
sec1_pb2 = _load_source("sec1_pb2", idf_path + "/components/protocomm/python/sec1_pb2.py")
session_pb2 = _load_source("session_pb2", idf_path + "/components/protocomm/python/session_pb2.py")
constants_pb2 = _load_source('constants_pb2', idf_path + '/components/protocomm/python/constants_pb2.py')
sec0_pb2 = _load_source('sec0_pb2', idf_path + '/components/protocomm/python/sec0_pb2.py')
sec1_pb2 = _load_source('sec1_pb2', idf_path + '/components/protocomm/python/sec1_pb2.py')
session_pb2 = _load_source('session_pb2', idf_path + '/components/protocomm/python/session_pb2.py')
# wifi_provisioning component related python files generated from .proto files
wifi_constants_pb2 = _load_source("wifi_constants_pb2", idf_path + "/components/wifi_provisioning/python/wifi_constants_pb2.py")
wifi_config_pb2 = _load_source("wifi_config_pb2", idf_path + "/components/wifi_provisioning/python/wifi_config_pb2.py")
wifi_scan_pb2 = _load_source("wifi_scan_pb2", idf_path + "/components/wifi_provisioning/python/wifi_scan_pb2.py")
wifi_constants_pb2 = _load_source('wifi_constants_pb2', idf_path + '/components/wifi_provisioning/python/wifi_constants_pb2.py')
wifi_config_pb2 = _load_source('wifi_config_pb2', idf_path + '/components/wifi_provisioning/python/wifi_config_pb2.py')
wifi_scan_pb2 = _load_source('wifi_scan_pb2', idf_path + '/components/wifi_provisioning/python/wifi_scan_pb2.py')
# custom_provisioning component related python files generated from .proto files
custom_config_pb2 = _load_source("custom_config_pb2", idf_path +
"/examples/provisioning/legacy/custom_config/components/custom_provisioning/python/custom_config_pb2.py")
custom_config_pb2 = _load_source('custom_config_pb2', idf_path +
'/examples/provisioning/legacy/custom_config/components/custom_provisioning/python/custom_config_pb2.py')

4
tools/esp_prov/prov/__init__.py
View File

@@ -13,6 +13,6 @@
# limitations under the License.
#
from .wifi_prov import * # noqa F403
from .custom_prov import * # noqa F403
from .wifi_scan import * # noqa F403
from .wifi_prov import * # noqa F403
from .wifi_scan import * # noqa F403

14
tools/esp_prov/prov/custom_prov.py
View File

@@ -16,15 +16,15 @@
# APIs for interpreting and creating protobuf packets for `custom-config` protocomm endpoint
from __future__ import print_function
from future.utils import tobytes
import utils
import proto
import utils
from future.utils import tobytes
def print_verbose(security_ctx, data):
if (security_ctx.verbose):
print("++++ " + data + " ++++")
print('++++ ' + data + ' ++++')
def custom_config_request(security_ctx, info, version):
@@ -33,7 +33,7 @@ def custom_config_request(security_ctx, info, version):
cmd.info = tobytes(info)
cmd.version = version
enc_cmd = security_ctx.encrypt_data(cmd.SerializeToString()).decode('latin-1')
print_verbose(security_ctx, "Client -> Device (CustomConfig cmd) " + utils.str_to_hexstr(enc_cmd))
print_verbose(security_ctx, 'Client -> Device (CustomConfig cmd) ' + utils.str_to_hexstr(enc_cmd))
return enc_cmd
@@ -42,19 +42,19 @@ def custom_config_response(security_ctx, response_data):
decrypt = security_ctx.decrypt_data(tobytes(response_data))
cmd_resp = proto.custom_config_pb2.CustomConfigResponse()
cmd_resp.ParseFromString(decrypt)
print_verbose(security_ctx, "CustomConfig status " + str(cmd_resp.status))
print_verbose(security_ctx, 'CustomConfig status ' + str(cmd_resp.status))
return cmd_resp.status
def custom_data_request(security_ctx, data):
# Encrypt the custom data
enc_cmd = security_ctx.encrypt_data(tobytes(data))
print_verbose(security_ctx, "Client -> Device (CustomData cmd) " + utils.str_to_hexstr(enc_cmd))
print_verbose(security_ctx, 'Client -> Device (CustomData cmd) ' + utils.str_to_hexstr(enc_cmd))
return enc_cmd
def custom_data_response(security_ctx, response_data):
# Decrypt response packet
decrypt = security_ctx.decrypt_data(tobytes(response_data))
print("CustomData response: " + str(decrypt))
print('CustomData response: ' + str(decrypt))
return 0

42
tools/esp_prov/prov/wifi_prov.py
View File

@@ -16,15 +16,15 @@
# APIs for interpreting and creating protobuf packets for Wi-Fi provisioning
from __future__ import print_function
from future.utils import tobytes
import utils
import proto
import utils
from future.utils import tobytes
def print_verbose(security_ctx, data):
if (security_ctx.verbose):
print("++++ " + data + " ++++")
print('++++ ' + data + ' ++++')
def config_get_status_request(security_ctx):
@@ -34,7 +34,7 @@ def config_get_status_request(security_ctx):
cmd_get_status = proto.wifi_config_pb2.CmdGetStatus()
cfg1.cmd_get_status.MergeFrom(cmd_get_status)
encrypted_cfg = security_ctx.encrypt_data(cfg1.SerializeToString()).decode('latin-1')
print_verbose(security_ctx, "Client -> Device (Encrypted CmdGetStatus) " + utils.str_to_hexstr(encrypted_cfg))
print_verbose(security_ctx, 'Client -> Device (Encrypted CmdGetStatus) ' + utils.str_to_hexstr(encrypted_cfg))
return encrypted_cfg
@@ -43,26 +43,26 @@ def config_get_status_response(security_ctx, response_data):
decrypted_message = security_ctx.decrypt_data(tobytes(response_data))
cmd_resp1 = proto.wifi_config_pb2.WiFiConfigPayload()
cmd_resp1.ParseFromString(decrypted_message)
print_verbose(security_ctx, "Response type " + str(cmd_resp1.msg))
print_verbose(security_ctx, "Response status " + str(cmd_resp1.resp_get_status.status))
print_verbose(security_ctx, 'Response type ' + str(cmd_resp1.msg))
print_verbose(security_ctx, 'Response status ' + str(cmd_resp1.resp_get_status.status))
if cmd_resp1.resp_get_status.sta_state == 0:
print("++++ WiFi state: " + "connected ++++")
return "connected"
print('++++ WiFi state: ' + 'connected ++++')
return 'connected'
elif cmd_resp1.resp_get_status.sta_state == 1:
print("++++ WiFi state: " + "connecting... ++++")
return "connecting"
print('++++ WiFi state: ' + 'connecting... ++++')
return 'connecting'
elif cmd_resp1.resp_get_status.sta_state == 2:
print("++++ WiFi state: " + "disconnected ++++")
return "disconnected"
print('++++ WiFi state: ' + 'disconnected ++++')
return 'disconnected'
elif cmd_resp1.resp_get_status.sta_state == 3:
print("++++ WiFi state: " + "connection failed ++++")
print('++++ WiFi state: ' + 'connection failed ++++')
if cmd_resp1.resp_get_status.fail_reason == 0:
print("++++ Failure reason: " + "Incorrect Password ++++")
print('++++ Failure reason: ' + 'Incorrect Password ++++')
elif cmd_resp1.resp_get_status.fail_reason == 1:
print("++++ Failure reason: " + "Incorrect SSID ++++")
return "failed"
return "unknown"
print('++++ Failure reason: ' + 'Incorrect SSID ++++')
return 'failed'
return 'unknown'
def config_set_config_request(security_ctx, ssid, passphrase):
@@ -72,7 +72,7 @@ def config_set_config_request(security_ctx, ssid, passphrase):
cmd.cmd_set_config.ssid = tobytes(ssid)
cmd.cmd_set_config.passphrase = tobytes(passphrase)
enc_cmd = security_ctx.encrypt_data(cmd.SerializeToString()).decode('latin-1')
print_verbose(security_ctx, "Client -> Device (SetConfig cmd) " + utils.str_to_hexstr(enc_cmd))
print_verbose(security_ctx, 'Client -> Device (SetConfig cmd) ' + utils.str_to_hexstr(enc_cmd))
return enc_cmd
@@ -81,7 +81,7 @@ def config_set_config_response(security_ctx, response_data):
decrypt = security_ctx.decrypt_data(tobytes(response_data))
cmd_resp4 = proto.wifi_config_pb2.WiFiConfigPayload()
cmd_resp4.ParseFromString(decrypt)
print_verbose(security_ctx, "SetConfig status " + str(cmd_resp4.resp_set_config.status))
print_verbose(security_ctx, 'SetConfig status ' + str(cmd_resp4.resp_set_config.status))
return cmd_resp4.resp_set_config.status
@@ -90,7 +90,7 @@ def config_apply_config_request(security_ctx):
cmd = proto.wifi_config_pb2.WiFiConfigPayload()
cmd.msg = proto.wifi_config_pb2.TypeCmdApplyConfig
enc_cmd = security_ctx.encrypt_data(cmd.SerializeToString()).decode('latin-1')
print_verbose(security_ctx, "Client -> Device (ApplyConfig cmd) " + utils.str_to_hexstr(enc_cmd))
print_verbose(security_ctx, 'Client -> Device (ApplyConfig cmd) ' + utils.str_to_hexstr(enc_cmd))
return enc_cmd
@@ -99,5 +99,5 @@ def config_apply_config_response(security_ctx, response_data):
decrypt = security_ctx.decrypt_data(tobytes(response_data))
cmd_resp5 = proto.wifi_config_pb2.WiFiConfigPayload()
cmd_resp5.ParseFromString(decrypt)
print_verbose(security_ctx, "ApplyConfig status " + str(cmd_resp5.resp_apply_config.status))
print_verbose(security_ctx, 'ApplyConfig status ' + str(cmd_resp5.resp_apply_config.status))
return cmd_resp5.resp_apply_config.status

42
tools/esp_prov/prov/wifi_scan.py
View File

@@ -16,15 +16,15 @@
# APIs for interpreting and creating protobuf packets for Wi-Fi Scanning
from __future__ import print_function
from future.utils import tobytes
import utils
import proto
import utils
from future.utils import tobytes
def print_verbose(security_ctx, data):
if (security_ctx.verbose):
print("++++ " + data + " ++++")
print('++++ ' + data + ' ++++')
def scan_start_request(security_ctx, blocking=True, passive=False, group_channels=5, period_ms=120):
@@ -36,7 +36,7 @@ def scan_start_request(security_ctx, blocking=True, passive=False, group_channel
cmd.cmd_scan_start.group_channels = group_channels
cmd.cmd_scan_start.period_ms = period_ms
enc_cmd = security_ctx.encrypt_data(cmd.SerializeToString()).decode('latin-1')
print_verbose(security_ctx, "Client -> Device (Encrypted CmdScanStart) " + utils.str_to_hexstr(enc_cmd))
print_verbose(security_ctx, 'Client -> Device (Encrypted CmdScanStart) ' + utils.str_to_hexstr(enc_cmd))
return enc_cmd
@@ -45,7 +45,7 @@ def scan_start_response(security_ctx, response_data):
dec_resp = security_ctx.decrypt_data(tobytes(response_data))
resp = proto.wifi_scan_pb2.WiFiScanPayload()
resp.ParseFromString(dec_resp)
print_verbose(security_ctx, "ScanStart status " + str(resp.status))
print_verbose(security_ctx, 'ScanStart status ' + str(resp.status))
if resp.status != 0:
raise RuntimeError
@@ -55,7 +55,7 @@ def scan_status_request(security_ctx):
cmd = proto.wifi_scan_pb2.WiFiScanPayload()
cmd.msg = proto.wifi_scan_pb2.TypeCmdScanStatus
enc_cmd = security_ctx.encrypt_data(cmd.SerializeToString()).decode('latin-1')
print_verbose(security_ctx, "Client -> Device (Encrypted CmdScanStatus) " + utils.str_to_hexstr(enc_cmd))
print_verbose(security_ctx, 'Client -> Device (Encrypted CmdScanStatus) ' + utils.str_to_hexstr(enc_cmd))
return enc_cmd
@@ -64,10 +64,10 @@ def scan_status_response(security_ctx, response_data):
dec_resp = security_ctx.decrypt_data(tobytes(response_data))
resp = proto.wifi_scan_pb2.WiFiScanPayload()
resp.ParseFromString(dec_resp)
print_verbose(security_ctx, "ScanStatus status " + str(resp.status))
print_verbose(security_ctx, 'ScanStatus status ' + str(resp.status))
if resp.status != 0:
raise RuntimeError
return {"finished": resp.resp_scan_status.scan_finished, "count": resp.resp_scan_status.result_count}
return {'finished': resp.resp_scan_status.scan_finished, 'count': resp.resp_scan_status.result_count}
def scan_result_request(security_ctx, index, count):
@@ -77,7 +77,7 @@ def scan_result_request(security_ctx, index, count):
cmd.cmd_scan_result.start_index = index
cmd.cmd_scan_result.count = count
enc_cmd = security_ctx.encrypt_data(cmd.SerializeToString()).decode('latin-1')
print_verbose(security_ctx, "Client -> Device (Encrypted CmdScanResult) " + utils.str_to_hexstr(enc_cmd))
print_verbose(security_ctx, 'Client -> Device (Encrypted CmdScanResult) ' + utils.str_to_hexstr(enc_cmd))
return enc_cmd
@@ -86,20 +86,20 @@ def scan_result_response(security_ctx, response_data):
dec_resp = security_ctx.decrypt_data(tobytes(response_data))
resp = proto.wifi_scan_pb2.WiFiScanPayload()
resp.ParseFromString(dec_resp)
print_verbose(security_ctx, "ScanResult status " + str(resp.status))
print_verbose(security_ctx, 'ScanResult status ' + str(resp.status))
if resp.status != 0:
raise RuntimeError
authmode_str = ["Open", "WEP", "WPA_PSK", "WPA2_PSK", "WPA_WPA2_PSK", "WPA2_ENTERPRISE"]
authmode_str = ['Open', 'WEP', 'WPA_PSK', 'WPA2_PSK', 'WPA_WPA2_PSK', 'WPA2_ENTERPRISE']
results = []
for entry in resp.resp_scan_result.entries:
results += [{"ssid": entry.ssid.decode('latin-1').rstrip('\x00'),
"bssid": utils.str_to_hexstr(entry.bssid.decode('latin-1')),
"channel": entry.channel,
"rssi": entry.rssi,
"auth": authmode_str[entry.auth]}]
print_verbose(security_ctx, "ScanResult SSID : " + str(results[-1]["ssid"]))
print_verbose(security_ctx, "ScanResult BSSID : " + str(results[-1]["bssid"]))
print_verbose(security_ctx, "ScanResult Channel : " + str(results[-1]["channel"]))
print_verbose(security_ctx, "ScanResult RSSI : " + str(results[-1]["rssi"]))
print_verbose(security_ctx, "ScanResult AUTH : " + str(results[-1]["auth"]))
results += [{'ssid': entry.ssid.decode('latin-1').rstrip('\x00'),
'bssid': utils.str_to_hexstr(entry.bssid.decode('latin-1')),
'channel': entry.channel,
'rssi': entry.rssi,
'auth': authmode_str[entry.auth]}]
print_verbose(security_ctx, 'ScanResult SSID : ' + str(results[-1]['ssid']))
print_verbose(security_ctx, 'ScanResult BSSID : ' + str(results[-1]['bssid']))
print_verbose(security_ctx, 'ScanResult Channel : ' + str(results[-1]['channel']))
print_verbose(security_ctx, 'ScanResult RSSI : ' + str(results[-1]['rssi']))
print_verbose(security_ctx, 'ScanResult AUTH : ' + str(results[-1]['auth']))
return results

5
tools/esp_prov/security/security0.py
View File

@@ -17,9 +17,10 @@
# protocomm endpoint with security type protocomm_security0
from __future__ import print_function
from future.utils import tobytes
import proto
from future.utils import tobytes
from .security import Security
@@ -54,7 +55,7 @@ class Security0(Security):
setup_resp.ParseFromString(tobytes(response_data))
# Check if security scheme matches
if setup_resp.sec_ver != proto.session_pb2.SecScheme0:
print("Incorrect sec scheme")
print('Incorrect sec scheme')
def encrypt_data(self, data):
# Passive. No encryption when security0 used

37
tools/esp_prov/security/security1.py
View File

@@ -17,18 +17,17 @@
# protocomm endpoint with security type protocomm_security1
from __future__ import print_function
from future.utils import tobytes
import utils
import proto
from .security import Security
import session_pb2
import utils
from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.primitives import hashes, serialization
from cryptography.hazmat.primitives.asymmetric.x25519 import X25519PrivateKey, X25519PublicKey
from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
from future.utils import tobytes
import session_pb2
from .security import Security
# Enum for state of protocomm_security1 FSM
@@ -76,7 +75,7 @@ class Security1(Security):
self.setup1_response(response_data)
return None
else:
print("Unexpected state")
print('Unexpected state')
return None
def __generate_key(self):
@@ -92,7 +91,7 @@ class Security1(Security):
def _print_verbose(self, data):
if (self.verbose):
print("++++ " + data + " ++++")
print('++++ ' + data + ' ++++')
def setup0_request(self):
# Form SessionCmd0 request packet using client public key
@@ -100,26 +99,26 @@ class Security1(Security):
setup_req.sec_ver = session_pb2.SecScheme1
self.__generate_key()
setup_req.sec1.sc0.client_pubkey = self.client_public_key
self._print_verbose("Client Public Key:\t" + utils.str_to_hexstr(self.client_public_key.decode('latin-1')))
self._print_verbose('Client Public Key:\t' + utils.str_to_hexstr(self.client_public_key.decode('latin-1')))
return setup_req.SerializeToString().decode('latin-1')
def setup0_response(self, response_data):
# Interpret SessionResp0 response packet
setup_resp = proto.session_pb2.SessionData()
setup_resp.ParseFromString(tobytes(response_data))
self._print_verbose("Security version:\t" + str(setup_resp.sec_ver))
self._print_verbose('Security version:\t' + str(setup_resp.sec_ver))
if setup_resp.sec_ver != session_pb2.SecScheme1:
print("Incorrect sec scheme")
print('Incorrect sec scheme')
exit(1)
self.device_public_key = setup_resp.sec1.sr0.device_pubkey
# Device random is the initialization vector
device_random = setup_resp.sec1.sr0.device_random
self._print_verbose("Device Public Key:\t" + utils.str_to_hexstr(self.device_public_key.decode('latin-1')))
self._print_verbose("Device Random:\t" + utils.str_to_hexstr(device_random.decode('latin-1')))
self._print_verbose('Device Public Key:\t' + utils.str_to_hexstr(self.device_public_key.decode('latin-1')))
self._print_verbose('Device Random:\t' + utils.str_to_hexstr(device_random.decode('latin-1')))
# Calculate Curve25519 shared key using Client private key and Device public key
sharedK = self.client_private_key.exchange(X25519PublicKey.from_public_bytes(self.device_public_key))
self._print_verbose("Shared Key:\t" + utils.str_to_hexstr(sharedK.decode('latin-1')))
self._print_verbose('Shared Key:\t' + utils.str_to_hexstr(sharedK.decode('latin-1')))
# If PoP is provided, XOR SHA256 of PoP with the previously
# calculated Shared Key to form the actual Shared Key
@@ -130,7 +129,7 @@ class Security1(Security):
digest = h.finalize()
# XOR with and update Shared Key
sharedK = xor(sharedK, digest)
self._print_verbose("New Shared Key XORed with PoP:\t" + utils.str_to_hexstr(sharedK.decode('latin-1')))
self._print_verbose('New Shared Key XORed with PoP:\t' + utils.str_to_hexstr(sharedK.decode('latin-1')))
# Initialize the encryption engine with Shared Key and initialization vector
cipher = Cipher(algorithms.AES(sharedK), modes.CTR(device_random), backend=default_backend())
self.cipher = cipher.encryptor()
@@ -142,7 +141,7 @@ class Security1(Security):
setup_req.sec1.msg = proto.sec1_pb2.Session_Command1
# Encrypt device public key and attach to the request packet
client_verify = self.cipher.update(self.device_public_key)
self._print_verbose("Client Verify:\t" + utils.str_to_hexstr(client_verify.decode('latin-1')))
self._print_verbose('Client Verify:\t' + utils.str_to_hexstr(client_verify.decode('latin-1')))
setup_req.sec1.sc1.client_verify_data = client_verify
return setup_req.SerializeToString().decode('latin-1')
@@ -154,16 +153,16 @@ class Security1(Security):
if setup_resp.sec_ver == session_pb2.SecScheme1:
# Read encrypyed device verify string
device_verify = setup_resp.sec1.sr1.device_verify_data
self._print_verbose("Device verify:\t" + utils.str_to_hexstr(device_verify.decode('latin-1')))
self._print_verbose('Device verify:\t' + utils.str_to_hexstr(device_verify.decode('latin-1')))
# Decrypt the device verify string
enc_client_pubkey = self.cipher.update(setup_resp.sec1.sr1.device_verify_data)
self._print_verbose("Enc client pubkey:\t " + utils.str_to_hexstr(enc_client_pubkey.decode('latin-1')))
self._print_verbose('Enc client pubkey:\t ' + utils.str_to_hexstr(enc_client_pubkey.decode('latin-1')))
# Match decryped string with client public key
if enc_client_pubkey != self.client_public_key:
print("Mismatch in device verify")
print('Mismatch in device verify')
return -2
else:
print("Unsupported security protocol")
print('Unsupported security protocol')
return -1
def encrypt_data(self, data):

4
tools/esp_prov/transport/__init__.py
View File

@@ -13,6 +13,6 @@
# limitations under the License.
#
from .transport_ble import * # noqa: F403, F401
from .transport_console import * # noqa: F403, F401
from .transport_http import * # noqa: F403, F401
from .transport_ble import * # noqa: F403, F401
from .transport_http import * # noqa: F403, F401

81
tools/esp_prov/transport/ble_cli.py
View File

@@ -14,12 +14,12 @@
#
from __future__ import print_function
from builtins import input
from future.utils import iteritems
import platform
from builtins import input
import utils
from future.utils import iteritems
fallback = True
@@ -28,9 +28,10 @@ fallback = True
# else fallback to console mode
if platform.system() == 'Linux':
try:
import time
import dbus
import dbus.mainloop.glib
import time
fallback = False
except ImportError:
pass
@@ -55,33 +56,33 @@ class BLE_Bluez_Client:
dbus.mainloop.glib.DBusGMainLoop(set_as_default=True)
bus = dbus.SystemBus()
manager = dbus.Interface(bus.get_object("org.bluez", "/"), "org.freedesktop.DBus.ObjectManager")
manager = dbus.Interface(bus.get_object('org.bluez', '/'), 'org.freedesktop.DBus.ObjectManager')
objects = manager.GetManagedObjects()
for path, interfaces in iteritems(objects):
adapter = interfaces.get("org.bluez.Adapter1")
adapter = interfaces.get('org.bluez.Adapter1')
if adapter is not None:
if path.endswith(iface):
self.adapter = dbus.Interface(bus.get_object("org.bluez", path), "org.bluez.Adapter1")
self.adapter_props = dbus.Interface(bus.get_object("org.bluez", path), "org.freedesktop.DBus.Properties")
self.adapter = dbus.Interface(bus.get_object('org.bluez', path), 'org.bluez.Adapter1')
self.adapter_props = dbus.Interface(bus.get_object('org.bluez', path), 'org.freedesktop.DBus.Properties')
break
if self.adapter is None:
raise RuntimeError("Bluetooth adapter not found")
raise RuntimeError('Bluetooth adapter not found')
self.adapter_props.Set("org.bluez.Adapter1", "Powered", dbus.Boolean(1))
self.adapter_props.Set('org.bluez.Adapter1', 'Powered', dbus.Boolean(1))
self.adapter.StartDiscovery()
retry = 10
while (retry > 0):
try:
if self.device is None:
print("Connecting...")
print('Connecting...')
# Wait for device to be discovered
time.sleep(5)
self._connect_()
print("Connected")
print("Getting Services...")
print('Connected')
print('Getting Services...')
# Wait for services to be discovered
time.sleep(5)
self._get_services_()
@@ -89,28 +90,28 @@ class BLE_Bluez_Client:
except Exception as e:
print(e)
retry -= 1
print("Retries left", retry)
print('Retries left', retry)
continue
self.adapter.StopDiscovery()
return False
def _connect_(self):
bus = dbus.SystemBus()
manager = dbus.Interface(bus.get_object("org.bluez", "/"), "org.freedesktop.DBus.ObjectManager")
manager = dbus.Interface(bus.get_object('org.bluez', '/'), 'org.freedesktop.DBus.ObjectManager')
objects = manager.GetManagedObjects()
dev_path = None
for path, interfaces in iteritems(objects):
if "org.bluez.Device1" not in interfaces:
if 'org.bluez.Device1' not in interfaces:
continue
if interfaces["org.bluez.Device1"].get("Name") == self.devname:
if interfaces['org.bluez.Device1'].get('Name') == self.devname:
dev_path = path
break
if dev_path is None:
raise RuntimeError("BLE device not found")
raise RuntimeError('BLE device not found')
try:
self.device = bus.get_object("org.bluez", dev_path)
self.device = bus.get_object('org.bluez', dev_path)
try:
uuids = self.device.Get('org.bluez.Device1', 'UUIDs',
dbus_interface='org.freedesktop.DBus.Properties')
@@ -128,19 +129,19 @@ class BLE_Bluez_Client:
except Exception as e:
print(e)
self.device = None
raise RuntimeError("BLE device could not connect")
raise RuntimeError('BLE device could not connect')
def _get_services_(self):
bus = dbus.SystemBus()
manager = dbus.Interface(bus.get_object("org.bluez", "/"), "org.freedesktop.DBus.ObjectManager")
manager = dbus.Interface(bus.get_object('org.bluez', '/'), 'org.freedesktop.DBus.ObjectManager')
objects = manager.GetManagedObjects()
service_found = False
for srv_path, srv_interfaces in iteritems(objects):
if "org.bluez.GattService1" not in srv_interfaces:
if 'org.bluez.GattService1' not in srv_interfaces:
continue
if not srv_path.startswith(self.device.object_path):
continue
service = bus.get_object("org.bluez", srv_path)
service = bus.get_object('org.bluez', srv_path)
srv_uuid = service.Get('org.bluez.GattService1', 'UUID',
dbus_interface='org.freedesktop.DBus.Properties')
@@ -152,20 +153,20 @@ class BLE_Bluez_Client:
nu_lookup = dict()
characteristics = dict()
for chrc_path, chrc_interfaces in iteritems(objects):
if "org.bluez.GattCharacteristic1" not in chrc_interfaces:
if 'org.bluez.GattCharacteristic1' not in chrc_interfaces:
continue
if not chrc_path.startswith(service.object_path):
continue
chrc = bus.get_object("org.bluez", chrc_path)
chrc = bus.get_object('org.bluez', chrc_path)
uuid = chrc.Get('org.bluez.GattCharacteristic1', 'UUID',
dbus_interface='org.freedesktop.DBus.Properties')
characteristics[uuid] = chrc
for desc_path, desc_interfaces in iteritems(objects):
if "org.bluez.GattDescriptor1" not in desc_interfaces:
if 'org.bluez.GattDescriptor1' not in desc_interfaces:
continue
if not desc_path.startswith(chrc.object_path):
continue
desc = bus.get_object("org.bluez", desc_path)
desc = bus.get_object('org.bluez', desc_path)
desc_uuid = desc.Get('org.bluez.GattDescriptor1', 'UUID',
dbus_interface='org.freedesktop.DBus.Properties')
if desc_uuid[4:8] != '2901':
@@ -205,7 +206,7 @@ class BLE_Bluez_Client:
self.device = None
self.nu_lookup = None
self.characteristics = dict()
raise RuntimeError("Provisioning service not found")
raise RuntimeError('Provisioning service not found')
def get_nu_lookup(self):
return self.nu_lookup
@@ -224,25 +225,25 @@ class BLE_Bluez_Client:
self.nu_lookup = None
self.characteristics = dict()
if self.adapter_props:
self.adapter_props.Set("org.bluez.Adapter1", "Powered", dbus.Boolean(0))
self.adapter_props.Set('org.bluez.Adapter1', 'Powered', dbus.Boolean(0))
def send_data(self, characteristic_uuid, data):
try:
path = self.characteristics[characteristic_uuid]
except KeyError:
raise RuntimeError("Invalid characteristic : " + characteristic_uuid)
raise RuntimeError('Invalid characteristic : ' + characteristic_uuid)
try:
path.WriteValue([ord(c) for c in data], {}, dbus_interface='org.bluez.GattCharacteristic1')
except TypeError: # python3 compatible
path.WriteValue([c for c in data], {}, dbus_interface='org.bluez.GattCharacteristic1')
except dbus.exceptions.DBusException as e:
raise RuntimeError("Failed to write value to characteristic " + characteristic_uuid + ": " + str(e))
raise RuntimeError('Failed to write value to characteristic ' + characteristic_uuid + ': ' + str(e))
try:
readval = path.ReadValue({}, dbus_interface='org.bluez.GattCharacteristic1')
except dbus.exceptions.DBusException as e:
raise RuntimeError("Failed to read value from characteristic " + characteristic_uuid + ": " + str(e))
raise RuntimeError('Failed to read value from characteristic ' + characteristic_uuid + ': ' + str(e))
return ''.join(chr(b) for b in readval)
@@ -252,14 +253,14 @@ class BLE_Bluez_Client:
# Console based BLE client for Cross Platform support
class BLE_Console_Client:
def connect(self, devname, iface, chrc_names, fallback_srv_uuid):
print("BLE client is running in console mode")
print("\tThis could be due to your platform not being supported or dependencies not being met")
print("\tPlease ensure all pre-requisites are met to run the full fledged client")
print("BLECLI >> Please connect to BLE device `" + devname + "` manually using your tool of choice")
resp = input("BLECLI >> Was the device connected successfully? [y/n] ")
print('BLE client is running in console mode')
print('\tThis could be due to your platform not being supported or dependencies not being met')
print('\tPlease ensure all pre-requisites are met to run the full fledged client')
print('BLECLI >> Please connect to BLE device `' + devname + '` manually using your tool of choice')
resp = input('BLECLI >> Was the device connected successfully? [y/n] ')
if resp != 'Y' and resp != 'y':
return False
print("BLECLI >> List available attributes of the connected device")
print('BLECLI >> List available attributes of the connected device')
resp = input("BLECLI >> Is the service UUID '" + fallback_srv_uuid + "' listed among available attributes? [y/n] ")
if resp != 'Y' and resp != 'y':
return False
@@ -279,9 +280,9 @@ class BLE_Console_Client:
def send_data(self, characteristic_uuid, data):
print("BLECLI >> Write following data to characteristic with UUID '" + characteristic_uuid + "' :")
print("\t>> " + utils.str_to_hexstr(data))
print("BLECLI >> Enter data read from characteristic (in hex) :")
resp = input("\t<< ")
print('\t>> ' + utils.str_to_hexstr(data))
print('BLECLI >> Enter data read from characteristic (in hex) :')
resp = input('\t<< ')
return utils.hexstr_to_str(resp)

7
tools/esp_prov/transport/transport_ble.py
View File

@@ -15,9 +15,8 @@
from __future__ import print_function
from .transport import Transport
from . import ble_cli
from .transport import Transport
class Transport_BLE(Transport):
@@ -35,7 +34,7 @@ class Transport_BLE(Transport):
if not self.cli.connect(devname=devname, iface='hci0',
chrc_names=nu_lookup.keys(),
fallback_srv_uuid=service_uuid):
raise RuntimeError("Failed to initialize transport")
raise RuntimeError('Failed to initialize transport')
# Irrespective of provided parameters, let the client
# generate a lookup table by reading advertisement data
@@ -63,5 +62,5 @@ class Transport_BLE(Transport):
def send_data(self, ep_name, data):
# Write (and read) data to characteristic corresponding to the endpoint
if ep_name not in self.name_uuid_lookup.keys():
raise RuntimeError("Invalid endpoint : " + ep_name)
raise RuntimeError('Invalid endpoint : ' + ep_name)
return self.cli.send_data(self.name_uuid_lookup[ep_name], data)

7
tools/esp_prov/transport/transport_console.py
View File

@@ -14,6 +14,7 @@
#
from __future__ import print_function
from builtins import input
import utils
@@ -24,10 +25,10 @@ from .transport import Transport
class Transport_Console(Transport):
def send_data(self, path, data, session_id=0):
print("Client->Device msg :", path, session_id, utils.str_to_hexstr(data))
print('Client->Device msg :', path, session_id, utils.str_to_hexstr(data))
try:
resp = input("Enter device->client msg : ")
resp = input('Enter device->client msg : ')
except Exception as err:
print("error:", err)
print('error:', err)
return None
return utils.hexstr_to_str(resp)

18
tools/esp_prov/transport/transport_http.py
View File

@@ -14,9 +14,11 @@
#
from __future__ import print_function
from future.utils import tobytes
import socket
from future.utils import tobytes
try:
from http.client import HTTPConnection, HTTPSConnection
except ImportError:
@@ -31,28 +33,28 @@ class Transport_HTTP(Transport):
try:
socket.gethostbyname(hostname.split(':')[0])
except socket.gaierror:
raise RuntimeError("Unable to resolve hostname :" + hostname)
raise RuntimeError('Unable to resolve hostname :' + hostname)
if ssl_context is None:
self.conn = HTTPConnection(hostname, timeout=45)
else:
self.conn = HTTPSConnection(hostname, context=ssl_context, timeout=45)
try:
print("Connecting to " + hostname)
print('Connecting to ' + hostname)
self.conn.connect()
except Exception as err:
raise RuntimeError("Connection Failure : " + str(err))
self.headers = {"Content-type": "application/x-www-form-urlencoded","Accept": "text/plain"}
raise RuntimeError('Connection Failure : ' + str(err))
self.headers = {'Content-type': 'application/x-www-form-urlencoded','Accept': 'text/plain'}
def _send_post_request(self, path, data):
try:
self.conn.request("POST", path, tobytes(data), self.headers)
self.conn.request('POST', path, tobytes(data), self.headers)
response = self.conn.getresponse()
if response.status == 200:
return response.read().decode('latin-1')
except Exception as err:
raise RuntimeError("Connection Failure : " + str(err))
raise RuntimeError("Server responded with error code " + str(response.status))
raise RuntimeError('Connection Failure : ' + str(err))
raise RuntimeError('Server responded with error code ' + str(response.status))
def send_data(self, ep_name, data):
return self._send_post_request('/' + ep_name, data)

1
tools/esp_prov/utils/convenience.py
View File

@@ -15,6 +15,7 @@
# Convenience functions for commonly used data type conversions
import binascii
from future.utils import tobytes

170
tools/find_apps.py
View File

@@ -11,28 +11,18 @@ import logging
import os
import re
import sys
import typing
from find_build_apps import (
BUILD_SYSTEMS,
BUILD_SYSTEM_CMAKE,
BuildSystem,
BuildItem,
setup_logging,
ConfigRule,
config_rules_from_str,
DEFAULT_TARGET,
)
from find_build_apps import (BUILD_SYSTEM_CMAKE, BUILD_SYSTEMS, DEFAULT_TARGET, BuildItem, BuildSystem, ConfigRule,
config_rules_from_str, setup_logging)
# Helper functions
def dict_from_sdkconfig(path):
"""
Parse the sdkconfig file at 'path', return name:value pairs as a dict
"""
regex = re.compile(r"^([^#=]+)=(.+)$")
regex = re.compile(r'^([^#=]+)=(.+)$')
result = {}
with open(path) as f:
for line in f:
@@ -67,7 +57,7 @@ def find_builds_for_app(app_path, work_dir, build_dir, build_log, target_arg,
:return: list of BuildItems representing build configuration of the app
"""
build_items = [] # type: typing.List[BuildItem]
default_config_name = ""
default_config_name = ''
for rule in config_rules:
if not rule.file_name:
@@ -80,17 +70,17 @@ def find_builds_for_app(app_path, work_dir, build_dir, build_log, target_arg,
# Check if the sdkconfig file specifies IDF_TARGET, and if it is matches the --target argument.
sdkconfig_dict = dict_from_sdkconfig(sdkconfig_path)
target_from_config = sdkconfig_dict.get("CONFIG_IDF_TARGET")
target_from_config = sdkconfig_dict.get('CONFIG_IDF_TARGET')
if target_from_config is not None and target_from_config != target_arg:
logging.debug("Skipping sdkconfig {} which requires target {}".format(
logging.debug('Skipping sdkconfig {} which requires target {}'.format(
sdkconfig_path, target_from_config))
continue
# Figure out the config name
config_name = rule.config_name or ""
if "*" in rule.file_name:
config_name = rule.config_name or ''
if '*' in rule.file_name:
# convert glob pattern into a regex
regex_str = r".*" + rule.file_name.replace(".", r"\.").replace("*", r"(.*)")
regex_str = r'.*' + rule.file_name.replace('.', r'\.').replace('*', r'(.*)')
groups = re.match(regex_str, sdkconfig_path)
assert groups
config_name = groups.group(1)
@@ -143,12 +133,12 @@ def find_apps(build_system_class, path, recursive, exclude_list, target):
:return: list of paths of the apps found
"""
build_system_name = build_system_class.NAME
logging.debug("Looking for {} apps in {}{}".format(build_system_name, path, " recursively" if recursive else ""))
logging.debug('Looking for {} apps in {}{}'.format(build_system_name, path, ' recursively' if recursive else ''))
if not recursive:
if exclude_list:
logging.warning("--exclude option is ignored when used without --recursive")
logging.warning('--exclude option is ignored when used without --recursive')
if not build_system_class.is_app(path):
logging.warning("Path {} specified without --recursive flag, but no {} app found there".format(
logging.warning('Path {} specified without --recursive flag, but no {} app found there'.format(
path, build_system_name))
return []
return [path]
@@ -156,14 +146,14 @@ def find_apps(build_system_class, path, recursive, exclude_list, target):
# The remaining part is for recursive == True
apps_found = [] # type: typing.List[str]
for root, dirs, _ in os.walk(path, topdown=True):
logging.debug("Entering {}".format(root))
logging.debug('Entering {}'.format(root))
if root in exclude_list:
logging.debug("Skipping {} (excluded)".format(root))
logging.debug('Skipping {} (excluded)'.format(root))
del dirs[:]
continue
if build_system_class.is_app(root):
logging.debug("Found {} app in {}".format(build_system_name, root))
logging.debug('Found {} app in {}'.format(build_system_name, root))
# Don't recurse into app subdirectories
del dirs[:]
@@ -172,88 +162,88 @@ def find_apps(build_system_class, path, recursive, exclude_list, target):
apps_found.append(root)
else:
if supported_targets:
logging.debug("Skipping, app only supports targets: " + ", ".join(supported_targets))
logging.debug('Skipping, app only supports targets: ' + ', '.join(supported_targets))
else:
logging.debug("Skipping, app has no supported targets")
logging.debug('Skipping, app has no supported targets')
continue
return apps_found
def main():
parser = argparse.ArgumentParser(description="Tool to generate build steps for IDF apps")
parser = argparse.ArgumentParser(description='Tool to generate build steps for IDF apps')
parser.add_argument(
"-v",
"--verbose",
action="count",
help="Increase the logging level of the script. Can be specified multiple times.",
'-v',
'--verbose',
action='count',
help='Increase the logging level of the script. Can be specified multiple times.',
)
parser.add_argument(
"--log-file",
type=argparse.FileType("w"),
help="Write the script log to the specified file, instead of stderr",
'--log-file',
type=argparse.FileType('w'),
help='Write the script log to the specified file, instead of stderr',
)
parser.add_argument(
"--recursive",
action="store_true",
help="Look for apps in the specified directories recursively.",
'--recursive',
action='store_true',
help='Look for apps in the specified directories recursively.',
)
parser.add_argument(
"--build-system",
'--build-system',
choices=BUILD_SYSTEMS.keys()
)
parser.add_argument(
"--work-dir",
help="If set, the app is first copied into the specified directory, and then built." +
"If not set, the work directory is the directory of the app.",
'--work-dir',
help='If set, the app is first copied into the specified directory, and then built.' +
'If not set, the work directory is the directory of the app.',
)
parser.add_argument(
"--config",
action="append",
help="Adds configurations (sdkconfig file names) to build. This can either be " +
"FILENAME[=NAME] or FILEPATTERN. FILENAME is the name of the sdkconfig file, " +
"relative to the project directory, to be used. Optional NAME can be specified, " +
"which can be used as a name of this configuration. FILEPATTERN is the name of " +
"the sdkconfig file, relative to the project directory, with at most one wildcard. " +
"The part captured by the wildcard is used as the name of the configuration.",
'--config',
action='append',
help='Adds configurations (sdkconfig file names) to build. This can either be ' +
'FILENAME[=NAME] or FILEPATTERN. FILENAME is the name of the sdkconfig file, ' +
'relative to the project directory, to be used. Optional NAME can be specified, ' +
'which can be used as a name of this configuration. FILEPATTERN is the name of ' +
'the sdkconfig file, relative to the project directory, with at most one wildcard. ' +
'The part captured by the wildcard is used as the name of the configuration.',
)
parser.add_argument(
"--build-dir",
help="If set, specifies the build directory name. Can expand placeholders. Can be either a " +
"name relative to the work directory, or an absolute path.",
'--build-dir',
help='If set, specifies the build directory name. Can expand placeholders. Can be either a ' +
'name relative to the work directory, or an absolute path.',
)
parser.add_argument(
"--build-log",
help="If specified, the build log will be written to this file. Can expand placeholders.",
'--build-log',
help='If specified, the build log will be written to this file. Can expand placeholders.',
)
parser.add_argument("--target", help="Build apps for given target.")
parser.add_argument('--target', help='Build apps for given target.')
parser.add_argument(
"--format",
default="json",
choices=["json"],
help="Format to write the list of builds as",
'--format',
default='json',
choices=['json'],
help='Format to write the list of builds as',
)
parser.add_argument(
"--exclude",
action="append",
help="Ignore specified directory (if --recursive is given). Can be used multiple times.",
'--exclude',
action='append',
help='Ignore specified directory (if --recursive is given). Can be used multiple times.',
)
parser.add_argument(
"-o",
"--output",
type=argparse.FileType("w"),
help="Output the list of builds to the specified file",
'-o',
'--output',
type=argparse.FileType('w'),
help='Output the list of builds to the specified file',
)
parser.add_argument(
"--app-list",
'--app-list',
default=None,
help="Scan tests results. Restrict the build/artifacts preservation behavior to apps need to be built. "
"If the file does not exist, will build all apps and upload all artifacts."
help='Scan tests results. Restrict the build/artifacts preservation behavior to apps need to be built. '
'If the file does not exist, will build all apps and upload all artifacts.'
)
parser.add_argument(
"-p", "--paths",
nargs="+",
help="One or more app paths."
'-p', '--paths',
nargs='+',
help='One or more app paths.'
)
args = parser.parse_args()
setup_logging(args)
@@ -265,19 +255,19 @@ def main():
raise ValueError('Conflict settings. "recursive", "build_system", "target", "exclude", "paths" should not '
'be specified with "app_list"')
if not os.path.exists(args.app_list):
raise OSError("File not found {}".format(args.app_list))
raise OSError('File not found {}'.format(args.app_list))
else:
# If the build target is not set explicitly, get it from the environment or use the default one (esp32)
if not args.target:
env_target = os.environ.get("IDF_TARGET")
env_target = os.environ.get('IDF_TARGET')
if env_target:
logging.info("--target argument not set, using IDF_TARGET={} from the environment".format(env_target))
logging.info('--target argument not set, using IDF_TARGET={} from the environment'.format(env_target))
args.target = env_target
else:
logging.info("--target argument not set, using IDF_TARGET={} as the default".format(DEFAULT_TARGET))
logging.info('--target argument not set, using IDF_TARGET={} as the default'.format(DEFAULT_TARGET))
args.target = DEFAULT_TARGET
if not args.build_system:
logging.info("--build-system argument not set, using {} as the default".format(BUILD_SYSTEM_CMAKE))
logging.info('--build-system argument not set, using {} as the default'.format(BUILD_SYSTEM_CMAKE))
args.build_system = BUILD_SYSTEM_CMAKE
required_args = [args.build_system, args.target, args.paths]
if not all(required_args):
@@ -293,38 +283,38 @@ def main():
build_system_class = BUILD_SYSTEMS[args.build_system]
for path in args.paths:
app_dirs += find_apps(build_system_class, path, args.recursive, args.exclude or [], args.target)
apps = [{"app_dir": app_dir, "build": True, "preserve": True} for app_dir in app_dirs]
apps = [{'app_dir': app_dir, 'build': True, 'preserve': True} for app_dir in app_dirs]
if not apps:
logging.warning("No apps found")
logging.warning('No apps found')
SystemExit(0)
logging.info("Found {} apps".format(len(apps)))
apps.sort(key=lambda x: x["app_dir"])
logging.info('Found {} apps'.format(len(apps)))
apps.sort(key=lambda x: x['app_dir'])
# Find compatible configurations of each app, collect them as BuildItems
build_items = [] # type: typing.List[BuildItem]
config_rules = config_rules_from_str(args.config or [])
for app in apps:
build_items += find_builds_for_app(
app["app_dir"],
app['app_dir'],
args.work_dir,
args.build_dir,
args.build_log,
args.target or app["target"],
args.build_system or app["build_system"],
args.target or app['target'],
args.build_system or app['build_system'],
config_rules,
app["preserve"],
app['preserve'],
)
logging.info("Found {} builds".format(len(build_items)))
logging.info('Found {} builds'.format(len(build_items)))
# Write out the BuildItems. Only JSON supported now (will add YAML later).
if args.format != "json":
if args.format != 'json':
raise NotImplementedError()
out = args.output or sys.stdout
out.writelines([item.to_json() + "\n" for item in build_items])
out.writelines([item.to_json() + '\n' for item in build_items])
if __name__ == "__main__":
if __name__ == '__main__':
main()

39
tools/find_build_apps/__init__.py
View File

@@ -1,14 +1,7 @@
from .common import (
BuildItem,
BuildSystem,
BuildError,
ConfigRule,
config_rules_from_str,
setup_logging,
DEFAULT_TARGET,
)
from .cmake import CMakeBuildSystem, BUILD_SYSTEM_CMAKE
from .make import MakeBuildSystem, BUILD_SYSTEM_MAKE
from .cmake import BUILD_SYSTEM_CMAKE, CMakeBuildSystem
from .common import (DEFAULT_TARGET, BuildError, BuildItem, BuildSystem, ConfigRule, config_rules_from_str,
setup_logging)
from .make import BUILD_SYSTEM_MAKE, MakeBuildSystem
BUILD_SYSTEMS = {
BUILD_SYSTEM_MAKE: MakeBuildSystem,
@@ -16,16 +9,16 @@ BUILD_SYSTEMS = {
}
__all__ = [
"BuildItem",
"BuildSystem",
"BuildError",
"ConfigRule",
"config_rules_from_str",
"setup_logging",
"DEFAULT_TARGET",
"CMakeBuildSystem",
"BUILD_SYSTEM_CMAKE",
"MakeBuildSystem",
"BUILD_SYSTEM_MAKE",
"BUILD_SYSTEMS",
'BuildItem',
'BuildSystem',
'BuildError',
'ConfigRule',
'config_rules_from_str',
'setup_logging',
'DEFAULT_TARGET',
'CMakeBuildSystem',
'BUILD_SYSTEM_CMAKE',
'MakeBuildSystem',
'BUILD_SYSTEM_MAKE',
'BUILD_SYSTEMS',
]

44
tools/find_build_apps/cmake.py
View File

@@ -4,14 +4,14 @@ import shutil
import subprocess
import sys
from .common import BuildSystem, BuildItem, BuildError
from .common import BuildError, BuildItem, BuildSystem
BUILD_SYSTEM_CMAKE = "cmake"
IDF_PY = os.path.join(os.environ["IDF_PATH"], "tools", "idf.py")
BUILD_SYSTEM_CMAKE = 'cmake'
IDF_PY = os.path.join(os.environ['IDF_PATH'], 'tools', 'idf.py')
# While ESP-IDF component CMakeLists files can be identified by the presence of 'idf_component_register' string,
# there is no equivalent for the project CMakeLists files. This seems to be the best option...
CMAKE_PROJECT_LINE = r"include($ENV{IDF_PATH}/tools/cmake/project.cmake)"
CMAKE_PROJECT_LINE = r'include($ENV{IDF_PATH}/tools/cmake/project.cmake)'
class CMakeBuildSystem(BuildSystem):
@@ -24,23 +24,23 @@ class CMakeBuildSystem(BuildSystem):
args = [
sys.executable,
IDF_PY,
"-B",
'-B',
build_path,
"-C",
'-C',
work_path,
"-DIDF_TARGET=" + build_item.target,
'-DIDF_TARGET=' + build_item.target,
]
if extra_cmakecache_items:
for key, val in extra_cmakecache_items.items():
args.append("-D{}={}".format(key, val))
if "TEST_EXCLUDE_COMPONENTS" in extra_cmakecache_items \
and "TEST_COMPONENTS" not in extra_cmakecache_items:
args.append("-DTESTS_ALL=1")
args.append('-D{}={}'.format(key, val))
if 'TEST_EXCLUDE_COMPONENTS' in extra_cmakecache_items \
and 'TEST_COMPONENTS' not in extra_cmakecache_items:
args.append('-DTESTS_ALL=1')
if build_item.verbose:
args.append("-v")
args.append("build")
cmdline = format(" ".join(args))
logging.info("Running {}".format(cmdline))
args.append('-v')
args.append('build')
cmdline = format(' '.join(args))
logging.info('Running {}'.format(cmdline))
if build_item.dry_run:
return
@@ -49,20 +49,20 @@ class CMakeBuildSystem(BuildSystem):
build_stdout = sys.stdout
build_stderr = sys.stderr
if build_item.build_log_path:
logging.info("Writing build log to {}".format(build_item.build_log_path))
log_file = open(build_item.build_log_path, "w")
logging.info('Writing build log to {}'.format(build_item.build_log_path))
log_file = open(build_item.build_log_path, 'w')
build_stdout = log_file
build_stderr = log_file
try:
subprocess.check_call(args, stdout=build_stdout, stderr=build_stderr)
except subprocess.CalledProcessError as e:
raise BuildError("Build failed with exit code {}".format(e.returncode))
raise BuildError('Build failed with exit code {}'.format(e.returncode))
else:
# Also save the sdkconfig file in the build directory
shutil.copyfile(
os.path.join(work_path, "sdkconfig"),
os.path.join(build_path, "sdkconfig"),
os.path.join(work_path, 'sdkconfig'),
os.path.join(build_path, 'sdkconfig'),
)
build_item.size_json_fp = build_item.get_size_json_fp()
finally:
@@ -71,10 +71,10 @@ class CMakeBuildSystem(BuildSystem):
@staticmethod
def _read_cmakelists(app_path):
cmakelists_path = os.path.join(app_path, "CMakeLists.txt")
cmakelists_path = os.path.join(app_path, 'CMakeLists.txt')
if not os.path.exists(cmakelists_path):
return None
with open(cmakelists_path, "r") as cmakelists_file:
with open(cmakelists_path, 'r') as cmakelists_file:
return cmakelists_file.read()
@staticmethod

111
tools/find_build_apps/common.py
View File

@@ -7,35 +7,34 @@ import re
import shutil
import subprocess
import sys
import typing
from abc import abstractmethod
from collections import namedtuple
from io import open
import typing
DEFAULT_TARGET = 'esp32'
DEFAULT_TARGET = "esp32"
TARGET_PLACEHOLDER = '@t'
WILDCARD_PLACEHOLDER = '@w'
NAME_PLACEHOLDER = '@n'
FULL_NAME_PLACEHOLDER = '@f'
INDEX_PLACEHOLDER = '@i'
TARGET_PLACEHOLDER = "@t"
WILDCARD_PLACEHOLDER = "@w"
NAME_PLACEHOLDER = "@n"
FULL_NAME_PLACEHOLDER = "@f"
INDEX_PLACEHOLDER = "@i"
IDF_SIZE_PY = os.path.join(os.environ["IDF_PATH"], "tools", "idf_size.py")
IDF_SIZE_PY = os.path.join(os.environ['IDF_PATH'], 'tools', 'idf_size.py')
SIZE_JSON_FN = 'size.json'
SDKCONFIG_LINE_REGEX = re.compile(r"^([^=]+)=\"?([^\"\n]*)\"?\n*$")
# If these keys are present in sdkconfig.defaults, they will be extracted and passed to CMake
SDKCONFIG_TEST_OPTS = [
"EXCLUDE_COMPONENTS",
"TEST_EXCLUDE_COMPONENTS",
"TEST_COMPONENTS",
'EXCLUDE_COMPONENTS',
'TEST_EXCLUDE_COMPONENTS',
'TEST_COMPONENTS',
]
# These keys in sdkconfig.defaults are not propagated to the final sdkconfig file:
SDKCONFIG_IGNORE_OPTS = [
"TEST_GROUPS"
'TEST_GROUPS'
]
# ConfigRule represents one --config argument of find_apps.py.
@@ -45,7 +44,7 @@ SDKCONFIG_IGNORE_OPTS = [
# For example:
# filename='', config_name='default' — represents the default app configuration, and gives it a name 'default'
# filename='sdkconfig.*', config_name=None - represents the set of configurations, names match the wildcard value
ConfigRule = namedtuple("ConfigRule", ["file_name", "config_name"])
ConfigRule = namedtuple('ConfigRule', ['file_name', 'config_name'])
def config_rules_from_str(rule_strings): # type: (typing.List[str]) -> typing.List[ConfigRule]
@@ -56,7 +55,7 @@ def config_rules_from_str(rule_strings): # type: (typing.List[str]) -> typing.L
"""
rules = [] # type: typing.List[ConfigRule]
for rule_str in rule_strings:
items = rule_str.split("=", 2)
items = rule_str.split('=', 2)
rules.append(ConfigRule(items[0], items[1] if len(items) == 2 else None))
return rules
@@ -128,7 +127,7 @@ class BuildItem(object):
self.work_path = self.work_dir or self.app_dir
if not self.build_dir:
self.build_path = os.path.join(self.work_path, "build")
self.build_path = os.path.join(self.work_path, 'build')
elif os.path.isabs(self.build_dir):
self.build_path = self.build_dir
else:
@@ -164,11 +163,11 @@ class BuildItem(object):
return self._expand(self._build_log_path)
def __repr__(self):
return "({}) Build app {} for target {}, sdkconfig {} in {}".format(
return '({}) Build app {} for target {}, sdkconfig {} in {}'.format(
self.build_system,
self.app_dir,
self.target,
self.sdkconfig_path or "(default)",
self.sdkconfig_path or '(default)',
self.build_dir,
)
@@ -189,16 +188,16 @@ class BuildItem(object):
Internal function, called by to_json and to_json_expanded
"""
return json.dumps({
"build_system": self.build_system,
"app_dir": app_dir,
"work_dir": work_dir,
"build_dir": build_dir,
"build_log_path": build_log_path,
"sdkconfig": self.sdkconfig_path,
"config": self.config_name,
"target": self.target,
"verbose": self.verbose,
"preserve": self.preserve,
'build_system': self.build_system,
'app_dir': app_dir,
'work_dir': work_dir,
'build_dir': build_dir,
'build_log_path': build_log_path,
'sdkconfig': self.sdkconfig_path,
'config': self.config_name,
'target': self.target,
'verbose': self.verbose,
'preserve': self.preserve,
})
@staticmethod
@@ -208,17 +207,17 @@ class BuildItem(object):
"""
d = json.loads(str(json_str))
result = BuildItem(
app_path=d["app_dir"],
work_dir=d["work_dir"],
build_path=d["build_dir"],
build_log_path=d["build_log_path"],
sdkconfig_path=d["sdkconfig"],
config_name=d["config"],
target=d["target"],
build_system=d["build_system"],
preserve_artifacts=d["preserve"]
app_path=d['app_dir'],
work_dir=d['work_dir'],
build_path=d['build_dir'],
build_log_path=d['build_log_path'],
sdkconfig_path=d['sdkconfig'],
config_name=d['config'],
target=d['target'],
build_system=d['build_system'],
preserve_artifacts=d['preserve']
)
result.verbose = d["verbose"]
result.verbose = d['verbose']
return result
def _expand(self, path): # type: (str) -> str
@@ -233,7 +232,7 @@ class BuildItem(object):
path = path.replace(TARGET_PLACEHOLDER, self.target)
path = path.replace(NAME_PLACEHOLDER, self._app_name)
if (FULL_NAME_PLACEHOLDER in path): # to avoid recursion to the call to app_dir in the next line:
path = path.replace(FULL_NAME_PLACEHOLDER, self.app_dir.replace(os.path.sep, "_"))
path = path.replace(FULL_NAME_PLACEHOLDER, self.app_dir.replace(os.path.sep, '_'))
wildcard_pos = path.find(WILDCARD_PLACEHOLDER)
if wildcard_pos != -1:
if self.config_name:
@@ -288,7 +287,7 @@ class BuildSystem:
Derived classes implement the methods below.
Objects of these classes aren't instantiated, instead the class (type object) is used.
"""
NAME = "undefined"
NAME = 'undefined'
SUPPORTED_TARGETS_REGEX = re.compile(r'Supported [Tt]argets((?:[ |]+(?:[0-9a-zA-Z\-]+))+)')
FORMAL_TO_USUAL = {
@@ -307,15 +306,15 @@ class BuildSystem:
if work_path != app_path:
if os.path.exists(work_path):
logging.debug("Work directory {} exists, removing".format(work_path))
logging.debug('Work directory {} exists, removing'.format(work_path))
if not build_item.dry_run:
shutil.rmtree(work_path)
logging.debug("Copying app from {} to {}".format(app_path, work_path))
logging.debug('Copying app from {} to {}'.format(app_path, work_path))
if not build_item.dry_run:
shutil.copytree(app_path, work_path)
if os.path.exists(build_path):
logging.debug("Build directory {} exists, removing".format(build_path))
logging.debug('Build directory {} exists, removing'.format(build_path))
if not build_item.dry_run:
shutil.rmtree(build_path)
@@ -328,29 +327,29 @@ class BuildSystem:
# Note: the build system supports taking multiple sdkconfig.defaults files via SDKCONFIG_DEFAULTS
# CMake variable. However here we do this manually to perform environment variable expansion in the
# sdkconfig files.
sdkconfig_defaults_list = ["sdkconfig.defaults", "sdkconfig.defaults." + build_item.target]
sdkconfig_defaults_list = ['sdkconfig.defaults', 'sdkconfig.defaults.' + build_item.target]
if build_item.sdkconfig_path:
sdkconfig_defaults_list.append(build_item.sdkconfig_path)
sdkconfig_file = os.path.join(work_path, "sdkconfig")
sdkconfig_file = os.path.join(work_path, 'sdkconfig')
if os.path.exists(sdkconfig_file):
logging.debug("Removing sdkconfig file: {}".format(sdkconfig_file))
logging.debug('Removing sdkconfig file: {}'.format(sdkconfig_file))
if not build_item.dry_run:
os.unlink(sdkconfig_file)
logging.debug("Creating sdkconfig file: {}".format(sdkconfig_file))
logging.debug('Creating sdkconfig file: {}'.format(sdkconfig_file))
extra_cmakecache_items = {}
if not build_item.dry_run:
with open(sdkconfig_file, "w") as f_out:
with open(sdkconfig_file, 'w') as f_out:
for sdkconfig_name in sdkconfig_defaults_list:
sdkconfig_path = os.path.join(work_path, sdkconfig_name)
if not sdkconfig_path or not os.path.exists(sdkconfig_path):
continue
logging.debug("Appending {} to sdkconfig".format(sdkconfig_name))
with open(sdkconfig_path, "r") as f_in:
logging.debug('Appending {} to sdkconfig'.format(sdkconfig_name))
with open(sdkconfig_path, 'r') as f_in:
for line in f_in:
if not line.endswith("\n"):
line += "\n"
if not line.endswith('\n'):
line += '\n'
if cls.NAME == 'cmake':
m = SDKCONFIG_LINE_REGEX.match(line)
key = m.group(1) if m else None
@@ -365,10 +364,10 @@ class BuildSystem:
sdkconfig_path = os.path.join(app_path, sdkconfig_name)
if not sdkconfig_path:
continue
logging.debug("Considering sdkconfig {}".format(sdkconfig_path))
logging.debug('Considering sdkconfig {}'.format(sdkconfig_path))
if not os.path.exists(sdkconfig_path):
continue
logging.debug("Appending {} to sdkconfig".format(sdkconfig_name))
logging.debug('Appending {} to sdkconfig'.format(sdkconfig_name))
# The preparation of build is finished. Implement the build part in sub classes.
if cls.NAME == 'cmake':
@@ -409,7 +408,7 @@ class BuildSystem:
readme_path = get_md_or_rst(os.path.dirname(app_path))
if not readme_path:
return None
with open(readme_path, "r", encoding='utf8') as readme_file:
with open(readme_path, 'r', encoding='utf8') as readme_file:
return readme_file.read()
@classmethod
@@ -460,7 +459,7 @@ def setup_logging(args):
log_level = logging.DEBUG
logging.basicConfig(
format="%(levelname)s: %(message)s",
format='%(levelname)s: %(message)s',
stream=args.log_file or sys.stderr,
level=log_level,
)

16
tools/find_build_apps/make.py
View File

@@ -4,12 +4,12 @@ import shlex
import subprocess
import sys
from .common import BuildSystem, BuildError
from .common import BuildError, BuildSystem
# Same for the Makefile projects:
MAKE_PROJECT_LINE = r"include $(IDF_PATH)/make/project.mk"
MAKE_PROJECT_LINE = r'include $(IDF_PATH)/make/project.mk'
BUILD_SYSTEM_MAKE = "make"
BUILD_SYSTEM_MAKE = 'make'
try:
string_type = basestring
@@ -34,8 +34,8 @@ class MakeBuildSystem(BuildSystem):
build_stdout = sys.stdout
build_stderr = sys.stderr
if build_item.build_log_path:
logging.info("Writing build log to {}".format(build_item.build_log_path))
log_file = open(build_item.build_log_path, "w")
logging.info('Writing build log to {}'.format(build_item.build_log_path))
log_file = open(build_item.build_log_path, 'w')
build_stdout = log_file
build_stderr = log_file
@@ -46,16 +46,16 @@ class MakeBuildSystem(BuildSystem):
except subprocess.CalledProcessError as e:
if log_file:
log_file.close()
raise BuildError("Build failed with exit code {}".format(e.returncode))
raise BuildError('Build failed with exit code {}'.format(e.returncode))
build_item.size_json_fp = build_item.get_size_json_fp()
@staticmethod
def is_app(path):
makefile_path = os.path.join(path, "Makefile")
makefile_path = os.path.join(path, 'Makefile')
if not os.path.exists(makefile_path):
return False
with open(makefile_path, "r") as makefile:
with open(makefile_path, 'r') as makefile:
makefile_content = makefile.read()
if MAKE_PROJECT_LINE not in makefile_content:
return False

145
tools/gdb_panic_server.py
View File

@@ -31,18 +31,17 @@
#
from builtins import bytes
import argparse
import binascii
import logging
import struct
import sys
import logging
import binascii
from builtins import bytes
from collections import namedtuple
from pyparsing import Literal, Word, nums, OneOrMore, srange, Group, Combine
# Used for type annotations only. Silence linter warnings.
from pyparsing import ParseResults, ParserElement # noqa: F401 # pylint: disable=unused-import
from pyparsing import (Combine, Group, Literal, OneOrMore, ParserElement, # noqa: F401 # pylint: disable=unused-import
ParseResults, Word, nums, srange)
try:
import typing # noqa: F401 # pylint: disable=unused-import
@@ -50,30 +49,30 @@ except ImportError:
pass
# pyparsing helper
hexnumber = srange("[0-9a-f]")
hexnumber = srange('[0-9a-f]')
# List of registers to be passed to GDB, in the order GDB expects.
# The names should match those used in IDF panic handler.
# Registers not present in IDF panic handler output (like X0) will be assumed to be 0.
GDB_REGS_INFO_RISCV_ILP32 = [
"X0", "RA", "SP", "GP",
"TP", "T0", "T1", "T2",
"S0/FP", "S1", "A0", "A1",
"A2", "A3", "A4", "A5",
"A6", "A7", "S2", "S3",
"S4", "S5", "S6", "S7",
"S8", "S9", "S10", "S11",
"T3", "T4", "T5", "T6",
"MEPC"
'X0', 'RA', 'SP', 'GP',
'TP', 'T0', 'T1', 'T2',
'S0/FP', 'S1', 'A0', 'A1',
'A2', 'A3', 'A4', 'A5',
'A6', 'A7', 'S2', 'S3',
'S4', 'S5', 'S6', 'S7',
'S8', 'S9', 'S10', 'S11',
'T3', 'T4', 'T5', 'T6',
'MEPC'
]
GDB_REGS_INFO = {
"esp32c3": GDB_REGS_INFO_RISCV_ILP32
'esp32c3': GDB_REGS_INFO_RISCV_ILP32
}
PanicInfo = namedtuple("PanicInfo", "core_id regs stack_base_addr stack_data")
PanicInfo = namedtuple('PanicInfo', 'core_id regs stack_base_addr stack_data')
def build_riscv_panic_output_parser(): # type: () -> typing.Type[ParserElement]
@@ -83,25 +82,25 @@ def build_riscv_panic_output_parser(): # type: () -> typing.Type[ParserElement]
# Guru Meditation Error: Core 0 panic'ed (Store access fault). Exception was unhandled.
# Core 0 register dump:
reg_dump_header = Group(Literal("Core") +
Word(nums)("core_id") +
Literal("register dump:"))("reg_dump_header")
reg_dump_header = Group(Literal('Core') +
Word(nums)('core_id') +
Literal('register dump:'))('reg_dump_header')
# MEPC : 0x4200232c RA : 0x42009694 SP : 0x3fc93a80 GP : 0x3fc8b320
reg_name = Word(srange("[A-Z_0-9/-]"))("name")
hexnumber_with_0x = Combine(Literal("0x") + Word(hexnumber))
reg_value = hexnumber_with_0x("value")
reg_dump_one_reg = Group(reg_name + Literal(":") + reg_value) # not named because there will be OneOrMore
reg_dump_all_regs = Group(OneOrMore(reg_dump_one_reg))("regs")
reg_name = Word(srange('[A-Z_0-9/-]'))('name')
hexnumber_with_0x = Combine(Literal('0x') + Word(hexnumber))
reg_value = hexnumber_with_0x('value')
reg_dump_one_reg = Group(reg_name + Literal(':') + reg_value) # not named because there will be OneOrMore
reg_dump_all_regs = Group(OneOrMore(reg_dump_one_reg))('regs')
reg_dump = Group(reg_dump_header + reg_dump_all_regs) # not named because there will be OneOrMore
reg_dumps = Group(OneOrMore(reg_dump))("reg_dumps")
reg_dumps = Group(OneOrMore(reg_dump))('reg_dumps')
# Stack memory:
# 3fc93a80: 0x00000030 0x00000021 0x3fc8aedc 0x4200232a 0xa5a5a5a5 0xa5a5a5a5 0x3fc8aedc 0x420099b0
stack_line = Group(Word(hexnumber)("base") + Literal(":") +
Group(OneOrMore(hexnumber_with_0x))("data"))
stack_dump = Group(Literal("Stack memory:") +
Group(OneOrMore(stack_line))("lines"))("stack_dump")
stack_line = Group(Word(hexnumber)('base') + Literal(':') +
Group(OneOrMore(hexnumber_with_0x))('data'))
stack_dump = Group(Literal('Stack memory:') +
Group(OneOrMore(stack_line))('lines'))('stack_dump')
# Parser for the complete panic output:
panic_output = reg_dumps + stack_dump
@@ -113,7 +112,7 @@ def get_stack_addr_and_data(res): # type: (ParseResults) -> typing.Tuple[int, b
stack_base_addr = 0 # First reported address in the dump
base_addr = 0 # keeps track of the address for the given line of the dump
bytes_in_line = 0 # bytes of stack parsed on the previous line; used to validate the next base address
stack_data = b"" # accumulates all the dumped stack data
stack_data = b'' # accumulates all the dumped stack data
for line in res.stack_dump.lines:
# update and validate the base address
prev_base_addr = base_addr
@@ -125,7 +124,7 @@ def get_stack_addr_and_data(res): # type: (ParseResults) -> typing.Tuple[int, b
# convert little-endian hex words to byte representation
words = [int(w, 16) for w in line.data]
line_data = b"".join([struct.pack("<I", w) for w in words])
line_data = b''.join([struct.pack('<I', w) for w in words])
bytes_in_line = len(line_data)
# accumulate in the whole stack data
@@ -143,7 +142,7 @@ def parse_idf_riscv_panic_output(panic_text): # type: (str) -> PanicInfo
res = results[0]
if len(res.reg_dumps) > 1:
raise NotImplementedError("Handling of multi-core register dumps not implemented")
raise NotImplementedError('Handling of multi-core register dumps not implemented')
# Build a dict of register names/values
rd = res.reg_dumps[0]
@@ -162,7 +161,7 @@ def parse_idf_riscv_panic_output(panic_text): # type: (str) -> PanicInfo
PANIC_OUTPUT_PARSERS = {
"esp32c3": parse_idf_riscv_panic_output
'esp32c3': parse_idf_riscv_panic_output
}
@@ -173,82 +172,82 @@ class GdbServer(object):
self.out_stream = sys.stdout
self.reg_list = GDB_REGS_INFO[target]
self.logger = logging.getLogger("GdbServer")
self.logger = logging.getLogger('GdbServer')
if log_file:
handler = logging.FileHandler(log_file, "w+")
handler = logging.FileHandler(log_file, 'w+')
self.logger.setLevel(logging.DEBUG)
formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
handler.setFormatter(formatter)
self.logger.addHandler(handler)
def run(self): # type: () -> None
""" Process GDB commands from stdin until GDB tells us to quit """
buffer = ""
buffer = ''
while True:
buffer += self.in_stream.read(1)
if len(buffer) > 3 and buffer[-3] == '#':
self._handle_command(buffer)
buffer = ""
buffer = ''
def _handle_command(self, buffer): # type: (str) -> None
command = buffer[1:-3] # ignore checksums
# Acknowledge the command
self.out_stream.write("+")
self.out_stream.write('+')
self.out_stream.flush()
self.logger.debug("Got command: %s", command)
if command == "?":
self.logger.debug('Got command: %s', command)
if command == '?':
# report sigtrap as the stop reason; the exact reason doesn't matter for backtracing
self._respond("T05")
elif command.startswith("Hg") or command.startswith("Hc"):
self._respond('T05')
elif command.startswith('Hg') or command.startswith('Hc'):
# Select thread command
self._respond("OK")
elif command == "qfThreadInfo":
self._respond('OK')
elif command == 'qfThreadInfo':
# Get list of threads.
# Only one thread for now, can be extended to show one thread for each core,
# if we dump both cores (e.g. on an interrupt watchdog)
self._respond("m1")
elif command == "qC":
self._respond('m1')
elif command == 'qC':
# That single thread is selected.
self._respond("QC1")
elif command == "g":
self._respond('QC1')
elif command == 'g':
# Registers read
self._respond_regs()
elif command.startswith("m"):
elif command.startswith('m'):
# Memory read
addr, size = [int(v, 16) for v in command[1:].split(",")]
addr, size = [int(v, 16) for v in command[1:].split(',')]
self._respond_mem(addr, size)
elif command.startswith("vKill") or command == "k":
elif command.startswith('vKill') or command == 'k':
# Quit
self._respond("OK")
self._respond('OK')
raise SystemExit(0)
else:
# Empty response required for any unknown command
self._respond("")
self._respond('')
def _respond(self, data): # type: (str) -> None
# calculate checksum
data_bytes = bytes(data.encode("ascii")) # bytes() for Py2 compatibility
data_bytes = bytes(data.encode('ascii')) # bytes() for Py2 compatibility
checksum = sum(data_bytes) & 0xff
# format and write the response
res = "${}#{:02x}".format(data, checksum)
self.logger.debug("Wrote: %s", res)
res = '${}#{:02x}'.format(data, checksum)
self.logger.debug('Wrote: %s', res)
self.out_stream.write(res)
self.out_stream.flush()
# get the result ('+' or '-')
ret = self.in_stream.read(1)
self.logger.debug("Response: %s", ret)
self.logger.debug('Response: %s', ret)
if ret != '+':
sys.stderr.write("GDB responded with '-' to {}".format(res))
raise SystemExit(1)
def _respond_regs(self): # type: () -> None
response = ""
response = ''
for reg_name in self.reg_list:
# register values are reported as hexadecimal strings
# in target byte order (i.e. LSB first for RISC-V)
reg_val = self.panic_info.regs.get(reg_name, 0)
reg_bytes = struct.pack("<L", reg_val)
response += binascii.hexlify(reg_bytes).decode("ascii")
reg_bytes = struct.pack('<L', reg_val)
response += binascii.hexlify(reg_bytes).decode('ascii')
self._respond(response)
def _respond_mem(self, start_addr, size): # type: (int, int) -> None
@@ -262,24 +261,24 @@ class GdbServer(object):
def in_stack(addr):
return stack_addr_min <= addr < stack_addr_max
result = ""
result = ''
for addr in range(start_addr, start_addr + size):
if not in_stack(addr):
result += "00"
result += '00'
else:
result += "{:02x}".format(stack_data[addr - stack_addr_min])
result += '{:02x}'.format(stack_data[addr - stack_addr_min])
self._respond(result)
def main():
parser = argparse.ArgumentParser()
parser.add_argument("input_file", type=argparse.FileType("r"),
help="File containing the panic handler output")
parser.add_argument("--target", choices=GDB_REGS_INFO.keys(),
help="Chip to use (determines the architecture)")
parser.add_argument("--gdb-log", default=None,
help="If specified, the file for logging GDB server debug information")
parser.add_argument('input_file', type=argparse.FileType('r'),
help='File containing the panic handler output')
parser.add_argument('--target', choices=GDB_REGS_INFO.keys(),
help='Chip to use (determines the architecture)')
parser.add_argument('--gdb-log', default=None,
help='If specified, the file for logging GDB server debug information')
args = parser.parse_args()
panic_info = PANIC_OUTPUT_PARSERS[args.target](args.input_file.read())
@@ -291,5 +290,5 @@ def main():
sys.exit(0)
if __name__ == "__main__":
if __name__ == '__main__':
main()

81
tools/gen_esp_err_to_name.py
View File

@@ -14,13 +14,12 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import print_function
from __future__ import unicode_literals
from __future__ import print_function, unicode_literals
import sys
try:
from builtins import str
from builtins import range
from builtins import object
from builtins import object, range, str
except ImportError:
# This should not happen because the Python packages are checked before invoking this script. However, here is
# some output which should help if we missed something.
@@ -30,14 +29,14 @@ except ImportError:
# requirements.txt from the IDF_PATH should be used) or from the documentation project (then the requirements.txt
# for the documentation directory should be used).
sys.exit(1)
from io import open
import os
import argparse
import re
import fnmatch
import collections
import textwrap
import fnmatch
import functools
import os
import re
import textwrap
from io import open
# list files here which should not be parsed
ignore_files = [os.path.join('components', 'mdns', 'test_afl_fuzz_host', 'esp32_compat.h'),
@@ -72,7 +71,7 @@ class ErrItem(object):
- rel_str - (optional) error string which is a base for the error
- rel_off - (optional) offset in relation to the base error
"""
def __init__(self, name, file, include_as=None, comment="", rel_str="", rel_off=0):
def __init__(self, name, file, include_as=None, comment='', rel_str='', rel_off=0):
self.name = name
self.file = file
self.include_as = include_as
@@ -81,11 +80,11 @@ class ErrItem(object):
self.rel_off = rel_off
def __str__(self):
ret = self.name + " from " + self.file
if (self.rel_str != ""):
ret += " is (" + self.rel_str + " + " + str(self.rel_off) + ")"
if self.comment != "":
ret += " // " + self.comment
ret = self.name + ' from ' + self.file
if (self.rel_str != ''):
ret += ' is (' + self.rel_str + ' + ' + str(self.rel_off) + ')'
if self.comment != '':
ret += ' // ' + self.comment
return ret
def __cmp__(self, other):
@@ -94,7 +93,7 @@ class ErrItem(object):
elif self.file not in priority_headers and other.file in priority_headers:
return 1
base = "_BASE"
base = '_BASE'
if self.file == other.file:
if self.name.endswith(base) and not(other.name.endswith(base)):
@@ -117,7 +116,7 @@ class InputError(RuntimeError):
Represents and error on the input
"""
def __init__(self, p, e):
super(InputError, self).__init__(p + ": " + e)
super(InputError, self).__init__(p + ': ' + e)
def process(line, idf_path, include_as):
@@ -125,20 +124,20 @@ def process(line, idf_path, include_as):
Process a line of text from file idf_path (relative to IDF project).
Fills the global list unproc_list and dictionaries err_dict, rev_err_dict
"""
if idf_path.endswith(".c"):
if idf_path.endswith('.c'):
# We would not try to include a C file
raise InputError(idf_path, "This line should be in a header file: %s" % line)
raise InputError(idf_path, 'This line should be in a header file: %s' % line)
words = re.split(r' +', line, 2)
# words[1] is the error name
# words[2] is the rest of the line (value, base + value, comment)
if len(words) < 3:
raise InputError(idf_path, "Error at line %s" % line)
raise InputError(idf_path, 'Error at line %s' % line)
line = ""
line = ''
todo_str = words[2]
comment = ""
comment = ''
# identify possible comment
m = re.search(r'/\*!<(.+?(?=\*/))', todo_str)
if m:
@@ -170,7 +169,7 @@ def process(line, idf_path, include_as):
related = todo_str # BASE error
num = 0 # (BASE + 0)
else:
raise InputError(idf_path, "Cannot parse line %s" % line)
raise InputError(idf_path, 'Cannot parse line %s' % line)
try:
related
@@ -199,7 +198,7 @@ def process_remaining_errors():
err_dict[num].append(ErrItem(item.name, item.file, item.include_as, item.comment))
rev_err_dict[item.name] = num
else:
print(item.rel_str + " referenced by " + item.name + " in " + item.file + " is unknown")
print(item.rel_str + ' referenced by ' + item.name + ' in ' + item.file + ' is unknown')
del unproc_list[:]
@@ -229,9 +228,9 @@ def print_warning(error_list, error_code):
"""
Print warning about errors with the same error code
"""
print("[WARNING] The following errors have the same code (%d):" % error_code)
print('[WARNING] The following errors have the same code (%d):' % error_code)
for e in error_list:
print(" " + str(e))
print(' ' + str(e))
def max_string_width():
@@ -270,14 +269,14 @@ def generate_c_output(fin, fout):
for line in fin:
if re.match(r'@COMMENT@', line):
fout.write("//Do not edit this file because it is autogenerated by " + os.path.basename(__file__) + "\n")
fout.write('//Do not edit this file because it is autogenerated by ' + os.path.basename(__file__) + '\n')
elif re.match(r'@HEADERS@', line):
for i in include_list:
if i not in dont_include:
fout.write("#if __has_include(\"" + i + "\")\n#include \"" + i + "\"\n#endif\n")
elif re.match(r'@ERROR_ITEMS@', line):
last_file = ""
last_file = ''
for k in sorted(err_dict.keys()):
if len(err_dict[k]) > 1:
err_dict[k].sort(key=functools.cmp_to_key(ErrItem.__cmp__))
@@ -285,26 +284,26 @@ def generate_c_output(fin, fout):
for e in err_dict[k]:
if e.file != last_file:
last_file = e.file
fout.write(" // %s\n" % last_file)
table_line = (" ERR_TBL_IT(" + e.name + "), ").ljust(max_width) + "/* " + str(k).rjust(max_decdig)
fout.write("# ifdef %s\n" % e.name)
fout.write(' // %s\n' % last_file)
table_line = (' ERR_TBL_IT(' + e.name + '), ').ljust(max_width) + '/* ' + str(k).rjust(max_decdig)
fout.write('# ifdef %s\n' % e.name)
fout.write(table_line)
hexnum_length = 0
if k > 0: # negative number and zero should be only ESP_FAIL and ESP_OK
hexnum = " 0x%x" % k
hexnum = ' 0x%x' % k
hexnum_length = len(hexnum)
fout.write(hexnum)
if e.comment != "":
if e.comment != '':
if len(e.comment) < 50:
fout.write(" %s" % e.comment)
fout.write(' %s' % e.comment)
else:
indent = " " * (len(table_line) + hexnum_length + 1)
indent = ' ' * (len(table_line) + hexnum_length + 1)
w = textwrap.wrap(e.comment, width=120, initial_indent=indent, subsequent_indent=indent)
# this couldn't be done with initial_indent because there is no initial_width option
fout.write(" %s" % w[0].strip())
fout.write(' %s' % w[0].strip())
for i in range(1, len(w)):
fout.write("\n%s" % w[i])
fout.write(" */\n# endif\n")
fout.write('\n%s' % w[i])
fout.write(' */\n# endif\n')
else:
fout.write(line)
@@ -359,7 +358,7 @@ def main():
except InputError as e:
print(e)
except UnicodeDecodeError:
raise ValueError("The encoding of {} is not Unicode.".format(path_in_idf))
raise ValueError('The encoding of {} is not Unicode.'.format(path_in_idf))
process_remaining_errors()
@@ -371,5 +370,5 @@ def main():
generate_c_output(fin, fout)
if __name__ == "__main__":
if __name__ == '__main__':
main()

265
tools/idf.py
View File

@@ -27,6 +27,7 @@
# any external libraries here - put in external script, or import in
# their specific function instead.
from __future__ import print_function
import codecs
import json
import locale
@@ -43,18 +44,18 @@ from pkgutil import iter_modules
sys.dont_write_bytecode = True
from idf_py_actions.errors import FatalError # noqa: E402
from idf_py_actions.tools import (executable_exists, idf_version, merge_action_lists, realpath) # noqa: E402
from idf_py_actions.tools import executable_exists, idf_version, merge_action_lists, realpath # noqa: E402
# Use this Python interpreter for any subprocesses we launch
PYTHON = sys.executable
# note: os.environ changes don't automatically propagate to child processes,
# you have to pass env=os.environ explicitly anywhere that we create a process
os.environ["PYTHON"] = sys.executable
os.environ['PYTHON'] = sys.executable
# Name of the program, normally 'idf.py'.
# Can be overridden from idf.bat using IDF_PY_PROGRAM_NAME
PROG = os.getenv("IDF_PY_PROGRAM_NAME", "idf.py")
PROG = os.getenv('IDF_PY_PROGRAM_NAME', 'idf.py')
# function prints warning when autocompletion is not being performed
@@ -73,37 +74,37 @@ def check_environment():
"""
checks_output = []
if not executable_exists(["cmake", "--version"]):
if not executable_exists(['cmake', '--version']):
debug_print_idf_version()
raise FatalError("'cmake' must be available on the PATH to use %s" % PROG)
# verify that IDF_PATH env variable is set
# find the directory idf.py is in, then the parent directory of this, and assume this is IDF_PATH
detected_idf_path = realpath(os.path.join(os.path.dirname(__file__), ".."))
if "IDF_PATH" in os.environ:
set_idf_path = realpath(os.environ["IDF_PATH"])
detected_idf_path = realpath(os.path.join(os.path.dirname(__file__), '..'))
if 'IDF_PATH' in os.environ:
set_idf_path = realpath(os.environ['IDF_PATH'])
if set_idf_path != detected_idf_path:
print_warning(
"WARNING: IDF_PATH environment variable is set to %s but %s path indicates IDF directory %s. "
"Using the environment variable directory, but results may be unexpected..." %
'WARNING: IDF_PATH environment variable is set to %s but %s path indicates IDF directory %s. '
'Using the environment variable directory, but results may be unexpected...' %
(set_idf_path, PROG, detected_idf_path))
else:
print_warning("Setting IDF_PATH environment variable: %s" % detected_idf_path)
os.environ["IDF_PATH"] = detected_idf_path
print_warning('Setting IDF_PATH environment variable: %s' % detected_idf_path)
os.environ['IDF_PATH'] = detected_idf_path
# check Python version
if sys.version_info[0] < 3:
print_warning("WARNING: Support for Python 2 is deprecated and will be removed in future versions.")
print_warning('WARNING: Support for Python 2 is deprecated and will be removed in future versions.')
elif sys.version_info[0] == 3 and sys.version_info[1] < 6:
print_warning("WARNING: Python 3 versions older than 3.6 are not supported.")
print_warning('WARNING: Python 3 versions older than 3.6 are not supported.')
# check Python dependencies
checks_output.append("Checking Python dependencies...")
checks_output.append('Checking Python dependencies...')
try:
out = subprocess.check_output(
[
os.environ["PYTHON"],
os.path.join(os.environ["IDF_PATH"], "tools", "check_python_dependencies.py"),
os.environ['PYTHON'],
os.path.join(os.environ['IDF_PATH'], 'tools', 'check_python_dependencies.py'),
],
env=os.environ,
)
@@ -131,9 +132,9 @@ def _safe_relpath(path, start=None):
def debug_print_idf_version():
version = idf_version()
if version:
print_warning("ESP-IDF %s" % version)
print_warning('ESP-IDF %s' % version)
else:
print_warning("ESP-IDF version unknown")
print_warning('ESP-IDF version unknown')
class PropertyDict(dict):
@@ -164,39 +165,39 @@ def init_cli(verbose_output=None):
self.since = None
self.removed = None
self.exit_with_error = None
self.custom_message = ""
self.custom_message = ''
if isinstance(deprecated, dict):
self.custom_message = deprecated.get("message", "")
self.since = deprecated.get("since", None)
self.removed = deprecated.get("removed", None)
self.exit_with_error = deprecated.get("exit_with_error", None)
self.custom_message = deprecated.get('message', '')
self.since = deprecated.get('since', None)
self.removed = deprecated.get('removed', None)
self.exit_with_error = deprecated.get('exit_with_error', None)
elif isinstance(deprecated, str):
self.custom_message = deprecated
def full_message(self, type="Option"):
def full_message(self, type='Option'):
if self.exit_with_error:
return "%s is deprecated %sand was removed%s.%s" % (
return '%s is deprecated %sand was removed%s.%s' % (
type,
"since %s " % self.since if self.since else "",
" in %s" % self.removed if self.removed else "",
" %s" % self.custom_message if self.custom_message else "",
'since %s ' % self.since if self.since else '',
' in %s' % self.removed if self.removed else '',
' %s' % self.custom_message if self.custom_message else '',
)
else:
return "%s is deprecated %sand will be removed in%s.%s" % (
return '%s is deprecated %sand will be removed in%s.%s' % (
type,
"since %s " % self.since if self.since else "",
" %s" % self.removed if self.removed else " future versions",
" %s" % self.custom_message if self.custom_message else "",
'since %s ' % self.since if self.since else '',
' %s' % self.removed if self.removed else ' future versions',
' %s' % self.custom_message if self.custom_message else '',
)
def help(self, text, type="Option", separator=" "):
text = text or ""
def help(self, text, type='Option', separator=' '):
text = text or ''
return self.full_message(type) + separator + text if self.deprecated else text
def short_help(self, text):
text = text or ""
return ("Deprecated! " + text) if self.deprecated else text
text = text or ''
return ('Deprecated! ' + text) if self.deprecated else text
def check_deprecation(ctx):
"""Prints deprecation warnings for arguments in given context"""
@@ -205,9 +206,9 @@ def init_cli(verbose_output=None):
if isinstance(option, Option) and option.deprecated and ctx.params[option.name] != default:
deprecation = Deprecation(option.deprecated)
if deprecation.exit_with_error:
raise FatalError("Error: %s" % deprecation.full_message('Option "%s"' % option.name))
raise FatalError('Error: %s' % deprecation.full_message('Option "%s"' % option.name))
else:
print_warning("Warning: %s" % deprecation.full_message('Option "%s"' % option.name))
print_warning('Warning: %s' % deprecation.full_message('Option "%s"' % option.name))
class Task(object):
def __init__(self, callback, name, aliases, dependencies, order_dependencies, action_args):
@@ -246,7 +247,7 @@ def init_cli(verbose_output=None):
self.help = self.help or self.callback.__doc__
if self.help is None:
self.help = ""
self.help = ''
if dependencies is None:
dependencies = []
@@ -255,19 +256,19 @@ def init_cli(verbose_output=None):
order_dependencies = []
# Show first line of help if short help is missing
self.short_help = self.short_help or self.help.split("\n")[0]
self.short_help = self.short_help or self.help.split('\n')[0]
if deprecated:
deprecation = Deprecation(deprecated)
self.short_help = deprecation.short_help(self.short_help)
self.help = deprecation.help(self.help, type="Command", separator="\n")
self.help = deprecation.help(self.help, type='Command', separator='\n')
# Add aliases to help string
if aliases:
aliases_help = "Aliases: %s." % ", ".join(aliases)
aliases_help = 'Aliases: %s.' % ', '.join(aliases)
self.help = "\n".join([self.help, aliases_help])
self.short_help = " ".join([aliases_help, self.short_help])
self.help = '\n'.join([self.help, aliases_help])
self.short_help = ' '.join([aliases_help, self.short_help])
self.unwrapped_callback = self.callback
if self.callback is not None:
@@ -290,9 +291,9 @@ def init_cli(verbose_output=None):
message = deprecation.full_message('Command "%s"' % self.name)
if deprecation.exit_with_error:
raise FatalError("Error: %s" % message)
raise FatalError('Error: %s' % message)
else:
print_warning("Warning: %s" % message)
print_warning('Warning: %s' % message)
self.deprecated = False # disable Click's built-in deprecation handling
@@ -307,7 +308,7 @@ def init_cli(verbose_output=None):
names - alias of 'param_decls'
"""
def __init__(self, **kwargs):
names = kwargs.pop("names")
names = kwargs.pop('names')
super(Argument, self).__init__(names, **kwargs)
class Scope(object):
@@ -319,25 +320,25 @@ def init_cli(verbose_output=None):
- shared - Opposite to 'global': when defined in global scope, also available for all actions
"""
SCOPES = ("default", "global", "shared")
SCOPES = ('default', 'global', 'shared')
def __init__(self, scope=None):
if scope is None:
self._scope = "default"
self._scope = 'default'
elif isinstance(scope, str) and scope in self.SCOPES:
self._scope = scope
elif isinstance(scope, Scope):
self._scope = str(scope)
else:
raise FatalError("Unknown scope for option: %s" % scope)
raise FatalError('Unknown scope for option: %s' % scope)
@property
def is_global(self):
return self._scope == "global"
return self._scope == 'global'
@property
def is_shared(self):
return self._scope == "shared"
return self._scope == 'shared'
def __str__(self):
return self._scope
@@ -356,7 +357,7 @@ def init_cli(verbose_output=None):
custom_message: Additional text to deprecation warning
"""
kwargs["param_decls"] = kwargs.pop("names")
kwargs['param_decls'] = kwargs.pop('names')
super(Option, self).__init__(**kwargs)
self.deprecated = deprecated
@@ -368,10 +369,10 @@ def init_cli(verbose_output=None):
self.help = deprecation.help(self.help)
if self.envvar:
self.help += " The default value can be set with the %s environment variable." % self.envvar
self.help += ' The default value can be set with the %s environment variable.' % self.envvar
if self.scope.is_global:
self.help += " This option can be used at most once either globally, or for one subcommand."
self.help += ' This option can be used at most once either globally, or for one subcommand.'
def get_help_record(self, ctx):
# Backport "hidden" parameter to click 5.0
@@ -387,7 +388,7 @@ def init_cli(verbose_output=None):
chain=True,
invoke_without_command=True,
result_callback=self.execute_tasks,
context_settings={"max_content_width": 140},
context_settings={'max_content_width': 140},
help=help,
)
self._actions = {}
@@ -405,7 +406,7 @@ def init_cli(verbose_output=None):
shared_options = []
# Global options
for option_args in all_actions.get("global_options", []):
for option_args in all_actions.get('global_options', []):
option = Option(**option_args)
self.params.append(option)
@@ -413,12 +414,12 @@ def init_cli(verbose_output=None):
shared_options.append(option)
# Global options validators
self.global_action_callbacks = all_actions.get("global_action_callbacks", [])
self.global_action_callbacks = all_actions.get('global_action_callbacks', [])
# Actions
for name, action in all_actions.get("actions", {}).items():
arguments = action.pop("arguments", [])
options = action.pop("options", [])
for name, action in all_actions.get('actions', {}).items():
arguments = action.pop('arguments', [])
options = action.pop('options', [])
if arguments is None:
arguments = []
@@ -427,7 +428,7 @@ def init_cli(verbose_output=None):
options = []
self._actions[name] = Action(name=name, **action)
for alias in [name] + action.get("aliases", []):
for alias in [name] + action.get('aliases', []):
self.commands_with_aliases[alias] = name
for argument_args in arguments:
@@ -465,72 +466,72 @@ def init_cli(verbose_output=None):
def _print_closing_message(self, args, actions):
# print a closing message of some kind
#
if any(t in str(actions) for t in ("flash", "dfu", "uf2", "uf2-app")):
print("Done")
if any(t in str(actions) for t in ('flash', 'dfu', 'uf2', 'uf2-app')):
print('Done')
return
if not os.path.exists(os.path.join(args.build_dir, "flasher_args.json")):
print("Done")
if not os.path.exists(os.path.join(args.build_dir, 'flasher_args.json')):
print('Done')
return
# Otherwise, if we built any binaries print a message about
# how to flash them
def print_flashing_message(title, key):
with open(os.path.join(args.build_dir, "flasher_args.json")) as f:
with open(os.path.join(args.build_dir, 'flasher_args.json')) as f:
flasher_args = json.load(f)
def flasher_path(f):
return _safe_relpath(os.path.join(args.build_dir, f))
if key != "project": # flashing a single item
if key != 'project': # flashing a single item
if key not in flasher_args:
# This is the case for 'idf.py bootloader' if Secure Boot is on, need to follow manual flashing steps
print("\n%s build complete." % title)
print('\n%s build complete.' % title)
return
cmd = ""
if (key == "bootloader"): # bootloader needs --flash-mode, etc to be passed in
cmd = " ".join(flasher_args["write_flash_args"]) + " "
cmd = ''
if (key == 'bootloader'): # bootloader needs --flash-mode, etc to be passed in
cmd = ' '.join(flasher_args['write_flash_args']) + ' '
cmd += flasher_args[key]["offset"] + " "
cmd += flasher_path(flasher_args[key]["file"])
cmd += flasher_args[key]['offset'] + ' '
cmd += flasher_path(flasher_args[key]['file'])
else: # flashing the whole project
cmd = " ".join(flasher_args["write_flash_args"]) + " "
cmd = ' '.join(flasher_args['write_flash_args']) + ' '
flash_items = sorted(
((o, f) for (o, f) in flasher_args["flash_files"].items() if len(o) > 0),
((o, f) for (o, f) in flasher_args['flash_files'].items() if len(o) > 0),
key=lambda x: int(x[0], 0),
)
for o, f in flash_items:
cmd += o + " " + flasher_path(f) + " "
cmd += o + ' ' + flasher_path(f) + ' '
print("\n%s build complete. To flash, run this command:" % title)
print('\n%s build complete. To flash, run this command:' % title)
print(
"%s %s -p %s -b %s --before %s --after %s --chip %s %s write_flash %s" % (
'%s %s -p %s -b %s --before %s --after %s --chip %s %s write_flash %s' % (
PYTHON,
_safe_relpath("%s/components/esptool_py/esptool/esptool.py" % os.environ["IDF_PATH"]),
args.port or "(PORT)",
_safe_relpath('%s/components/esptool_py/esptool/esptool.py' % os.environ['IDF_PATH']),
args.port or '(PORT)',
args.baud,
flasher_args["extra_esptool_args"]["before"],
flasher_args["extra_esptool_args"]["after"],
flasher_args["extra_esptool_args"]["chip"],
"--no-stub" if not flasher_args["extra_esptool_args"]["stub"] else "",
flasher_args['extra_esptool_args']['before'],
flasher_args['extra_esptool_args']['after'],
flasher_args['extra_esptool_args']['chip'],
'--no-stub' if not flasher_args['extra_esptool_args']['stub'] else '',
cmd.strip(),
))
print(
"or run 'idf.py -p %s %s'" % (
args.port or "(PORT)",
key + "-flash" if key != "project" else "flash",
args.port or '(PORT)',
key + '-flash' if key != 'project' else 'flash',
))
if "all" in actions or "build" in actions:
print_flashing_message("Project", "project")
if 'all' in actions or 'build' in actions:
print_flashing_message('Project', 'project')
else:
if "app" in actions:
print_flashing_message("App", "app")
if "partition_table" in actions:
print_flashing_message("Partition Table", "partition_table")
if "bootloader" in actions:
print_flashing_message("Bootloader", "bootloader")
if 'app' in actions:
print_flashing_message('App', 'app')
if 'partition_table' in actions:
print_flashing_message('Partition Table', 'partition_table')
if 'bootloader' in actions:
print_flashing_message('Bootloader', 'bootloader')
def execute_tasks(self, tasks, **kwargs):
ctx = click.get_current_context()
@@ -544,12 +545,12 @@ def init_cli(verbose_output=None):
dupplicated_tasks = sorted(
[item for item, count in Counter(task.name for task in tasks).items() if count > 1])
if dupplicated_tasks:
dupes = ", ".join('"%s"' % t for t in dupplicated_tasks)
dupes = ', '.join('"%s"' % t for t in dupplicated_tasks)
print_warning(
"WARNING: Command%s found in the list of commands more than once. " %
("s %s are" % dupes if len(dupplicated_tasks) > 1 else " %s is" % dupes) +
"Only first occurrence will be executed.")
'WARNING: Command%s found in the list of commands more than once. ' %
('s %s are' % dupes if len(dupplicated_tasks) > 1 else ' %s is' % dupes) +
'Only first occurrence will be executed.')
for task in tasks:
# Show help and exit if help is in the list of commands
@@ -569,7 +570,7 @@ def init_cli(verbose_output=None):
if global_value != default and local_value != default and global_value != local_value:
raise FatalError(
'Option "%s" provided for "%s" is already defined to a different value. '
"This option can appear at most once in the command line." % (key, task.name))
'This option can appear at most once in the command line.' % (key, task.name))
if local_value != default:
global_args[key] = local_value
@@ -638,9 +639,9 @@ def init_cli(verbose_output=None):
for task in tasks_to_run.values():
name_with_aliases = task.name
if task.aliases:
name_with_aliases += " (aliases: %s)" % ", ".join(task.aliases)
name_with_aliases += ' (aliases: %s)' % ', '.join(task.aliases)
print("Executing action: %s" % name_with_aliases)
print('Executing action: %s' % name_with_aliases)
task(ctx, global_args, task.action_args)
self._print_closing_message(global_args, tasks_to_run.keys())
@@ -652,21 +653,21 @@ def init_cli(verbose_output=None):
@click.command(
add_help_option=False,
context_settings={
"allow_extra_args": True,
"ignore_unknown_options": True
'allow_extra_args': True,
'ignore_unknown_options': True
},
)
@click.option("-C", "--project-dir", default=os.getcwd(), type=click.Path())
@click.option('-C', '--project-dir', default=os.getcwd(), type=click.Path())
def parse_project_dir(project_dir):
return realpath(project_dir)
# Set `complete_var` to not existing environment variable name to prevent early cmd completion
project_dir = parse_project_dir(standalone_mode=False, complete_var="_IDF.PY_COMPLETE_NOT_EXISTING")
project_dir = parse_project_dir(standalone_mode=False, complete_var='_IDF.PY_COMPLETE_NOT_EXISTING')
all_actions = {}
# Load extensions from components dir
idf_py_extensions_path = os.path.join(os.environ["IDF_PATH"], "tools", "idf_py_actions")
idf_py_extensions_path = os.path.join(os.environ['IDF_PATH'], 'tools', 'idf_py_actions')
extension_dirs = [realpath(idf_py_extensions_path)]
extra_paths = os.environ.get("IDF_EXTRA_ACTIONS_PATH")
extra_paths = os.environ.get('IDF_EXTRA_ACTIONS_PATH')
if extra_paths is not None:
for path in extra_paths.split(';'):
path = realpath(path)
@@ -702,12 +703,12 @@ def init_cli(verbose_output=None):
print_warning('WARNING: Cannot load idf.py extension "%s"' % name)
# Load extensions from project dir
if os.path.exists(os.path.join(project_dir, "idf_ext.py")):
if os.path.exists(os.path.join(project_dir, 'idf_ext.py')):
sys.path.append(project_dir)
try:
from idf_ext import action_extensions
except ImportError:
print_warning("Error importing extension file idf_ext.py. Skipping.")
print_warning('Error importing extension file idf_ext.py. Skipping.')
print_warning("Please make sure that it contains implementation (even if it's empty) of add_action_extensions")
try:
@@ -716,8 +717,8 @@ def init_cli(verbose_output=None):
pass
cli_help = (
"ESP-IDF CLI build management tool. "
"For commands that are not known to idf.py an attempt to execute it as a build system target will be made.")
'ESP-IDF CLI build management tool. '
'For commands that are not known to idf.py an attempt to execute it as a build system target will be made.')
return CLI(help=cli_help, verbose_output=verbose_output, all_actions=all_actions)
@@ -726,7 +727,7 @@ def main():
checks_output = check_environment()
cli = init_cli(verbose_output=checks_output)
# the argument `prog_name` must contain name of the file - not the absolute path to it!
cli(sys.argv[1:], prog_name=PROG, complete_var="_IDF.PY_COMPLETE")
cli(sys.argv[1:], prog_name=PROG, complete_var='_IDF.PY_COMPLETE')
def _valid_unicode_config():
@@ -736,73 +737,73 @@ def _valid_unicode_config():
# With python 3 unicode environment is required
try:
return codecs.lookup(locale.getpreferredencoding()).name != "ascii"
return codecs.lookup(locale.getpreferredencoding()).name != 'ascii'
except Exception:
return False
def _find_usable_locale():
try:
locales = subprocess.Popen(["locale", "-a"], stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0]
locales = subprocess.Popen(['locale', '-a'], stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0]
except OSError:
locales = ""
locales = ''
if isinstance(locales, bytes):
locales = locales.decode("ascii", "replace")
locales = locales.decode('ascii', 'replace')
usable_locales = []
for line in locales.splitlines():
locale = line.strip()
locale_name = locale.lower().replace("-", "")
locale_name = locale.lower().replace('-', '')
# C.UTF-8 is the best option, if supported
if locale_name == "c.utf8":
if locale_name == 'c.utf8':
return locale
if locale_name.endswith(".utf8"):
if locale_name.endswith('.utf8'):
# Make a preference of english locales
if locale.startswith("en_"):
if locale.startswith('en_'):
usable_locales.insert(0, locale)
else:
usable_locales.append(locale)
if not usable_locales:
raise FatalError(
"Support for Unicode filenames is required, but no suitable UTF-8 locale was found on your system."
" Please refer to the manual for your operating system for details on locale reconfiguration.")
'Support for Unicode filenames is required, but no suitable UTF-8 locale was found on your system.'
' Please refer to the manual for your operating system for details on locale reconfiguration.')
return usable_locales[0]
if __name__ == "__main__":
if __name__ == '__main__':
try:
# On MSYS2 we need to run idf.py with "winpty" in order to be able to cancel the subprocesses properly on
# keyboard interrupt (CTRL+C).
# Using an own global variable for indicating that we are running with "winpty" seems to be the most suitable
# option as os.environment['_'] contains "winpty" only when it is run manually from console.
WINPTY_VAR = "WINPTY"
WINPTY_EXE = "winpty"
if ("MSYSTEM" in os.environ) and (not os.environ.get("_", "").endswith(WINPTY_EXE)
WINPTY_VAR = 'WINPTY'
WINPTY_EXE = 'winpty'
if ('MSYSTEM' in os.environ) and (not os.environ.get('_', '').endswith(WINPTY_EXE)
and WINPTY_VAR not in os.environ):
if 'menuconfig' in sys.argv:
# don't use winpty for menuconfig because it will print weird characters
main()
else:
os.environ[WINPTY_VAR] = "1" # the value is of no interest to us
os.environ[WINPTY_VAR] = '1' # the value is of no interest to us
# idf.py calls itself with "winpty" and WINPTY global variable set
ret = subprocess.call([WINPTY_EXE, sys.executable] + sys.argv, env=os.environ)
if ret:
raise SystemExit(ret)
elif os.name == "posix" and not _valid_unicode_config():
elif os.name == 'posix' and not _valid_unicode_config():
# Trying to find best utf-8 locale available on the system and restart python with it
best_locale = _find_usable_locale()
print_warning(
"Your environment is not configured to handle unicode filenames outside of ASCII range."
" Environment variable LC_ALL is temporary set to %s for unicode support." % best_locale)
'Your environment is not configured to handle unicode filenames outside of ASCII range.'
' Environment variable LC_ALL is temporary set to %s for unicode support.' % best_locale)
os.environ["LC_ALL"] = best_locale
os.environ['LC_ALL'] = best_locale
ret = subprocess.call([sys.executable] + sys.argv, env=os.environ)
if ret:
raise SystemExit(ret)

275
tools/idf_monitor.py
View File

@@ -29,35 +29,36 @@
#
# Originally released under BSD-3-Clause license.
#
from __future__ import print_function, division
from __future__ import unicode_literals
from builtins import chr
from builtins import object
from builtins import bytes
import subprocess
from __future__ import division, print_function, unicode_literals
import argparse
import codecs
import datetime
import re
import os
import re
import subprocess
from builtins import bytes, chr, object
try:
import queue
except ImportError:
import Queue as queue
import shlex
import time
import sys
import serial
import serial.tools.list_ports
import serial.tools.miniterm as miniterm
import threading
import ctypes
import json
import shlex
import sys
import tempfile
import textwrap
import threading
import time
import types
from distutils.version import StrictVersion
from io import open
import textwrap
import tempfile
import json
import serial
import serial.tools.list_ports
import serial.tools.miniterm as miniterm
try:
import websocket
@@ -97,7 +98,7 @@ ANSI_NORMAL = '\033[0m'
def color_print(message, color, newline='\n'):
""" Print a message to stderr with colored highlighting """
sys.stderr.write("%s%s%s%s" % (color, message, ANSI_NORMAL, newline))
sys.stderr.write('%s%s%s%s' % (color, message, ANSI_NORMAL, newline))
def yellow_print(message, newline='\n'):
@@ -108,7 +109,7 @@ def red_print(message, newline='\n'):
color_print(message, ANSI_RED, newline)
__version__ = "1.1"
__version__ = '1.1'
# Tags for tuples in queues
TAG_KEY = 0
@@ -119,14 +120,14 @@ TAG_CMD = 3
# regex matches an potential PC value (0x4xxxxxxx)
MATCH_PCADDR = re.compile(r'0x4[0-9a-f]{7}', re.IGNORECASE)
DEFAULT_TOOLCHAIN_PREFIX = "xtensa-esp32-elf-"
DEFAULT_TOOLCHAIN_PREFIX = 'xtensa-esp32-elf-'
DEFAULT_PRINT_FILTER = ""
DEFAULT_PRINT_FILTER = ''
# coredump related messages
COREDUMP_UART_START = b"================= CORE DUMP START ================="
COREDUMP_UART_END = b"================= CORE DUMP END ================="
COREDUMP_UART_PROMPT = b"Press Enter to print core dump to UART..."
COREDUMP_UART_START = b'================= CORE DUMP START ================='
COREDUMP_UART_END = b'================= CORE DUMP END ================='
COREDUMP_UART_PROMPT = b'Press Enter to print core dump to UART...'
# coredump states
COREDUMP_IDLE = 0
@@ -134,21 +135,21 @@ COREDUMP_READING = 1
COREDUMP_DONE = 2
# coredump decoding options
COREDUMP_DECODE_DISABLE = "disable"
COREDUMP_DECODE_INFO = "info"
COREDUMP_DECODE_DISABLE = 'disable'
COREDUMP_DECODE_INFO = 'info'
# panic handler related messages
PANIC_START = r"Core \s*\d+ register dump:"
PANIC_END = b"ELF file SHA256:"
PANIC_STACK_DUMP = b"Stack memory:"
PANIC_START = r'Core \s*\d+ register dump:'
PANIC_END = b'ELF file SHA256:'
PANIC_STACK_DUMP = b'Stack memory:'
# panic handler decoding states
PANIC_IDLE = 0
PANIC_READING = 1
# panic handler decoding options
PANIC_DECODE_DISABLE = "disable"
PANIC_DECODE_BACKTRACE = "backtrace"
PANIC_DECODE_DISABLE = 'disable'
PANIC_DECODE_BACKTRACE = 'backtrace'
class StoppableThread(object):
@@ -266,11 +267,11 @@ class ConsoleReader(StoppableThread):
class ConsoleParser(object):
def __init__(self, eol="CRLF"):
def __init__(self, eol='CRLF'):
self.translate_eol = {
"CRLF": lambda c: c.replace("\n", "\r\n"),
"CR": lambda c: c.replace("\n", "\r"),
"LF": lambda c: c.replace("\r", "\n"),
'CRLF': lambda c: c.replace('\n', '\r\n'),
'CR': lambda c: c.replace('\n', '\r'),
'LF': lambda c: c.replace('\r', '\n'),
}[eol]
self.menu_key = CTRL_T
self.exit_key = CTRL_RBRACKET
@@ -308,7 +309,7 @@ class ConsoleParser(object):
elif c == CTRL_L: # Toggle saving output into file
ret = (TAG_CMD, CMD_TOGGLE_LOGGING)
elif c == CTRL_P:
yellow_print("Pause app (enter bootloader mode), press Ctrl-T Ctrl-R to restart")
yellow_print('Pause app (enter bootloader mode), press Ctrl-T Ctrl-R to restart')
# to fast trigger pause without press menu key
ret = (TAG_CMD, CMD_ENTER_BOOT)
elif c in [CTRL_X, 'x', 'X']: # Exiting from within the menu
@@ -448,7 +449,7 @@ class LineMatcher(object):
self._re = re.compile(r'^(?:\033\[[01];?[0-9]+m?)?([EWIDV]) \([0-9]+\) ([^:]+): ')
items = print_filter.split()
if len(items) == 0:
self._dict["*"] = self.LEVEL_V # default is to print everything
self._dict['*'] = self.LEVEL_V # default is to print everything
for f in items:
s = f.split(r':')
if len(s) == 1:
@@ -472,13 +473,13 @@ class LineMatcher(object):
lev = self.level[m.group(1)]
if m.group(2) in self._dict:
return self._dict[m.group(2)] >= lev
return self._dict.get("*", self.LEVEL_N) >= lev
return self._dict.get('*', self.LEVEL_N) >= lev
except (KeyError, IndexError):
# Regular line written with something else than ESP_LOG*
# or an empty line.
pass
# We need something more than "*.N" for printing.
return self._dict.get("*", self.LEVEL_N) > self.LEVEL_N
return self._dict.get('*', self.LEVEL_N) > self.LEVEL_N
class SerialStopException(Exception):
@@ -497,8 +498,8 @@ class Monitor(object):
Main difference is that all event processing happens in the main thread, not the worker threads.
"""
def __init__(self, serial_instance, elf_file, print_filter, make="make", encrypted=False,
toolchain_prefix=DEFAULT_TOOLCHAIN_PREFIX, eol="CRLF",
def __init__(self, serial_instance, elf_file, print_filter, make='make', encrypted=False,
toolchain_prefix=DEFAULT_TOOLCHAIN_PREFIX, eol='CRLF',
decode_coredumps=COREDUMP_DECODE_INFO,
decode_panic=PANIC_DECODE_DISABLE,
target=None,
@@ -524,7 +525,7 @@ class Monitor(object):
self.console.getkey = types.MethodType(getkey_patched, self.console)
socket_mode = serial_instance.port.startswith("socket://") # testing hook - data from serial can make exit the monitor
socket_mode = serial_instance.port.startswith('socket://') # testing hook - data from serial can make exit the monitor
self.serial = serial_instance
self.console_parser = ConsoleParser(eol)
self.console_reader = ConsoleReader(self.console, self.event_queue, self.cmd_queue, self.console_parser, socket_mode)
@@ -540,9 +541,9 @@ class Monitor(object):
self.target = target
# internal state
self._last_line_part = b""
self._gdb_buffer = b""
self._pc_address_buffer = b""
self._last_line_part = b''
self._gdb_buffer = b''
self._pc_address_buffer = b''
self._line_matcher = LineMatcher(print_filter)
self._invoke_processing_last_line_timer = None
self._force_line_print = False
@@ -551,10 +552,10 @@ class Monitor(object):
self._log_file = None
self._decode_coredumps = decode_coredumps
self._reading_coredump = COREDUMP_IDLE
self._coredump_buffer = b""
self._coredump_buffer = b''
self._decode_panic = decode_panic
self._reading_panic = PANIC_IDLE
self._panic_buffer = b""
self._panic_buffer = b''
def invoke_processing_last_line(self):
self.event_queue.put((TAG_SERIAL_FLUSH, b''), False)
@@ -596,9 +597,9 @@ class Monitor(object):
elif event_tag == TAG_SERIAL_FLUSH:
self.handle_serial_input(data, finalize_line=True)
else:
raise RuntimeError("Bad event data %r" % ((event_tag,data),))
raise RuntimeError('Bad event data %r' % ((event_tag,data),))
except SerialStopException:
sys.stderr.write(ANSI_NORMAL + "Stopping condition has been received\n")
sys.stderr.write(ANSI_NORMAL + 'Stopping condition has been received\n')
finally:
try:
self.console_reader.stop()
@@ -609,24 +610,24 @@ class Monitor(object):
self._invoke_processing_last_line_timer = None
except Exception:
pass
sys.stderr.write(ANSI_NORMAL + "\n")
sys.stderr.write(ANSI_NORMAL + '\n')
def handle_serial_input(self, data, finalize_line=False):
sp = data.split(b'\n')
if self._last_line_part != b"":
if self._last_line_part != b'':
# add unprocessed part from previous "data" to the first line
sp[0] = self._last_line_part + sp[0]
self._last_line_part = b""
if sp[-1] != b"":
self._last_line_part = b''
if sp[-1] != b'':
# last part is not a full line
self._last_line_part = sp.pop()
for line in sp:
if line != b"":
if line != b'':
if self._serial_check_exit and line == self.console_parser.exit_key.encode('latin-1'):
raise SerialStopException()
self.check_panic_decode_trigger(line)
self.check_coredump_trigger_before_print(line)
if self._force_line_print or self._line_matcher.match(line.decode(errors="ignore")):
if self._force_line_print or self._line_matcher.match(line.decode(errors='ignore')):
self._print(line + b'\n')
self.handle_possible_pc_address_in_line(line)
self.check_coredump_trigger_after_print(line)
@@ -636,8 +637,8 @@ class Monitor(object):
# default we don't touch it and just wait for the arrival of the rest
# of the line. But after some time when we didn't received it we need
# to make a decision.
if self._last_line_part != b"":
if self._force_line_print or (finalize_line and self._line_matcher.match(self._last_line_part.decode(errors="ignore"))):
if self._last_line_part != b'':
if self._force_line_print or (finalize_line and self._line_matcher.match(self._last_line_part.decode(errors='ignore'))):
self._force_line_print = True
self._print(self._last_line_part)
self.handle_possible_pc_address_in_line(self._last_line_part)
@@ -651,15 +652,15 @@ class Monitor(object):
# GDB sequence can be cut in half also. GDB sequence is 7
# characters long, therefore, we save the last 6 characters.
self._gdb_buffer = self._last_line_part[-6:]
self._last_line_part = b""
self._last_line_part = b''
# else: keeping _last_line_part and it will be processed the next time
# handle_serial_input is invoked
def handle_possible_pc_address_in_line(self, line):
line = self._pc_address_buffer + line
self._pc_address_buffer = b""
self._pc_address_buffer = b''
if self.enable_address_decoding:
for m in re.finditer(MATCH_PCADDR, line.decode(errors="ignore")):
for m in re.finditer(MATCH_PCADDR, line.decode(errors='ignore')):
self.lookup_pc_address(m.group())
def __enter__(self):
@@ -675,7 +676,7 @@ class Monitor(object):
def prompt_next_action(self, reason):
self.console.setup() # set up console to trap input characters
try:
red_print("--- {}".format(reason))
red_print('--- {}'.format(reason))
red_print(self.console_parser.get_next_action_text())
k = CTRL_T # ignore CTRL-T here, so people can muscle-memory Ctrl-T Ctrl-F, etc.
@@ -698,31 +699,31 @@ class Monitor(object):
popen_args = self.make + [target]
else:
popen_args = [self.make, target]
yellow_print("Running %s..." % " ".join(popen_args))
yellow_print('Running %s...' % ' '.join(popen_args))
p = subprocess.Popen(popen_args, env=os.environ)
try:
p.wait()
except KeyboardInterrupt:
p.wait()
if p.returncode != 0:
self.prompt_next_action("Build failed")
self.prompt_next_action('Build failed')
else:
self.output_enable(True)
def lookup_pc_address(self, pc_addr):
cmd = ["%saddr2line" % self.toolchain_prefix,
"-pfiaC", "-e", self.elf_file, pc_addr]
cmd = ['%saddr2line' % self.toolchain_prefix,
'-pfiaC', '-e', self.elf_file, pc_addr]
try:
translation = subprocess.check_output(cmd, cwd=".")
if b"?? ??:0" not in translation:
translation = subprocess.check_output(cmd, cwd='.')
if b'?? ??:0' not in translation:
self._print(translation.decode(), console_printer=yellow_print)
except OSError as e:
red_print("%s: %s" % (" ".join(cmd), e))
red_print('%s: %s' % (' '.join(cmd), e))
def check_gdbstub_trigger(self, line):
line = self._gdb_buffer + line
self._gdb_buffer = b""
m = re.search(b"\\$(T..)#(..)", line) # look for a gdb "reason" for a break
self._gdb_buffer = b''
m = re.search(b'\\$(T..)#(..)', line) # look for a gdb "reason" for a break
if m is not None:
try:
chsum = sum(ord(bytes([p])) for p in m.group(1)) & 0xFF
@@ -741,27 +742,27 @@ class Monitor(object):
else:
self.run_gdb()
else:
red_print("Malformed gdb message... calculated checksum %02x received %02x" % (chsum, calc_chsum))
red_print('Malformed gdb message... calculated checksum %02x received %02x' % (chsum, calc_chsum))
def check_coredump_trigger_before_print(self, line):
if self._decode_coredumps == COREDUMP_DECODE_DISABLE:
return
if COREDUMP_UART_PROMPT in line:
yellow_print("Initiating core dump!")
yellow_print('Initiating core dump!')
self.event_queue.put((TAG_KEY, '\n'))
return
if COREDUMP_UART_START in line:
yellow_print("Core dump started (further output muted)")
yellow_print('Core dump started (further output muted)')
self._reading_coredump = COREDUMP_READING
self._coredump_buffer = b""
self._coredump_buffer = b''
self._output_enabled = False
return
if COREDUMP_UART_END in line:
self._reading_coredump = COREDUMP_DONE
yellow_print("\nCore dump finished!")
yellow_print('\nCore dump finished!')
self.process_coredump()
return
@@ -771,7 +772,7 @@ class Monitor(object):
self._coredump_buffer += line.replace(b'\r', b'') + b'\n'
new_buffer_len_kb = len(self._coredump_buffer) // kb
if new_buffer_len_kb > buffer_len_kb:
yellow_print("Received %3d kB..." % (new_buffer_len_kb), newline='\r')
yellow_print('Received %3d kB...' % (new_buffer_len_kb), newline='\r')
def check_coredump_trigger_after_print(self, line):
if self._decode_coredumps == COREDUMP_DECODE_DISABLE:
@@ -781,18 +782,18 @@ class Monitor(object):
if not self._output_enabled and self._reading_coredump == COREDUMP_DONE:
self._reading_coredump = COREDUMP_IDLE
self._output_enabled = True
self._coredump_buffer = b""
self._coredump_buffer = b''
def process_coredump(self):
if self._decode_coredumps != COREDUMP_DECODE_INFO:
raise NotImplementedError("process_coredump: %s not implemented" % self._decode_coredumps)
raise NotImplementedError('process_coredump: %s not implemented' % self._decode_coredumps)
coredump_script = os.path.join(os.path.dirname(__file__), "..", "components", "espcoredump", "espcoredump.py")
coredump_script = os.path.join(os.path.dirname(__file__), '..', 'components', 'espcoredump', 'espcoredump.py')
coredump_file = None
try:
# On Windows, the temporary file can't be read unless it is closed.
# Set delete=False and delete the file manually later.
with tempfile.NamedTemporaryFile(mode="wb", delete=False) as coredump_file:
with tempfile.NamedTemporaryFile(mode='wb', delete=False) as coredump_file:
coredump_file.write(self._coredump_buffer)
coredump_file.flush()
@@ -808,9 +809,9 @@ class Monitor(object):
else:
cmd = [sys.executable,
coredump_script,
"info_corefile",
"--core", coredump_file.name,
"--core-format", "b64",
'info_corefile',
'--core', coredump_file.name,
'--core-format', 'b64',
self.elf_file
]
output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
@@ -818,7 +819,7 @@ class Monitor(object):
self._print(output)
self._output_enabled = False # Will be reenabled in check_coredump_trigger_after_print
except subprocess.CalledProcessError as e:
yellow_print("Failed to run espcoredump script: {}\n{}\n\n".format(e, e.output))
yellow_print('Failed to run espcoredump script: {}\n{}\n\n'.format(e, e.output))
self._output_enabled = True
self._print(COREDUMP_UART_START + b'\n')
self._print(self._coredump_buffer)
@@ -834,9 +835,9 @@ class Monitor(object):
if self._decode_panic == PANIC_DECODE_DISABLE:
return
if self._reading_panic == PANIC_IDLE and re.search(PANIC_START, line.decode("ascii", errors='ignore')):
if self._reading_panic == PANIC_IDLE and re.search(PANIC_START, line.decode('ascii', errors='ignore')):
self._reading_panic = PANIC_READING
yellow_print("Stack dump detected")
yellow_print('Stack dump detected')
if self._reading_panic == PANIC_READING and PANIC_STACK_DUMP in line:
self._output_enabled = False
@@ -848,33 +849,33 @@ class Monitor(object):
self._reading_panic = PANIC_IDLE
self._output_enabled = True
self.process_panic_output(self._panic_buffer)
self._panic_buffer = b""
self._panic_buffer = b''
def process_panic_output(self, panic_output):
panic_output_decode_script = os.path.join(os.path.dirname(__file__), "..", "tools", "gdb_panic_server.py")
panic_output_decode_script = os.path.join(os.path.dirname(__file__), '..', 'tools', 'gdb_panic_server.py')
panic_output_file = None
try:
# On Windows, the temporary file can't be read unless it is closed.
# Set delete=False and delete the file manually later.
with tempfile.NamedTemporaryFile(mode="wb", delete=False) as panic_output_file:
with tempfile.NamedTemporaryFile(mode='wb', delete=False) as panic_output_file:
panic_output_file.write(panic_output)
panic_output_file.flush()
cmd = [self.toolchain_prefix + "gdb",
"--batch", "-n",
cmd = [self.toolchain_prefix + 'gdb',
'--batch', '-n',
self.elf_file,
"-ex", "target remote | \"{python}\" \"{script}\" --target {target} \"{output_file}\""
'-ex', "target remote | \"{python}\" \"{script}\" --target {target} \"{output_file}\""
.format(python=sys.executable,
script=panic_output_decode_script,
target=self.target,
output_file=panic_output_file.name),
"-ex", "bt"]
'-ex', 'bt']
output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
yellow_print("\nBacktrace:\n\n")
yellow_print('\nBacktrace:\n\n')
self._print(output)
except subprocess.CalledProcessError as e:
yellow_print("Failed to run gdb_panic_server.py script: {}\n{}\n\n".format(e, e.output))
yellow_print('Failed to run gdb_panic_server.py script: {}\n{}\n\n'.format(e, e.output))
self._print(panic_output)
finally:
if panic_output_file is not None:
@@ -887,15 +888,15 @@ class Monitor(object):
with self: # disable console control
sys.stderr.write(ANSI_NORMAL)
try:
cmd = ["%sgdb" % self.toolchain_prefix,
"-ex", "set serial baud %d" % self.serial.baudrate,
"-ex", "target remote %s" % self.serial.port,
"-ex", "interrupt", # monitor has already parsed the first 'reason' command, need a second
cmd = ['%sgdb' % self.toolchain_prefix,
'-ex', 'set serial baud %d' % self.serial.baudrate,
'-ex', 'target remote %s' % self.serial.port,
'-ex', 'interrupt', # monitor has already parsed the first 'reason' command, need a second
self.elf_file]
process = subprocess.Popen(cmd, cwd=".")
process = subprocess.Popen(cmd, cwd='.')
process.wait()
except OSError as e:
red_print("%s: %s" % (" ".join(cmd), e))
red_print('%s: %s' % (' '.join(cmd), e))
except KeyboardInterrupt:
pass # happens on Windows, maybe other OSes
finally:
@@ -906,17 +907,17 @@ class Monitor(object):
pass
try:
# also on Linux, maybe other OSes, gdb sometimes exits uncleanly and breaks the tty mode
subprocess.call(["stty", "sane"])
subprocess.call(['stty', 'sane'])
except Exception:
pass # don't care if there's no stty, we tried...
self.prompt_next_action("gdb exited")
self.prompt_next_action('gdb exited')
def output_enable(self, enable):
self._output_enabled = enable
def output_toggle(self):
self._output_enabled = not self._output_enabled
yellow_print("\nToggle output display: {}, Type Ctrl-T Ctrl-Y to show/disable output again.".format(self._output_enabled))
yellow_print('\nToggle output display: {}, Type Ctrl-T Ctrl-Y to show/disable output again.'.format(self._output_enabled))
def toggle_logging(self):
if self._log_file:
@@ -927,21 +928,21 @@ class Monitor(object):
def start_logging(self):
if not self._log_file:
try:
name = "log.{}.{}.txt".format(os.path.splitext(os.path.basename(self.elf_file))[0],
name = 'log.{}.{}.txt'.format(os.path.splitext(os.path.basename(self.elf_file))[0],
datetime.datetime.now().strftime('%Y%m%d%H%M%S'))
self._log_file = open(name, "wb+")
yellow_print("\nLogging is enabled into file {}".format(name))
self._log_file = open(name, 'wb+')
yellow_print('\nLogging is enabled into file {}'.format(name))
except Exception as e:
red_print("\nLog file {} cannot be created: {}".format(name, e))
red_print('\nLog file {} cannot be created: {}'.format(name, e))
def stop_logging(self):
if self._log_file:
try:
name = self._log_file.name
self._log_file.close()
yellow_print("\nLogging is disabled and file {} has been closed".format(name))
yellow_print('\nLogging is disabled and file {} has been closed'.format(name))
except Exception as e:
red_print("\nLog file cannot be closed: {}".format(e))
red_print('\nLog file cannot be closed: {}'.format(e))
finally:
self._log_file = None
@@ -956,7 +957,7 @@ class Monitor(object):
string = string.encode()
self._log_file.write(string)
except Exception as e:
red_print("\nCannot write to file: {}".format(e))
red_print('\nCannot write to file: {}'.format(e))
# don't fill-up the screen with the previous errors (probably consequent prints would fail also)
self.stop_logging()
@@ -972,9 +973,9 @@ class Monitor(object):
self.serial.setDTR(self.serial.dtr) # usbser.sys workaround
self.output_enable(True)
elif cmd == CMD_MAKE:
self.run_make("encrypted-flash" if self.encrypted else "flash")
self.run_make('encrypted-flash' if self.encrypted else 'flash')
elif cmd == CMD_APP_FLASH:
self.run_make("encrypted-app-flash" if self.encrypted else "app-flash")
self.run_make('encrypted-app-flash' if self.encrypted else 'app-flash')
elif cmd == CMD_OUTPUT_TOGGLE:
self.output_toggle()
elif cmd == CMD_TOGGLE_LOGGING:
@@ -990,11 +991,11 @@ class Monitor(object):
time.sleep(0.45) # timeouts taken from esptool.py, includes esp32r0 workaround. defaults: 0.05
self.serial.setDTR(False) # IO0=HIGH, done
else:
raise RuntimeError("Bad command data %d" % (cmd))
raise RuntimeError('Bad command data %d' % (cmd))
def main():
parser = argparse.ArgumentParser("idf_monitor - a serial output monitor for esp-idf")
parser = argparse.ArgumentParser('idf_monitor - a serial output monitor for esp-idf')
parser.add_argument(
'--port', '-p',
@@ -1005,8 +1006,8 @@ def main():
parser.add_argument(
'--disable-address-decoding', '-d',
help="Don't print lines about decoded addresses from the application ELF file.",
action="store_true",
default=True if os.environ.get("ESP_MONITOR_DECODE") == 0 else False
action='store_true',
default=True if os.environ.get('ESP_MONITOR_DECODE') == 0 else False
)
parser.add_argument(
@@ -1027,14 +1028,14 @@ def main():
parser.add_argument(
'--toolchain-prefix',
help="Triplet prefix to add before cross-toolchain names",
help='Triplet prefix to add before cross-toolchain names',
default=DEFAULT_TOOLCHAIN_PREFIX)
parser.add_argument(
"--eol",
'--eol',
choices=['CR', 'LF', 'CRLF'],
type=lambda c: c.upper(),
help="End of line to use when sending to the serial port",
help='End of line to use when sending to the serial port',
default='CR')
parser.add_argument(
@@ -1043,47 +1044,47 @@ def main():
parser.add_argument(
'--print_filter',
help="Filtering string",
help='Filtering string',
default=DEFAULT_PRINT_FILTER)
parser.add_argument(
'--decode-coredumps',
choices=[COREDUMP_DECODE_INFO, COREDUMP_DECODE_DISABLE],
default=COREDUMP_DECODE_INFO,
help="Handling of core dumps found in serial output"
help='Handling of core dumps found in serial output'
)
parser.add_argument(
'--decode-panic',
choices=[PANIC_DECODE_BACKTRACE, PANIC_DECODE_DISABLE],
default=PANIC_DECODE_DISABLE,
help="Handling of panic handler info found in serial output"
help='Handling of panic handler info found in serial output'
)
parser.add_argument(
'--target',
required=False,
help="Target name (used when stack dump decoding is enabled)"
help='Target name (used when stack dump decoding is enabled)'
)
parser.add_argument(
'--ws',
default=os.environ.get('ESP_IDF_MONITOR_WS', None),
help="WebSocket URL for communicating with IDE tools for debugging purposes"
help='WebSocket URL for communicating with IDE tools for debugging purposes'
)
args = parser.parse_args()
# GDB uses CreateFile to open COM port, which requires the COM name to be r'\\.\COMx' if the COM
# number is larger than 10
if os.name == 'nt' and args.port.startswith("COM"):
if os.name == 'nt' and args.port.startswith('COM'):
args.port = args.port.replace('COM', r'\\.\COM')
yellow_print("--- WARNING: GDB cannot open serial ports accessed as COMx")
yellow_print("--- Using %s instead..." % args.port)
elif args.port.startswith("/dev/tty.") and sys.platform == 'darwin':
args.port = args.port.replace("/dev/tty.", "/dev/cu.")
yellow_print("--- WARNING: Serial ports accessed as /dev/tty.* will hang gdb if launched.")
yellow_print("--- Using %s instead..." % args.port)
yellow_print('--- WARNING: GDB cannot open serial ports accessed as COMx')
yellow_print('--- Using %s instead...' % args.port)
elif args.port.startswith('/dev/tty.') and sys.platform == 'darwin':
args.port = args.port.replace('/dev/tty.', '/dev/cu.')
yellow_print('--- WARNING: Serial ports accessed as /dev/tty.* will hang gdb if launched.')
yellow_print('--- Using %s instead...' % args.port)
serial_instance = serial.serial_for_url(args.port, args.baud,
do_not_open=True)
@@ -1097,16 +1098,16 @@ def main():
# all of the child makes we need (the -j argument remains part of
# MAKEFLAGS)
try:
makeflags = os.environ["MAKEFLAGS"]
makeflags = re.sub(r"--jobserver[^ =]*=[0-9,]+ ?", "", makeflags)
os.environ["MAKEFLAGS"] = makeflags
makeflags = os.environ['MAKEFLAGS']
makeflags = re.sub(r'--jobserver[^ =]*=[0-9,]+ ?', '', makeflags)
os.environ['MAKEFLAGS'] = makeflags
except KeyError:
pass # not running a make jobserver
# Pass the actual used port to callee of idf_monitor (e.g. make) through `ESPPORT` environment
# variable
# To make sure the key as well as the value are str type, by the requirements of subprocess
espport_key = str("ESPPORT")
espport_key = str('ESPPORT')
espport_val = str(args.port)
os.environ.update({espport_key: espport_val})
@@ -1310,5 +1311,5 @@ if os.name == 'nt':
pass
if __name__ == "__main__":
if __name__ == '__main__':
main()

44
tools/idf_py_actions/constants.py
View File

@@ -4,38 +4,38 @@ import os
import platform
# Make flavors, across the various kinds of Windows environments & POSIX...
if "MSYSTEM" in os.environ: # MSYS
MAKE_CMD = "make"
MAKE_GENERATOR = "MSYS Makefiles"
elif os.name == "nt": # other Windows
MAKE_CMD = "mingw32-make"
MAKE_GENERATOR = "MinGW Makefiles"
elif platform.system() == "FreeBSD":
MAKE_CMD = "gmake"
MAKE_GENERATOR = "Unix Makefiles"
if 'MSYSTEM' in os.environ: # MSYS
MAKE_CMD = 'make'
MAKE_GENERATOR = 'MSYS Makefiles'
elif os.name == 'nt': # other Windows
MAKE_CMD = 'mingw32-make'
MAKE_GENERATOR = 'MinGW Makefiles'
elif platform.system() == 'FreeBSD':
MAKE_CMD = 'gmake'
MAKE_GENERATOR = 'Unix Makefiles'
else:
MAKE_CMD = "make"
MAKE_GENERATOR = "Unix Makefiles"
MAKE_CMD = 'make'
MAKE_GENERATOR = 'Unix Makefiles'
GENERATORS = collections.OrderedDict([
# - command: build command line
# - version: version command line
# - dry_run: command to run in dry run mode
# - verbose_flag: verbose flag
("Ninja", {
"command": ["ninja"],
"version": ["ninja", "--version"],
"dry_run": ["ninja", "-n"],
"verbose_flag": "-v"
('Ninja', {
'command': ['ninja'],
'version': ['ninja', '--version'],
'dry_run': ['ninja', '-n'],
'verbose_flag': '-v'
}),
(MAKE_GENERATOR, {
"command": [MAKE_CMD, "-j", str(multiprocessing.cpu_count() + 2)],
"version": [MAKE_CMD, "--version"],
"dry_run": [MAKE_CMD, "-n"],
"verbose_flag": "VERBOSE=1",
'command': [MAKE_CMD, '-j', str(multiprocessing.cpu_count() + 2)],
'version': [MAKE_CMD, '--version'],
'dry_run': [MAKE_CMD, '-n'],
'verbose_flag': 'VERBOSE=1',
})
])
SUPPORTED_TARGETS = ["esp32", "esp32s2"]
SUPPORTED_TARGETS = ['esp32', 'esp32s2']
PREVIEW_TARGETS = ["esp32s3", "esp32c3", "linux"]
PREVIEW_TARGETS = ['esp32s3', 'esp32c3', 'linux']

436
tools/idf_py_actions/core_ext.py
View File

@@ -5,11 +5,11 @@ import subprocess
import sys
import click
from idf_py_actions.constants import GENERATORS, SUPPORTED_TARGETS, PREVIEW_TARGETS
from idf_py_actions.constants import GENERATORS, PREVIEW_TARGETS, SUPPORTED_TARGETS
from idf_py_actions.errors import FatalError
from idf_py_actions.global_options import global_options
from idf_py_actions.tools import ensure_build_directory, idf_version, merge_action_lists, realpath, run_target, TargetChoice
from idf_py_actions.tools import (TargetChoice, ensure_build_directory, idf_version, merge_action_lists, realpath,
run_target)
def action_extensions(base_actions, project_path):
@@ -46,7 +46,7 @@ def action_extensions(base_actions, project_path):
ensure_build_directory(args, ctx.info_name)
try:
subprocess.check_output(GENERATORS[args.generator]["dry_run"] + [target_name], cwd=args.build_dir)
subprocess.check_output(GENERATORS[args.generator]['dry_run'] + [target_name], cwd=args.build_dir)
except Exception:
raise FatalError(
@@ -67,21 +67,21 @@ def action_extensions(base_actions, project_path):
if not os.path.isdir(args.build_dir):
print("Build directory '%s' not found. Nothing to clean." % args.build_dir)
return
build_target("clean", ctx, args)
build_target('clean', ctx, args)
def _delete_windows_symlinks(directory):
"""
It deletes symlinks recursively on Windows. It is useful for Python 2 which doesn't detect symlinks on Windows.
"""
deleted_paths = []
if os.name == "nt":
if os.name == 'nt':
import ctypes
for root, dirnames, _filenames in os.walk(directory):
for d in dirnames:
full_path = os.path.join(root, d)
try:
full_path = full_path.decode("utf-8")
full_path = full_path.decode('utf-8')
except Exception:
pass
if ctypes.windll.kernel32.GetFileAttributesW(full_path) & 0x0400:
@@ -98,11 +98,11 @@ def action_extensions(base_actions, project_path):
print("Build directory '%s' is empty. Nothing to clean." % build_dir)
return
if not os.path.exists(os.path.join(build_dir, "CMakeCache.txt")):
if not os.path.exists(os.path.join(build_dir, 'CMakeCache.txt')):
raise FatalError(
"Directory '%s' doesn't seem to be a CMake build directory. Refusing to automatically "
"delete files in this directory. Delete the directory manually to 'clean' it." % build_dir)
red_flags = ["CMakeLists.txt", ".git", ".svn"]
red_flags = ['CMakeLists.txt', '.git', '.svn']
for red in red_flags:
red = os.path.join(build_dir, red)
if os.path.exists(red):
@@ -115,43 +115,43 @@ def action_extensions(base_actions, project_path):
# outside of this directory.
deleted_symlinks = _delete_windows_symlinks(build_dir)
if args.verbose and len(deleted_symlinks) > 1:
print("The following symlinks were identified and removed:\n%s" % "\n".join(deleted_symlinks))
print('The following symlinks were identified and removed:\n%s' % '\n'.join(deleted_symlinks))
for f in os.listdir(build_dir): # TODO: once we are Python 3 only, this can be os.scandir()
f = os.path.join(build_dir, f)
if args.verbose:
print("Removing: %s" % f)
print('Removing: %s' % f)
if os.path.isdir(f):
shutil.rmtree(f)
else:
os.remove(f)
def python_clean(action, ctx, args):
for root, dirnames, filenames in os.walk(os.environ["IDF_PATH"]):
for root, dirnames, filenames in os.walk(os.environ['IDF_PATH']):
for d in dirnames:
if d == "__pycache__":
if d == '__pycache__':
dir_to_delete = os.path.join(root, d)
if args.verbose:
print("Removing: %s" % dir_to_delete)
print('Removing: %s' % dir_to_delete)
shutil.rmtree(dir_to_delete)
for filename in fnmatch.filter(filenames, '*.py[co]'):
file_to_delete = os.path.join(root, filename)
if args.verbose:
print("Removing: %s" % file_to_delete)
print('Removing: %s' % file_to_delete)
os.remove(file_to_delete)
def set_target(action, ctx, args, idf_target):
if (not args["preview"] and idf_target in PREVIEW_TARGETS):
if (not args['preview'] and idf_target in PREVIEW_TARGETS):
raise FatalError(
"%s is still in preview. You have to append '--preview' option after idf.py to use any preview feature."
% idf_target)
args.define_cache_entry.append("IDF_TARGET=" + idf_target)
args.define_cache_entry.append('IDF_TARGET=' + idf_target)
sdkconfig_path = os.path.join(args.project_dir, 'sdkconfig')
sdkconfig_old = sdkconfig_path + ".old"
sdkconfig_old = sdkconfig_path + '.old'
if os.path.exists(sdkconfig_old):
os.remove(sdkconfig_old)
if os.path.exists(sdkconfig_path):
os.rename(sdkconfig_path, sdkconfig_old)
print("Set Target to: %s, new sdkconfig created. Existing sdkconfig renamed to sdkconfig.old." % idf_target)
print('Set Target to: %s, new sdkconfig created. Existing sdkconfig renamed to sdkconfig.old.' % idf_target)
ensure_build_directory(args, ctx.info_name, True)
def reconfigure(action, ctx, args):
@@ -161,10 +161,10 @@ def action_extensions(base_actions, project_path):
args.project_dir = realpath(args.project_dir)
if args.build_dir is not None and args.project_dir == realpath(args.build_dir):
raise FatalError(
"Setting the build directory to the project directory is not supported. Suggest dropping "
'Setting the build directory to the project directory is not supported. Suggest dropping '
"--build-dir option, the default is a 'build' subdirectory inside the project directory.")
if args.build_dir is None:
args.build_dir = os.path.join(args.project_dir, "build")
args.build_dir = os.path.join(args.project_dir, 'build')
args.build_dir = realpath(args.build_dir)
def idf_version_callback(ctx, param, value):
@@ -174,9 +174,9 @@ def action_extensions(base_actions, project_path):
version = idf_version()
if not version:
raise FatalError("ESP-IDF version cannot be determined")
raise FatalError('ESP-IDF version cannot be determined')
print("ESP-IDF %s" % version)
print('ESP-IDF %s' % version)
sys.exit(0)
def list_targets_callback(ctx, param, value):
@@ -186,272 +186,272 @@ def action_extensions(base_actions, project_path):
for target in SUPPORTED_TARGETS:
print(target)
if "preview" in ctx.params:
if 'preview' in ctx.params:
for target in PREVIEW_TARGETS:
print(target)
sys.exit(0)
root_options = {
"global_options": [
'global_options': [
{
"names": ["--version"],
"help": "Show IDF version and exit.",
"is_flag": True,
"expose_value": False,
"callback": idf_version_callback,
'names': ['--version'],
'help': 'Show IDF version and exit.',
'is_flag': True,
'expose_value': False,
'callback': idf_version_callback,
},
{
"names": ["--list-targets"],
"help": "Print list of supported targets and exit.",
"is_flag": True,
"expose_value": False,
"callback": list_targets_callback,
'names': ['--list-targets'],
'help': 'Print list of supported targets and exit.',
'is_flag': True,
'expose_value': False,
'callback': list_targets_callback,
},
{
"names": ["-C", "--project-dir"],
"scope": "shared",
"help": "Project directory.",
"type": click.Path(),
"default": os.getcwd(),
'names': ['-C', '--project-dir'],
'scope': 'shared',
'help': 'Project directory.',
'type': click.Path(),
'default': os.getcwd(),
},
{
"names": ["-B", "--build-dir"],
"help": "Build directory.",
"type": click.Path(),
"default": None,
'names': ['-B', '--build-dir'],
'help': 'Build directory.',
'type': click.Path(),
'default': None,
},
{
"names": ["-w/-n", "--cmake-warn-uninitialized/--no-warnings"],
"help": ("Enable CMake uninitialized variable warnings for CMake files inside the project directory. "
'names': ['-w/-n', '--cmake-warn-uninitialized/--no-warnings'],
'help': ('Enable CMake uninitialized variable warnings for CMake files inside the project directory. '
"(--no-warnings is now the default, and doesn't need to be specified.)"),
"envvar": "IDF_CMAKE_WARN_UNINITIALIZED",
"is_flag": True,
"default": False,
'envvar': 'IDF_CMAKE_WARN_UNINITIALIZED',
'is_flag': True,
'default': False,
},
{
"names": ["-v", "--verbose"],
"help": "Verbose build output.",
"is_flag": True,
"is_eager": True,
"default": False,
"callback": verbose_callback,
'names': ['-v', '--verbose'],
'help': 'Verbose build output.',
'is_flag': True,
'is_eager': True,
'default': False,
'callback': verbose_callback,
},
{
"names": ["--preview"],
"help": "Enable IDF features that are still in preview.",
"is_flag": True,
"default": False,
'names': ['--preview'],
'help': 'Enable IDF features that are still in preview.',
'is_flag': True,
'default': False,
},
{
"names": ["--ccache/--no-ccache"],
"help": "Use ccache in build. Disabled by default.",
"is_flag": True,
"envvar": "IDF_CCACHE_ENABLE",
"default": False,
'names': ['--ccache/--no-ccache'],
'help': 'Use ccache in build. Disabled by default.',
'is_flag': True,
'envvar': 'IDF_CCACHE_ENABLE',
'default': False,
},
{
"names": ["-G", "--generator"],
"help": "CMake generator.",
"type": click.Choice(GENERATORS.keys()),
'names': ['-G', '--generator'],
'help': 'CMake generator.',
'type': click.Choice(GENERATORS.keys()),
},
{
"names": ["--dry-run"],
"help": "Only process arguments, but don't execute actions.",
"is_flag": True,
"hidden": True,
"default": False,
'names': ['--dry-run'],
'help': "Only process arguments, but don't execute actions.",
'is_flag': True,
'hidden': True,
'default': False,
},
],
"global_action_callbacks": [validate_root_options],
'global_action_callbacks': [validate_root_options],
}
build_actions = {
"actions": {
"all": {
"aliases": ["build"],
"callback": build_target,
"short_help": "Build the project.",
"help": (
"Build the project. This can involve multiple steps:\n\n"
"1. Create the build directory if needed. "
'actions': {
'all': {
'aliases': ['build'],
'callback': build_target,
'short_help': 'Build the project.',
'help': (
'Build the project. This can involve multiple steps:\n\n'
'1. Create the build directory if needed. '
"The sub-directory 'build' is used to hold build output, "
"although this can be changed with the -B option.\n\n"
"2. Run CMake as necessary to configure the project "
"and generate build files for the main build tool.\n\n"
"3. Run the main build tool (Ninja or GNU Make). "
"By default, the build tool is automatically detected "
"but it can be explicitly set by passing the -G option to idf.py.\n\n"),
"options": global_options,
"order_dependencies": [
"reconfigure",
"menuconfig",
"clean",
"fullclean",
'although this can be changed with the -B option.\n\n'
'2. Run CMake as necessary to configure the project '
'and generate build files for the main build tool.\n\n'
'3. Run the main build tool (Ninja or GNU Make). '
'By default, the build tool is automatically detected '
'but it can be explicitly set by passing the -G option to idf.py.\n\n'),
'options': global_options,
'order_dependencies': [
'reconfigure',
'menuconfig',
'clean',
'fullclean',
],
},
"menuconfig": {
"callback": menuconfig,
"help": 'Run "menuconfig" project configuration tool.',
"options": global_options + [
'menuconfig': {
'callback': menuconfig,
'help': 'Run "menuconfig" project configuration tool.',
'options': global_options + [
{
"names": ["--style", "--color-scheme", "style"],
"help": (
"Menuconfig style.\n"
"The built-in styles include:\n\n"
"- default - a yellowish theme,\n\n"
"- monochrome - a black and white theme, or\n\n"
"- aquatic - a blue theme.\n\n"
"It is possible to customize these themes further"
" as it is described in the Color schemes section of the kconfiglib documentation.\n"
'names': ['--style', '--color-scheme', 'style'],
'help': (
'Menuconfig style.\n'
'The built-in styles include:\n\n'
'- default - a yellowish theme,\n\n'
'- monochrome - a black and white theme, or\n\n'
'- aquatic - a blue theme.\n\n'
'It is possible to customize these themes further'
' as it is described in the Color schemes section of the kconfiglib documentation.\n'
'The default value is \"aquatic\".'),
"envvar": "MENUCONFIG_STYLE",
"default": "aquatic",
'envvar': 'MENUCONFIG_STYLE',
'default': 'aquatic',
}
],
},
"confserver": {
"callback": build_target,
"help": "Run JSON configuration server.",
"options": global_options,
'confserver': {
'callback': build_target,
'help': 'Run JSON configuration server.',
'options': global_options,
},
"size": {
"callback": build_target,
"help": "Print basic size information about the app.",
"options": global_options,
"dependencies": ["app"],
'size': {
'callback': build_target,
'help': 'Print basic size information about the app.',
'options': global_options,
'dependencies': ['app'],
},
"size-components": {
"callback": build_target,
"help": "Print per-component size information.",
"options": global_options,
"dependencies": ["app"],
'size-components': {
'callback': build_target,
'help': 'Print per-component size information.',
'options': global_options,
'dependencies': ['app'],
},
"size-files": {
"callback": build_target,
"help": "Print per-source-file size information.",
"options": global_options,
"dependencies": ["app"],
'size-files': {
'callback': build_target,
'help': 'Print per-source-file size information.',
'options': global_options,
'dependencies': ['app'],
},
"bootloader": {
"callback": build_target,
"help": "Build only bootloader.",
"options": global_options,
'bootloader': {
'callback': build_target,
'help': 'Build only bootloader.',
'options': global_options,
},
"app": {
"callback": build_target,
"help": "Build only the app.",
"order_dependencies": ["clean", "fullclean", "reconfigure"],
"options": global_options,
'app': {
'callback': build_target,
'help': 'Build only the app.',
'order_dependencies': ['clean', 'fullclean', 'reconfigure'],
'options': global_options,
},
"efuse_common_table": {
"callback": build_target,
"help": "Generate C-source for IDF's eFuse fields.",
"order_dependencies": ["reconfigure"],
"options": global_options,
'efuse_common_table': {
'callback': build_target,
'help': "Generate C-source for IDF's eFuse fields.",
'order_dependencies': ['reconfigure'],
'options': global_options,
},
"efuse_custom_table": {
"callback": build_target,
"help": "Generate C-source for user's eFuse fields.",
"order_dependencies": ["reconfigure"],
"options": global_options,
'efuse_custom_table': {
'callback': build_target,
'help': "Generate C-source for user's eFuse fields.",
'order_dependencies': ['reconfigure'],
'options': global_options,
},
"show_efuse_table": {
"callback": build_target,
"help": "Print eFuse table.",
"order_dependencies": ["reconfigure"],
"options": global_options,
'show_efuse_table': {
'callback': build_target,
'help': 'Print eFuse table.',
'order_dependencies': ['reconfigure'],
'options': global_options,
},
"partition_table": {
"callback": build_target,
"help": "Build only partition table.",
"order_dependencies": ["reconfigure"],
"options": global_options,
'partition_table': {
'callback': build_target,
'help': 'Build only partition table.',
'order_dependencies': ['reconfigure'],
'options': global_options,
},
"erase_otadata": {
"callback": build_target,
"help": "Erase otadata partition.",
"options": global_options,
'erase_otadata': {
'callback': build_target,
'help': 'Erase otadata partition.',
'options': global_options,
},
"read_otadata": {
"callback": build_target,
"help": "Read otadata partition.",
"options": global_options,
'read_otadata': {
'callback': build_target,
'help': 'Read otadata partition.',
'options': global_options,
},
"build-system-targets": {
"callback": list_build_system_targets,
"help": "Print list of build system targets.",
'build-system-targets': {
'callback': list_build_system_targets,
'help': 'Print list of build system targets.',
},
"fallback": {
"callback": fallback_target,
"help": "Handle for targets not known for idf.py.",
"hidden": True,
'fallback': {
'callback': fallback_target,
'help': 'Handle for targets not known for idf.py.',
'hidden': True,
}
}
}
clean_actions = {
"actions": {
"reconfigure": {
"callback": reconfigure,
"short_help": "Re-run CMake.",
"help": (
'actions': {
'reconfigure': {
'callback': reconfigure,
'short_help': 'Re-run CMake.',
'help': (
"Re-run CMake even if it doesn't seem to need re-running. "
"This isn't necessary during normal usage, "
"but can be useful after adding/removing files from the source tree, "
"or when modifying CMake cache variables. "
'but can be useful after adding/removing files from the source tree, '
'or when modifying CMake cache variables. '
"For example, \"idf.py -DNAME='VALUE' reconfigure\" "
'can be used to set variable "NAME" in CMake cache to value "VALUE".'),
"options": global_options,
"order_dependencies": ["menuconfig", "fullclean"],
'options': global_options,
'order_dependencies': ['menuconfig', 'fullclean'],
},
"set-target": {
"callback": set_target,
"short_help": "Set the chip target to build.",
"help": (
"Set the chip target to build. This will remove the "
"existing sdkconfig file and corresponding CMakeCache and "
"create new ones according to the new target.\nFor example, "
'set-target': {
'callback': set_target,
'short_help': 'Set the chip target to build.',
'help': (
'Set the chip target to build. This will remove the '
'existing sdkconfig file and corresponding CMakeCache and '
'create new ones according to the new target.\nFor example, '
"\"idf.py set-target esp32\" will select esp32 as the new chip "
"target."),
"arguments": [
'target.'),
'arguments': [
{
"names": ["idf-target"],
"nargs": 1,
"type": TargetChoice(SUPPORTED_TARGETS + PREVIEW_TARGETS),
'names': ['idf-target'],
'nargs': 1,
'type': TargetChoice(SUPPORTED_TARGETS + PREVIEW_TARGETS),
},
],
"dependencies": ["fullclean"],
'dependencies': ['fullclean'],
},
"clean": {
"callback": clean,
"short_help": "Delete build output files from the build directory.",
"help": (
"Delete build output files from the build directory, "
'clean': {
'callback': clean,
'short_help': 'Delete build output files from the build directory.',
'help': (
'Delete build output files from the build directory, '
"forcing a 'full rebuild' the next time "
"the project is built. Cleaning doesn't delete "
"CMake configuration output and some other files"),
"order_dependencies": ["fullclean"],
'CMake configuration output and some other files'),
'order_dependencies': ['fullclean'],
},
"fullclean": {
"callback": fullclean,
"short_help": "Delete the entire build directory contents.",
"help": (
"Delete the entire build directory contents. "
"This includes all CMake configuration output."
"The next time the project is built, "
"CMake will configure it from scratch. "
"Note that this option recursively deletes all files "
"in the build directory, so use with care."
"Project configuration is not deleted.")
'fullclean': {
'callback': fullclean,
'short_help': 'Delete the entire build directory contents.',
'help': (
'Delete the entire build directory contents. '
'This includes all CMake configuration output.'
'The next time the project is built, '
'CMake will configure it from scratch. '
'Note that this option recursively deletes all files '
'in the build directory, so use with care.'
'Project configuration is not deleted.')
},
"python-clean": {
"callback": python_clean,
"short_help": "Delete generated Python byte code from the IDF directory",
"help": (
"Delete generated Python byte code from the IDF directory "
"which may cause issues when switching between IDF and Python versions. "
"It is advised to run this target after switching versions.")
'python-clean': {
'callback': python_clean,
'short_help': 'Delete generated Python byte code from the IDF directory',
'help': (
'Delete generated Python byte code from the IDF directory '
'which may cause issues when switching between IDF and Python versions. '
'It is advised to run this target after switching versions.')
},
}
}

116
tools/idf_py_actions/create_ext.py
View File

@@ -1,13 +1,13 @@
from __future__ import print_function
from distutils.dir_util import copy_tree
import os
import re
import sys
from distutils.dir_util import copy_tree
def get_type(action):
return action.split("-")[1]
return action.split('-')[1]
def replace_in_file(filename, pattern, replacement):
@@ -24,37 +24,37 @@ def is_empty_and_create(path, action):
if not os.path.exists(abspath):
os.makedirs(abspath)
elif not os.path.isdir(abspath):
print("Your target path is not a directory. Please remove the", os.path.abspath(abspath),
"or use different target path.")
print('Your target path is not a directory. Please remove the', os.path.abspath(abspath),
'or use different target path.')
sys.exit(4)
elif len(os.listdir(path)) > 0:
print("The directory", abspath, "is not empty. To create a", get_type(action),
"you must empty the directory or choose a different path.")
print('The directory', abspath, 'is not empty. To create a', get_type(action),
'you must empty the directory or choose a different path.')
sys.exit(3)
def create_project(target_path, name):
copy_tree(os.path.join(os.environ['IDF_PATH'], "examples", "get-started", "sample_project"), target_path)
main_folder = os.path.join(target_path, "main")
os.rename(os.path.join(main_folder, "main.c"), os.path.join(main_folder, ".".join((name, "c"))))
replace_in_file(os.path.join(main_folder, "CMakeLists.txt"), "main", name)
replace_in_file(os.path.join(target_path, "CMakeLists.txt"), "main", name)
os.remove(os.path.join(target_path, "README.md"))
copy_tree(os.path.join(os.environ['IDF_PATH'], 'examples', 'get-started', 'sample_project'), target_path)
main_folder = os.path.join(target_path, 'main')
os.rename(os.path.join(main_folder, 'main.c'), os.path.join(main_folder, '.'.join((name, 'c'))))
replace_in_file(os.path.join(main_folder, 'CMakeLists.txt'), 'main', name)
replace_in_file(os.path.join(target_path, 'CMakeLists.txt'), 'main', name)
os.remove(os.path.join(target_path, 'README.md'))
# after manual removing "Makefile" and "component.mk" from `examples/get-started/sample_project`
# remove following two lines as well
os.remove(os.path.join(target_path, "Makefile"))
os.remove(os.path.join(target_path, "main", "component.mk"))
os.remove(os.path.join(target_path, 'Makefile'))
os.remove(os.path.join(target_path, 'main', 'component.mk'))
def create_component(target_path, name):
copy_tree(os.path.join(os.environ['IDF_PATH'], "tools", "templates", "sample_component"), target_path)
os.rename(os.path.join(target_path, "main.c"), os.path.join(target_path, ".".join((name, "c"))))
os.rename(os.path.join(target_path, "include", "main.h"),
os.path.join(target_path, "include", ".".join((name, "h"))))
copy_tree(os.path.join(os.environ['IDF_PATH'], 'tools', 'templates', 'sample_component'), target_path)
os.rename(os.path.join(target_path, 'main.c'), os.path.join(target_path, '.'.join((name, 'c'))))
os.rename(os.path.join(target_path, 'include', 'main.h'),
os.path.join(target_path, 'include', '.'.join((name, 'h'))))
replace_in_file(os.path.join(target_path, ".".join((name, "c"))), "main", name)
replace_in_file(os.path.join(target_path, "CMakeLists.txt"), "main", name)
replace_in_file(os.path.join(target_path, '.'.join((name, 'c'))), 'main', name)
replace_in_file(os.path.join(target_path, 'CMakeLists.txt'), 'main', name)
def action_extensions(base_actions, project_path):
@@ -63,58 +63,58 @@ def action_extensions(base_actions, project_path):
is_empty_and_create(target_path, action)
func_action_map = {"create-project": create_project, "create-component": create_component}
func_action_map = {'create-project': create_project, 'create-component': create_component}
func_action_map[action](target_path, action_args['name'])
print("The", get_type(action), "was created in", os.path.abspath(target_path))
print('The', get_type(action), 'was created in', os.path.abspath(target_path))
# after the command execution, no other commands are accepted and idf.py terminates
sys.exit(0)
return {
"actions": {
"create-project": {
"callback": create_new,
"short_help": "Create a new project.",
"help": ("Create a new project with the name NAME specified as argument. "
"For example: "
"`idf.py create-project new_proj` "
"will create a new project in subdirectory called `new_proj` "
"of the current working directory. "
'actions': {
'create-project': {
'callback': create_new,
'short_help': 'Create a new project.',
'help': ('Create a new project with the name NAME specified as argument. '
'For example: '
'`idf.py create-project new_proj` '
'will create a new project in subdirectory called `new_proj` '
'of the current working directory. '
"For specifying the new project's path, use either the option --path for specifying the "
"destination directory, or the global option -C if the project should be created as a "
"subdirectory of the specified directory. "
"If the target path does not exist it will be created. If the target folder is not empty "
"then the operation will fail with return code 3. "
"If the target path is not a folder, the script will fail with return code 4. "
"After the execution idf.py terminates "
"so this operation should be used alone."),
"arguments": [{"names": ["name"]}],
"options": [
'destination directory, or the global option -C if the project should be created as a '
'subdirectory of the specified directory. '
'If the target path does not exist it will be created. If the target folder is not empty '
'then the operation will fail with return code 3. '
'If the target path is not a folder, the script will fail with return code 4. '
'After the execution idf.py terminates '
'so this operation should be used alone.'),
'arguments': [{'names': ['name']}],
'options': [
{
"names": ["-p", "--path"],
"help": ("Set the path for the new project. The project "
"will be created directly in the given folder if it does not contain anything"),
'names': ['-p', '--path'],
'help': ('Set the path for the new project. The project '
'will be created directly in the given folder if it does not contain anything'),
},
],
},
"create-component": {
"callback": create_new,
"short_help": "Create a new component.",
"help": ("Create a new component with the name NAME specified as argument. "
"For example: "
"`idf.py create-component new_comp` "
"will create a new component in subdirectory called `new_comp` "
"of the current working directory. "
'create-component': {
'callback': create_new,
'short_help': 'Create a new component.',
'help': ('Create a new component with the name NAME specified as argument. '
'For example: '
'`idf.py create-component new_comp` '
'will create a new component in subdirectory called `new_comp` '
'of the current working directory. '
"For specifying the new component's path use the option -C. "
"If the target path does not exist then it will be created. "
"If the target folder is not empty "
"then the operation will fail with return code 3. "
"If the target path is not a folder, the script will fail with return code 4. "
"After the execution idf.py terminates "
"so this operation should be used alone."),
"arguments": [{"names": ["name"]}],
'If the target path does not exist then it will be created. '
'If the target folder is not empty '
'then the operation will fail with return code 3. '
'If the target path is not a folder, the script will fail with return code 4. '
'After the execution idf.py terminates '
'so this operation should be used alone.'),
'arguments': [{'names': ['name']}],
}
}
}

308
tools/idf_py_actions/debug_ext.py
View File

@@ -1,12 +1,12 @@
import json
import os
import sys
import subprocess
import shlex
import time
import re
from threading import Thread
import shlex
import subprocess
import sys
import threading
import time
from threading import Thread
from idf_py_actions.errors import FatalError
from idf_py_actions.tools import ensure_build_directory
@@ -15,42 +15,42 @@ PYTHON = sys.executable
def action_extensions(base_actions, project_path):
OPENOCD_OUT_FILE = "openocd_out.txt"
GDBGUI_OUT_FILE = "gdbgui_out.txt"
OPENOCD_OUT_FILE = 'openocd_out.txt'
GDBGUI_OUT_FILE = 'gdbgui_out.txt'
# Internal dictionary of currently active processes, threads and their output files
processes = {"threads_to_join": [], "openocd_issues": None}
processes = {'threads_to_join': [], 'openocd_issues': None}
def _check_for_common_openocd_issues(file_name, print_all=True):
if processes["openocd_issues"] is not None:
return processes["openocd_issues"]
if processes['openocd_issues'] is not None:
return processes['openocd_issues']
try:
message = "Please check JTAG connection!"
with open(file_name, "r") as f:
message = 'Please check JTAG connection!'
with open(file_name, 'r') as f:
content = f.read()
if print_all:
print(content)
if re.search(r"Address already in use", content):
message = ("Please check if another process uses the mentioned ports. OpenOCD already running, perhaps in the background?\n"
"Please list all processes to check if OpenOCD is already running; if so, terminate it before starting OpenOCD from idf.py")
if re.search(r'Address already in use', content):
message = ('Please check if another process uses the mentioned ports. OpenOCD already running, perhaps in the background?\n'
'Please list all processes to check if OpenOCD is already running; if so, terminate it before starting OpenOCD from idf.py')
finally:
processes["openocd_issues"] = message
processes['openocd_issues'] = message
return message
def _check_openocd_errors(fail_if_openocd_failed, target, ctx):
if fail_if_openocd_failed:
if "openocd" in processes and processes["openocd"] is not None:
p = processes["openocd"]
name = processes["openocd_outfile_name"]
if 'openocd' in processes and processes['openocd'] is not None:
p = processes['openocd']
name = processes['openocd_outfile_name']
# watch OpenOCD (for 5x500ms) to check if it hasn't terminated or outputs an error
for _ in range(5):
if p.poll() is not None:
print("OpenOCD exited with {}".format(p.poll()))
print('OpenOCD exited with {}'.format(p.poll()))
break
with open(name, "r") as f:
with open(name, 'r') as f:
content = f.read()
if re.search(r"no device found", content):
if re.search(r'no device found', content):
break
if re.search(r"Listening on port \d+ for gdb connections", content):
if re.search(r'Listening on port \d+ for gdb connections', content):
# expect OpenOCD has started successfully - stop watching
return
time.sleep(0.5)
@@ -62,8 +62,8 @@ def action_extensions(base_actions, project_path):
def _terminate_async_target(target):
if target in processes and processes[target] is not None:
try:
if target + "_outfile" in processes:
processes[target + "_outfile"].close()
if target + '_outfile' in processes:
processes[target + '_outfile'].close()
p = processes[target]
if p.poll() is None:
p.terminate()
@@ -74,13 +74,13 @@ def action_extensions(base_actions, project_path):
time.sleep(0.1)
else:
p.kill()
if target + "_outfile_name" in processes:
if target == "openocd":
print(_check_for_common_openocd_issues(processes[target + "_outfile_name"], print_all=False))
os.unlink(processes[target + "_outfile_name"])
if target + '_outfile_name' in processes:
if target == 'openocd':
print(_check_for_common_openocd_issues(processes[target + '_outfile_name'], print_all=False))
os.unlink(processes[target + '_outfile_name'])
except Exception as e:
print(e)
print("Failed to close/kill {}".format(target))
print('Failed to close/kill {}'.format(target))
processes[target] = None # to indicate this has ended
def _get_commandline_options(ctx):
@@ -97,39 +97,39 @@ def action_extensions(base_actions, project_path):
return result
def create_local_gdbinit(gdbinit, elf_file):
with open(gdbinit, "w") as f:
f.write("target remote :3333\n")
if os.name == "nt":
with open(gdbinit, 'w') as f:
f.write('target remote :3333\n')
if os.name == 'nt':
elf_file = elf_file.replace('\\','\\\\')
f.write("symbol-file {}\n".format(elf_file))
f.write("mon reset halt\n")
f.write("flushregs\n")
f.write("thb app_main\n")
f.write("c\n")
f.write('symbol-file {}\n'.format(elf_file))
f.write('mon reset halt\n')
f.write('flushregs\n')
f.write('thb app_main\n')
f.write('c\n')
def debug_cleanup():
print("cleaning up debug targets")
for t in processes["threads_to_join"]:
print('cleaning up debug targets')
for t in processes['threads_to_join']:
if threading.currentThread() != t:
t.join()
_terminate_async_target("openocd")
_terminate_async_target("gdbgui")
_terminate_async_target("gdb")
_terminate_async_target('openocd')
_terminate_async_target('gdbgui')
_terminate_async_target('gdb')
def post_debug(action, ctx, args, **kwargs):
""" Deal with asynchronous targets, such as openocd running in background """
if kwargs["block"] == 1:
for target in ["openocd", "gdbgui"]:
if kwargs['block'] == 1:
for target in ['openocd', 'gdbgui']:
if target in processes and processes[target] is not None:
break
else:
return
try:
p = processes[target]
name = processes[target + "_outfile_name"]
name = processes[target + '_outfile_name']
pos = 0
while True:
with open(name, "r") as f:
with open(name, 'r') as f:
f.seek(pos)
for line in f:
print(line.rstrip())
@@ -139,15 +139,15 @@ def action_extensions(base_actions, project_path):
break
time.sleep(0.5)
except KeyboardInterrupt:
print("Terminated -> exiting debug utility targets")
_terminate_async_target("openocd")
_terminate_async_target("gdbgui")
print('Terminated -> exiting debug utility targets')
_terminate_async_target('openocd')
_terminate_async_target('gdbgui')
def get_project_desc(args, ctx):
desc_path = os.path.join(args.build_dir, "project_description.json")
desc_path = os.path.join(args.build_dir, 'project_description.json')
if not os.path.exists(desc_path):
ensure_build_directory(args, ctx.info_name)
with open(desc_path, "r") as f:
with open(desc_path, 'r') as f:
project_desc = json.load(f)
return project_desc
@@ -156,63 +156,63 @@ def action_extensions(base_actions, project_path):
Execute openocd as external tool
"""
OPENOCD_TAGET_CONFIG = {
"esp32": "-f board/esp32-wrover-kit-3.3v.cfg",
"esp32s2": "-f board/esp32s2-kaluga-1.cfg",
'esp32': '-f board/esp32-wrover-kit-3.3v.cfg',
'esp32s2': '-f board/esp32s2-kaluga-1.cfg',
}
if os.getenv("OPENOCD_SCRIPTS") is None:
raise FatalError("OPENOCD_SCRIPTS not found in the environment: Please run export.sh/export.bat", ctx)
openocd_arguments = os.getenv("OPENOCD_COMMANDS") if openocd_commands is None else openocd_commands
if os.getenv('OPENOCD_SCRIPTS') is None:
raise FatalError('OPENOCD_SCRIPTS not found in the environment: Please run export.sh/export.bat', ctx)
openocd_arguments = os.getenv('OPENOCD_COMMANDS') if openocd_commands is None else openocd_commands
project_desc = get_project_desc(args, ctx)
if openocd_arguments is None:
# use default value if commands not defined in the environment nor command line
target = project_desc["target"]
default_args = "-f interface/ftdi/esp32_devkitj_v1.cfg -f target/{}.cfg".format(target)
target = project_desc['target']
default_args = '-f interface/ftdi/esp32_devkitj_v1.cfg -f target/{}.cfg'.format(target)
openocd_arguments = OPENOCD_TAGET_CONFIG.get(target, default_args)
print('Note: OpenOCD cfg not found (via env variable OPENOCD_COMMANDS nor as a --openocd-commands argument)\n'
'OpenOCD arguments default to: "{}"'.format(openocd_arguments))
# script directory is taken from the environment by OpenOCD, update only if command line arguments to override
if openocd_scripts is not None:
openocd_arguments += " -s {}".format(openocd_scripts)
local_dir = project_desc["build_dir"]
args = ["openocd"] + shlex.split(openocd_arguments)
openocd_arguments += ' -s {}'.format(openocd_scripts)
local_dir = project_desc['build_dir']
args = ['openocd'] + shlex.split(openocd_arguments)
openocd_out_name = os.path.join(local_dir, OPENOCD_OUT_FILE)
openocd_out = open(openocd_out_name, "a+")
openocd_out = open(openocd_out_name, 'a+')
try:
process = subprocess.Popen(args, stdout=openocd_out, stderr=subprocess.STDOUT, bufsize=1)
except Exception as e:
print(e)
raise FatalError("Error starting openocd. Please make sure it is installed and is present in executable paths", ctx)
raise FatalError('Error starting openocd. Please make sure it is installed and is present in executable paths', ctx)
processes["openocd"] = process
processes["openocd_outfile"] = openocd_out
processes["openocd_outfile_name"] = openocd_out_name
print("OpenOCD started as a background task {}".format(process.pid))
processes['openocd'] = process
processes['openocd_outfile'] = openocd_out
processes['openocd_outfile_name'] = openocd_out_name
print('OpenOCD started as a background task {}'.format(process.pid))
def gdbui(action, ctx, args, gdbgui_port, gdbinit, require_openocd):
"""
Asynchronous GDB-UI target
"""
project_desc = get_project_desc(args, ctx)
local_dir = project_desc["build_dir"]
gdb = project_desc["monitor_toolprefix"] + "gdb"
local_dir = project_desc['build_dir']
gdb = project_desc['monitor_toolprefix'] + 'gdb'
if gdbinit is None:
gdbinit = os.path.join(local_dir, 'gdbinit')
create_local_gdbinit(gdbinit, os.path.join(args.build_dir, project_desc["app_elf"]))
args = ["gdbgui", "-g", gdb, '--gdb-args="-x={}"'.format(gdbinit)]
create_local_gdbinit(gdbinit, os.path.join(args.build_dir, project_desc['app_elf']))
args = ['gdbgui', '-g', gdb, '--gdb-args="-x={}"'.format(gdbinit)]
if gdbgui_port is not None:
args += ["--port", gdbgui_port]
args += ['--port', gdbgui_port]
gdbgui_out_name = os.path.join(local_dir, GDBGUI_OUT_FILE)
gdbgui_out = open(gdbgui_out_name, "a+")
gdbgui_out = open(gdbgui_out_name, 'a+')
try:
process = subprocess.Popen(args, stdout=gdbgui_out, stderr=subprocess.STDOUT, bufsize=1)
except Exception as e:
print(e)
raise FatalError("Error starting gdbgui. Please make sure gdbgui can be started", ctx)
raise FatalError('Error starting gdbgui. Please make sure gdbgui can be started', ctx)
processes["gdbgui"] = process
processes["gdbgui_outfile"] = gdbgui_out
processes["gdbgui_outfile_name"] = gdbgui_out_name
print("gdbgui started as a background task {}".format(process.pid))
processes['gdbgui'] = process
processes['gdbgui_outfile'] = gdbgui_out
processes['gdbgui_outfile_name'] = gdbgui_out_name
print('gdbgui started as a background task {}'.format(process.pid))
_check_openocd_errors(fail_if_openocd_failed, action, ctx)
def global_callback(ctx, global_args, tasks):
@@ -222,28 +222,28 @@ def action_extensions(base_actions, project_path):
tasks.insert(0, tasks.pop(index))
break
debug_targets = any([task.name in ("openocd", "gdbgui") for task in tasks])
debug_targets = any([task.name in ('openocd', 'gdbgui') for task in tasks])
if debug_targets:
# Register the meta cleanup callback -> called on FatalError
ctx.meta["cleanup"] = debug_cleanup
move_to_front("gdbgui") # possibly 2nd
move_to_front("openocd") # always 1st
ctx.meta['cleanup'] = debug_cleanup
move_to_front('gdbgui') # possibly 2nd
move_to_front('openocd') # always 1st
# followed by "monitor", "gdb" or "gdbtui" in any order
post_action = ctx.invoke(ctx.command.get_command(ctx, "post_debug"))
if any([task.name in ("monitor", "gdb", "gdbtui") for task in tasks]):
post_action.action_args["block"] = 0
post_action = ctx.invoke(ctx.command.get_command(ctx, 'post_debug'))
if any([task.name in ('monitor', 'gdb', 'gdbtui') for task in tasks]):
post_action.action_args['block'] = 0
else:
post_action.action_args["block"] = 1
post_action.action_args['block'] = 1
tasks.append(post_action) # always last
if any([task.name == "openocd" for task in tasks]):
if any([task.name == 'openocd' for task in tasks]):
for task in tasks:
if task.name in ("gdb", "gdbgui", "gdbtui"):
task.action_args["require_openocd"] = True
if task.name in ('gdb', 'gdbgui', 'gdbtui'):
task.action_args['require_openocd'] = True
def run_gdb(gdb_args):
p = subprocess.Popen(gdb_args)
processes["gdb"] = p
processes['gdb'] = p
return p.wait()
def gdbtui(action, ctx, args, gdbinit, require_openocd):
@@ -258,18 +258,18 @@ def action_extensions(base_actions, project_path):
"""
watch_openocd = Thread(target=_check_openocd_errors, args=(fail_if_openocd_failed, action, ctx, ))
watch_openocd.start()
processes["threads_to_join"].append(watch_openocd)
desc_path = os.path.join(args.build_dir, "project_description.json")
processes['threads_to_join'].append(watch_openocd)
desc_path = os.path.join(args.build_dir, 'project_description.json')
if not os.path.exists(desc_path):
ensure_build_directory(args, ctx.info_name)
with open(desc_path, "r") as f:
with open(desc_path, 'r') as f:
project_desc = json.load(f)
elf_file = os.path.join(args.build_dir, project_desc["app_elf"])
elf_file = os.path.join(args.build_dir, project_desc['app_elf'])
if not os.path.exists(elf_file):
raise FatalError("ELF file not found. You need to build & flash the project before running debug targets", ctx)
gdb = project_desc["monitor_toolprefix"] + "gdb"
local_dir = project_desc["build_dir"]
raise FatalError('ELF file not found. You need to build & flash the project before running debug targets', ctx)
gdb = project_desc['monitor_toolprefix'] + 'gdb'
local_dir = project_desc['build_dir']
if gdbinit is None:
gdbinit = os.path.join(local_dir, 'gdbinit')
create_local_gdbinit(gdbinit, elf_file)
@@ -288,92 +288,92 @@ def action_extensions(base_actions, project_path):
finally:
watch_openocd.join()
try:
processes["threads_to_join"].remove(watch_openocd)
processes['threads_to_join'].remove(watch_openocd)
except ValueError:
# Valid scenario: watch_openocd task won't be in the list if openocd not started from idf.py
pass
fail_if_openocd_failed = {
"names": ["--require-openocd", "--require_openocd"],
"help":
("Fail this target if openocd (this targets dependency) failed.\n"),
"is_flag": True,
"default": False,
'names': ['--require-openocd', '--require_openocd'],
'help':
('Fail this target if openocd (this targets dependency) failed.\n'),
'is_flag': True,
'default': False,
}
gdbinit = {
"names": ["--gdbinit"],
"help": ("Specify the name of gdbinit file to use\n"),
"default": None,
'names': ['--gdbinit'],
'help': ('Specify the name of gdbinit file to use\n'),
'default': None,
}
debug_actions = {
"global_action_callbacks": [global_callback],
"actions": {
"openocd": {
"callback": openocd,
"help": "Run openocd from current path",
"options": [
'global_action_callbacks': [global_callback],
'actions': {
'openocd': {
'callback': openocd,
'help': 'Run openocd from current path',
'options': [
{
"names": ["--openocd-scripts", "--openocd_scripts"],
"help":
("Script directory for openocd cfg files.\n"),
"default":
'names': ['--openocd-scripts', '--openocd_scripts'],
'help':
('Script directory for openocd cfg files.\n'),
'default':
None,
},
{
"names": ["--openocd-commands", "--openocd_commands"],
"help":
("Command line arguments for openocd.\n"),
"default": None,
'names': ['--openocd-commands', '--openocd_commands'],
'help':
('Command line arguments for openocd.\n'),
'default': None,
}
],
"order_dependencies": ["all", "flash"],
'order_dependencies': ['all', 'flash'],
},
"gdb": {
"callback": gdb,
"help": "Run the GDB.",
"options": [
'gdb': {
'callback': gdb,
'help': 'Run the GDB.',
'options': [
{
"names": ["--gdb-tui", "--gdb_tui"],
"help":
("run gdb in TUI mode\n"),
"default":
'names': ['--gdb-tui', '--gdb_tui'],
'help':
('run gdb in TUI mode\n'),
'default':
None,
}, gdbinit, fail_if_openocd_failed
],
"order_dependencies": ["all", "flash"],
'order_dependencies': ['all', 'flash'],
},
"gdbgui": {
"callback": gdbui,
"help": "GDB UI in default browser.",
"options": [
'gdbgui': {
'callback': gdbui,
'help': 'GDB UI in default browser.',
'options': [
{
"names": ["--gdbgui-port", "--gdbgui_port"],
"help":
("The port on which gdbgui will be hosted. Default: 5000\n"),
"default":
'names': ['--gdbgui-port', '--gdbgui_port'],
'help':
('The port on which gdbgui will be hosted. Default: 5000\n'),
'default':
None,
}, gdbinit, fail_if_openocd_failed
],
"order_dependencies": ["all", "flash"],
'order_dependencies': ['all', 'flash'],
},
"gdbtui": {
"callback": gdbtui,
"help": "GDB TUI mode.",
"options": [gdbinit, fail_if_openocd_failed],
"order_dependencies": ["all", "flash"],
'gdbtui': {
'callback': gdbtui,
'help': 'GDB TUI mode.',
'options': [gdbinit, fail_if_openocd_failed],
'order_dependencies': ['all', 'flash'],
},
"post_debug": {
"callback": post_debug,
"help": "Utility target to read the output of async debug action and stop them.",
"options": [
'post_debug': {
'callback': post_debug,
'help': 'Utility target to read the output of async debug action and stop them.',
'options': [
{
"names": ["--block", "--block"],
"help":
("Set to 1 for blocking the console on the outputs of async debug actions\n"),
"default": 0,
'names': ['--block', '--block'],
'help':
('Set to 1 for blocking the console on the outputs of async debug actions\n'),
'default': 0,
},
],
"order_dependencies": [],
'order_dependencies': [],
},
},
}

42
tools/idf_py_actions/dfu_ext.py
View File

@@ -1,5 +1,5 @@
from idf_py_actions.tools import is_target_supported, ensure_build_directory, run_target
from idf_py_actions.errors import FatalError
from idf_py_actions.tools import ensure_build_directory, is_target_supported, run_target
def action_extensions(base_actions, project_path):
@@ -14,7 +14,7 @@ def action_extensions(base_actions, project_path):
ensure_build_directory(args, ctx.info_name)
try:
run_target(target_name, args, {"ESP_DFU_PATH": path})
run_target(target_name, args, {'ESP_DFU_PATH': path})
except FatalError:
# Cannot capture the error from dfu-util here so the best advise is:
print('Please have a look at the "Device Firmware Upgrade through USB" chapter in API Guides of the '
@@ -22,28 +22,28 @@ def action_extensions(base_actions, project_path):
raise
dfu_actions = {
"actions": {
"dfu": {
"callback": dfu_target,
"short_help": "Build the DFU binary",
"dependencies": ["all"],
'actions': {
'dfu': {
'callback': dfu_target,
'short_help': 'Build the DFU binary',
'dependencies': ['all'],
},
"dfu-list": {
"callback": dfu_target,
"short_help": "List DFU capable devices",
"dependencies": [],
'dfu-list': {
'callback': dfu_target,
'short_help': 'List DFU capable devices',
'dependencies': [],
},
"dfu-flash": {
"callback": dfu_flash_target,
"short_help": "Flash the DFU binary",
"order_dependencies": ["dfu"],
"options": [
'dfu-flash': {
'callback': dfu_flash_target,
'short_help': 'Flash the DFU binary',
'order_dependencies': ['dfu'],
'options': [
{
"names": ["--path"],
"default": "",
"help": "Specify path to DFU device. The default empty path works if there is just one "
"ESP device with the same product identificator. See the device list for paths "
"of available devices."
'names': ['--path'],
'default': '',
'help': 'Specify path to DFU device. The default empty path works if there is just one '
'ESP device with the same product identificator. See the device list for paths '
'of available devices.'
}
],
},

4
tools/idf_py_actions/errors.py
View File

@@ -5,6 +5,6 @@ class FatalError(RuntimeError):
def __init__(self, message, ctx=None):
super(RuntimeError, self).__init__(message)
# if context is defined, check for the cleanup tasks
if ctx is not None and "cleanup" in ctx.meta:
if ctx is not None and 'cleanup' in ctx.meta:
# cleans up the environment before failure
ctx.meta["cleanup"]()
ctx.meta['cleanup']()

8
tools/idf_py_actions/global_options.py
View File

@@ -1,6 +1,6 @@
global_options = [{
"names": ["-D", "--define-cache-entry"],
"help": "Create a cmake cache entry.",
"scope": "global",
"multiple": True,
'names': ['-D', '--define-cache-entry'],
'help': 'Create a cmake cache entry.',
'scope': 'global',
'multiple': True,
}]

223
tools/idf_py_actions/serial_ext.py
View File

@@ -3,10 +3,9 @@ import os
import sys
import click
from idf_py_actions.errors import FatalError
from idf_py_actions.global_options import global_options
from idf_py_actions.tools import ensure_build_directory, run_tool, run_target, get_sdkconfig_value
from idf_py_actions.tools import ensure_build_directory, get_sdkconfig_value, run_target, run_tool
PYTHON = sys.executable
@@ -16,7 +15,7 @@ def action_extensions(base_actions, project_path):
# Import is done here in order to move it after the check_environment() ensured that pyserial has been installed
try:
import serial.tools.list_ports
esptool_path = os.path.join(os.environ["IDF_PATH"], "components/esptool_py/esptool/")
esptool_path = os.path.join(os.environ['IDF_PATH'], 'components/esptool_py/esptool/')
sys.path.insert(0, esptool_path)
import esptool
ports = list(sorted(p.device for p in serial.tools.list_ports.comports()))
@@ -28,22 +27,22 @@ def action_extensions(base_actions, project_path):
raise FatalError("No serial ports found. Connect a device, or use '-p PORT' option to set a specific port.")
def _get_esptool_args(args):
esptool_path = os.path.join(os.environ["IDF_PATH"], "components/esptool_py/esptool/esptool.py")
esptool_path = os.path.join(os.environ['IDF_PATH'], 'components/esptool_py/esptool/esptool.py')
if args.port is None:
args.port = _get_default_serial_port(args)
result = [PYTHON, esptool_path]
result += ["-p", args.port]
result += ["-b", str(args.baud)]
result += ['-p', args.port]
result += ['-b', str(args.baud)]
with open(os.path.join(args.build_dir, "flasher_args.json")) as f:
with open(os.path.join(args.build_dir, 'flasher_args.json')) as f:
flasher_args = json.load(f)
extra_esptool_args = flasher_args["extra_esptool_args"]
result += ["--before", extra_esptool_args["before"]]
result += ["--after", extra_esptool_args["after"]]
result += ["--chip", extra_esptool_args["chip"]]
if not extra_esptool_args["stub"]:
result += ["--no-stub"]
extra_esptool_args = flasher_args['extra_esptool_args']
result += ['--before', extra_esptool_args['before']]
result += ['--after', extra_esptool_args['after']]
result += ['--chip', extra_esptool_args['chip']]
if not extra_esptool_args['stub']:
result += ['--no-stub']
return result
def _get_commandline_options(ctx):
@@ -64,49 +63,49 @@ def action_extensions(base_actions, project_path):
Run idf_monitor.py to watch build output
"""
desc_path = os.path.join(args.build_dir, "project_description.json")
desc_path = os.path.join(args.build_dir, 'project_description.json')
if not os.path.exists(desc_path):
ensure_build_directory(args, ctx.info_name)
with open(desc_path, "r") as f:
with open(desc_path, 'r') as f:
project_desc = json.load(f)
elf_file = os.path.join(args.build_dir, project_desc["app_elf"])
elf_file = os.path.join(args.build_dir, project_desc['app_elf'])
if not os.path.exists(elf_file):
raise FatalError("ELF file '%s' not found. You need to build & flash the project before running 'monitor', "
"and the binary on the device must match the one in the build directory exactly. "
'and the binary on the device must match the one in the build directory exactly. '
"Try '%s flash monitor'." % (elf_file, ctx.info_name), ctx)
idf_monitor = os.path.join(os.environ["IDF_PATH"], "tools/idf_monitor.py")
idf_monitor = os.path.join(os.environ['IDF_PATH'], 'tools/idf_monitor.py')
monitor_args = [PYTHON, idf_monitor]
esp_port = args.port or _get_default_serial_port(args)
monitor_args += ["-p", esp_port]
monitor_args += ['-p', esp_port]
if not monitor_baud:
monitor_baud = os.getenv("IDF_MONITOR_BAUD") or os.getenv("MONITORBAUD") or project_desc["monitor_baud"]
monitor_baud = os.getenv('IDF_MONITOR_BAUD') or os.getenv('MONITORBAUD') or project_desc['monitor_baud']
monitor_args += ["-b", monitor_baud]
monitor_args += ["--toolchain-prefix", project_desc["monitor_toolprefix"]]
monitor_args += ['-b', monitor_baud]
monitor_args += ['--toolchain-prefix', project_desc['monitor_toolprefix']]
coredump_decode = get_sdkconfig_value(project_desc["config_file"], "CONFIG_ESP_COREDUMP_DECODE")
coredump_decode = get_sdkconfig_value(project_desc['config_file'], 'CONFIG_ESP_COREDUMP_DECODE')
if coredump_decode is not None:
monitor_args += ["--decode-coredumps", coredump_decode]
monitor_args += ['--decode-coredumps', coredump_decode]
target_arch_riscv = get_sdkconfig_value(project_desc["config_file"], "CONFIG_IDF_TARGET_ARCH_RISCV")
target_arch_riscv = get_sdkconfig_value(project_desc['config_file'], 'CONFIG_IDF_TARGET_ARCH_RISCV')
if target_arch_riscv:
monitor_args += ["--decode-panic", "backtrace", "--target", project_desc["target"]]
monitor_args += ['--decode-panic', 'backtrace', '--target', project_desc['target']]
if print_filter is not None:
monitor_args += ["--print_filter", print_filter]
monitor_args += ['--print_filter', print_filter]
monitor_args += [elf_file]
if encrypted:
monitor_args += ['--encrypted']
idf_py = [PYTHON] + _get_commandline_options(ctx) # commands to re-run idf.py
monitor_args += ["-m", " ".join("'%s'" % a for a in idf_py)]
monitor_args += ['-m', ' '.join("'%s'" % a for a in idf_py)]
if "MSYSTEM" in os.environ:
monitor_args = ["winpty"] + monitor_args
run_tool("idf_monitor", monitor_args, args.project_dir)
if 'MSYSTEM' in os.environ:
monitor_args = ['winpty'] + monitor_args
run_tool('idf_monitor', monitor_args, args.project_dir)
def flash(action, ctx, args):
"""
@@ -114,126 +113,126 @@ def action_extensions(base_actions, project_path):
"""
ensure_build_directory(args, ctx.info_name)
esp_port = args.port or _get_default_serial_port(args)
run_target(action, args, {"ESPBAUD": str(args.baud),"ESPPORT": esp_port})
run_target(action, args, {'ESPBAUD': str(args.baud),'ESPPORT': esp_port})
def erase_flash(action, ctx, args):
ensure_build_directory(args, ctx.info_name)
esptool_args = _get_esptool_args(args)
esptool_args += ["erase_flash"]
run_tool("esptool.py", esptool_args, args.build_dir)
esptool_args += ['erase_flash']
run_tool('esptool.py', esptool_args, args.build_dir)
def global_callback(ctx, global_args, tasks):
encryption = any([task.name in ("encrypted-flash", "encrypted-app-flash") for task in tasks])
encryption = any([task.name in ('encrypted-flash', 'encrypted-app-flash') for task in tasks])
if encryption:
for task in tasks:
if task.name == "monitor":
task.action_args["encrypted"] = True
if task.name == 'monitor':
task.action_args['encrypted'] = True
break
baud_rate = {
"names": ["-b", "--baud"],
"help": "Baud rate for flashing.",
"scope": "global",
"envvar": "ESPBAUD",
"default": 460800,
'names': ['-b', '--baud'],
'help': 'Baud rate for flashing.',
'scope': 'global',
'envvar': 'ESPBAUD',
'default': 460800,
}
port = {
"names": ["-p", "--port"],
"help": "Serial port.",
"scope": "global",
"envvar": "ESPPORT",
"default": None,
'names': ['-p', '--port'],
'help': 'Serial port.',
'scope': 'global',
'envvar': 'ESPPORT',
'default': None,
}
serial_actions = {
"global_action_callbacks": [global_callback],
"actions": {
"flash": {
"callback": flash,
"help": "Flash the project.",
"options": global_options + [baud_rate, port],
"order_dependencies": ["all", "erase_flash"],
'global_action_callbacks': [global_callback],
'actions': {
'flash': {
'callback': flash,
'help': 'Flash the project.',
'options': global_options + [baud_rate, port],
'order_dependencies': ['all', 'erase_flash'],
},
"erase_flash": {
"callback": erase_flash,
"help": "Erase entire flash chip.",
"options": [baud_rate, port],
'erase_flash': {
'callback': erase_flash,
'help': 'Erase entire flash chip.',
'options': [baud_rate, port],
},
"monitor": {
"callback":
'monitor': {
'callback':
monitor,
"help":
"Display serial output.",
"options": [
'help':
'Display serial output.',
'options': [
port, {
"names": ["--print-filter", "--print_filter"],
"help":
("Filter monitor output. "
"Restrictions on what to print can be specified as a series of <tag>:<log_level> items "
"where <tag> is the tag string and <log_level> is a character from the set "
"{N, E, W, I, D, V, *} referring to a level. "
'names': ['--print-filter', '--print_filter'],
'help':
('Filter monitor output. '
'Restrictions on what to print can be specified as a series of <tag>:<log_level> items '
'where <tag> is the tag string and <log_level> is a character from the set '
'{N, E, W, I, D, V, *} referring to a level. '
'For example, "tag1:W" matches and prints only the outputs written with '
'ESP_LOGW("tag1", ...) or at lower verbosity level, i.e. ESP_LOGE("tag1", ...). '
'Not specifying a <log_level> or using "*" defaults to Verbose level. '
'Please see the IDF Monitor section of the ESP-IDF documentation '
'for a more detailed description and further examples.'),
"default":
'default':
None,
}, {
"names": ["--monitor-baud", "-B"],
"type":
'names': ['--monitor-baud', '-B'],
'type':
click.INT,
"help": ("Baud rate for monitor. "
"If this option is not provided IDF_MONITOR_BAUD and MONITORBAUD "
"environment variables and project_description.json in build directory "
'help': ('Baud rate for monitor. '
'If this option is not provided IDF_MONITOR_BAUD and MONITORBAUD '
'environment variables and project_description.json in build directory '
"(generated by CMake from project's sdkconfig) "
"will be checked for default value."),
'will be checked for default value.'),
}, {
"names": ["--encrypted", "-E"],
"is_flag": True,
"help": ("Enable encrypted flash targets. "
"IDF Monitor will invoke encrypted-flash and encrypted-app-flash targets "
"if this option is set. This option is set by default if IDF Monitor was invoked "
"together with encrypted-flash or encrypted-app-flash target."),
'names': ['--encrypted', '-E'],
'is_flag': True,
'help': ('Enable encrypted flash targets. '
'IDF Monitor will invoke encrypted-flash and encrypted-app-flash targets '
'if this option is set. This option is set by default if IDF Monitor was invoked '
'together with encrypted-flash or encrypted-app-flash target.'),
}
],
"order_dependencies": [
"flash",
"encrypted-flash",
"partition_table-flash",
"bootloader-flash",
"app-flash",
"encrypted-app-flash",
'order_dependencies': [
'flash',
'encrypted-flash',
'partition_table-flash',
'bootloader-flash',
'app-flash',
'encrypted-app-flash',
],
},
"partition_table-flash": {
"callback": flash,
"help": "Flash partition table only.",
"options": [baud_rate, port],
"order_dependencies": ["partition_table", "erase_flash"],
'partition_table-flash': {
'callback': flash,
'help': 'Flash partition table only.',
'options': [baud_rate, port],
'order_dependencies': ['partition_table', 'erase_flash'],
},
"bootloader-flash": {
"callback": flash,
"help": "Flash bootloader only.",
"options": [baud_rate, port],
"order_dependencies": ["bootloader", "erase_flash"],
'bootloader-flash': {
'callback': flash,
'help': 'Flash bootloader only.',
'options': [baud_rate, port],
'order_dependencies': ['bootloader', 'erase_flash'],
},
"app-flash": {
"callback": flash,
"help": "Flash the app only.",
"options": [baud_rate, port],
"order_dependencies": ["app", "erase_flash"],
'app-flash': {
'callback': flash,
'help': 'Flash the app only.',
'options': [baud_rate, port],
'order_dependencies': ['app', 'erase_flash'],
},
"encrypted-app-flash": {
"callback": flash,
"help": "Flash the encrypted app only.",
"order_dependencies": ["app", "erase_flash"],
'encrypted-app-flash': {
'callback': flash,
'help': 'Flash the encrypted app only.',
'order_dependencies': ['app', 'erase_flash'],
},
"encrypted-flash": {
"callback": flash,
"help": "Flash the encrypted project.",
"order_dependencies": ["all", "erase_flash"],
'encrypted-flash': {
'callback': flash,
'help': 'Flash the encrypted project.',
'order_dependencies': ['all', 'erase_flash'],
},
},
}

101
tools/idf_py_actions/tools.py
View File

@@ -1,10 +1,11 @@
import click
import os
import re
import subprocess
import sys
from io import open
import click
from .constants import GENERATORS
from .errors import FatalError
@@ -29,8 +30,8 @@ def realpath(path):
def _idf_version_from_cmake():
version_path = os.path.join(os.environ["IDF_PATH"], "tools/cmake/version.cmake")
regex = re.compile(r"^\s*set\s*\(\s*IDF_VERSION_([A-Z]{5})\s+(\d+)")
version_path = os.path.join(os.environ['IDF_PATH'], 'tools/cmake/version.cmake')
regex = re.compile(r'^\s*set\s*\(\s*IDF_VERSION_([A-Z]{5})\s+(\d+)')
ver = {}
try:
with open(version_path) as f:
@@ -40,9 +41,9 @@ def _idf_version_from_cmake():
if m:
ver[m.group(1)] = m.group(2)
return "v%s.%s.%s" % (ver["MAJOR"], ver["MINOR"], ver["PATCH"])
return 'v%s.%s.%s' % (ver['MAJOR'], ver['MINOR'], ver['PATCH'])
except (KeyError, OSError):
sys.stderr.write("WARNING: Cannot find ESP-IDF version in version.cmake\n")
sys.stderr.write('WARNING: Cannot find ESP-IDF version in version.cmake\n')
return None
@@ -52,13 +53,13 @@ def idf_version():
# Try to get version from git:
try:
version = subprocess.check_output([
"git",
"--git-dir=%s" % os.path.join(os.environ["IDF_PATH"], '.git'),
"--work-tree=%s" % os.environ["IDF_PATH"], "describe", "--tags", "--dirty"
'git',
'--git-dir=%s' % os.path.join(os.environ['IDF_PATH'], '.git'),
'--work-tree=%s' % os.environ['IDF_PATH'], 'describe', '--tags', '--dirty'
]).decode('utf-8', 'ignore').strip()
except (subprocess.CalledProcessError, UnicodeError):
# if failed, then try to parse cmake.version file
sys.stderr.write("WARNING: Git version unavailable, reading from source\n")
sys.stderr.write('WARNING: Git version unavailable, reading from source\n')
version = _idf_version_from_cmake()
return version
@@ -67,13 +68,13 @@ def idf_version():
def run_tool(tool_name, args, cwd, env=dict()):
def quote_arg(arg):
" Quote 'arg' if necessary "
if " " in arg and not (arg.startswith('"') or arg.startswith("'")):
if ' ' in arg and not (arg.startswith('"') or arg.startswith("'")):
return "'" + arg + "'"
return arg
args = [str(arg) for arg in args]
display_args = " ".join(quote_arg(arg) for arg in args)
print("Running %s in directory %s" % (tool_name, quote_arg(cwd)))
display_args = ' '.join(quote_arg(arg) for arg in args)
print('Running %s in directory %s' % (tool_name, quote_arg(cwd)))
print('Executing "%s"...' % str(display_args))
env_copy = dict(os.environ)
@@ -90,14 +91,14 @@ def run_tool(tool_name, args, cwd, env=dict()):
# Note: we explicitly pass in os.environ here, as we may have set IDF_PATH there during startup
subprocess.check_call(args, env=env_copy, cwd=cwd)
except subprocess.CalledProcessError as e:
raise FatalError("%s failed with exit code %d" % (tool_name, e.returncode))
raise FatalError('%s failed with exit code %d' % (tool_name, e.returncode))
def run_target(target_name, args, env=dict()):
generator_cmd = GENERATORS[args.generator]["command"]
generator_cmd = GENERATORS[args.generator]['command']
if args.verbose:
generator_cmd += [GENERATORS[args.generator]["verbose_flag"]]
generator_cmd += [GENERATORS[args.generator]['verbose_flag']]
run_tool(generator_cmd[0], generator_cmd + [target_name], args.build_dir, env)
@@ -123,7 +124,7 @@ def _parse_cmakecache(path):
for line in f:
# cmake cache lines look like: CMAKE_CXX_FLAGS_DEBUG:STRING=-g
# groups are name, type, value
m = re.match(r"^([^#/:=]+):([^:=]+)=(.*)\n$", line)
m = re.match(r'^([^#/:=]+):([^:=]+)=(.*)\n$', line)
if m:
result[m.group(1)] = m.group(3)
return result
@@ -137,7 +138,7 @@ def _new_cmakecache_entries(cache_path, new_cache_entries):
current_cache = _parse_cmakecache(cache_path)
for entry in new_cache_entries:
key, value = entry.split("=", 1)
key, value = entry.split('=', 1)
current_value = current_cache.get(key, None)
if current_value is None or _strip_quotes(value) != current_value:
return True
@@ -150,7 +151,7 @@ def _detect_cmake_generator(prog_name):
Find the default cmake generator, if none was specified. Raises an exception if no valid generator is found.
"""
for (generator_name, generator) in GENERATORS.items():
if executable_exists(generator["version"]):
if executable_exists(generator['version']):
return generator_name
raise FatalError("To use %s, either the 'ninja' or 'GNU make' build tool must be available in the PATH" % prog_name)
@@ -169,11 +170,11 @@ def ensure_build_directory(args, prog_name, always_run_cmake=False):
# Verify the project directory
if not os.path.isdir(project_dir):
if not os.path.exists(project_dir):
raise FatalError("Project directory %s does not exist" % project_dir)
raise FatalError('Project directory %s does not exist' % project_dir)
else:
raise FatalError("%s must be a project directory" % project_dir)
if not os.path.exists(os.path.join(project_dir, "CMakeLists.txt")):
raise FatalError("CMakeLists.txt not found in project directory %s" % project_dir)
raise FatalError('%s must be a project directory' % project_dir)
if not os.path.exists(os.path.join(project_dir, 'CMakeLists.txt')):
raise FatalError('CMakeLists.txt not found in project directory %s' % project_dir)
# Verify/create the build directory
build_dir = args.build_dir
@@ -181,33 +182,33 @@ def ensure_build_directory(args, prog_name, always_run_cmake=False):
os.makedirs(build_dir)
# Parse CMakeCache, if it exists
cache_path = os.path.join(build_dir, "CMakeCache.txt")
cache_path = os.path.join(build_dir, 'CMakeCache.txt')
cache = _parse_cmakecache(cache_path) if os.path.exists(cache_path) else {}
# Validate or set IDF_TARGET
_guess_or_check_idf_target(args, prog_name, cache)
args.define_cache_entry.append("CCACHE_ENABLE=%d" % args.ccache)
args.define_cache_entry.append('CCACHE_ENABLE=%d' % args.ccache)
if always_run_cmake or _new_cmakecache_entries(cache_path, args.define_cache_entry):
if args.generator is None:
args.generator = _detect_cmake_generator(prog_name)
try:
cmake_args = [
"cmake",
"-G",
'cmake',
'-G',
args.generator,
"-DPYTHON_DEPS_CHECKED=1",
"-DESP_PLATFORM=1",
'-DPYTHON_DEPS_CHECKED=1',
'-DESP_PLATFORM=1',
]
if args.cmake_warn_uninitialized:
cmake_args += ["--warn-uninitialized"]
cmake_args += ['--warn-uninitialized']
if args.define_cache_entry:
cmake_args += ["-D" + d for d in args.define_cache_entry]
cmake_args += ['-D' + d for d in args.define_cache_entry]
cmake_args += [project_dir]
run_tool("cmake", cmake_args, cwd=args.build_dir)
run_tool('cmake', cmake_args, cwd=args.build_dir)
except Exception:
# don't allow partially valid CMakeCache.txt files,
# to keep the "should I run cmake?" logic simple
@@ -219,7 +220,7 @@ def ensure_build_directory(args, prog_name, always_run_cmake=False):
cache = _parse_cmakecache(cache_path) if os.path.exists(cache_path) else {}
try:
generator = cache["CMAKE_GENERATOR"]
generator = cache['CMAKE_GENERATOR']
except KeyError:
generator = _detect_cmake_generator(prog_name)
if args.generator is None:
@@ -229,7 +230,7 @@ def ensure_build_directory(args, prog_name, always_run_cmake=False):
(generator, args.generator, prog_name))
try:
home_dir = cache["CMAKE_HOME_DIRECTORY"]
home_dir = cache['CMAKE_HOME_DIRECTORY']
if realpath(home_dir) != realpath(project_dir):
raise FatalError(
"Build directory '%s' configured for project '%s' not '%s'. Run '%s fullclean' to start again." %
@@ -240,14 +241,14 @@ def ensure_build_directory(args, prog_name, always_run_cmake=False):
def merge_action_lists(*action_lists):
merged_actions = {
"global_options": [],
"actions": {},
"global_action_callbacks": [],
'global_options': [],
'actions': {},
'global_action_callbacks': [],
}
for action_list in action_lists:
merged_actions["global_options"].extend(action_list.get("global_options", []))
merged_actions["actions"].update(action_list.get("actions", {}))
merged_actions["global_action_callbacks"].extend(action_list.get("global_action_callbacks", []))
merged_actions['global_options'].extend(action_list.get('global_options', []))
merged_actions['actions'].update(action_list.get('actions', {}))
merged_actions['global_action_callbacks'].extend(action_list.get('global_action_callbacks', []))
return merged_actions
@@ -256,14 +257,14 @@ def get_sdkconfig_value(sdkconfig_file, key):
Return the value of given key from sdkconfig_file.
If sdkconfig_file does not exist or the option is not present, returns None.
"""
assert key.startswith("CONFIG_")
assert key.startswith('CONFIG_')
if not os.path.exists(sdkconfig_file):
return None
# keep track of the last seen value for the given key
value = None
# if the value is quoted, this excludes the quotes from the value
pattern = re.compile(r"^{}=\"?([^\"]*)\"?$".format(key))
with open(sdkconfig_file, "r") as f:
with open(sdkconfig_file, 'r') as f:
for line in f:
match = re.match(pattern, line)
if match:
@@ -275,7 +276,7 @@ def is_target_supported(project_path, supported_targets):
"""
Returns True if the active target is supported, or False otherwise.
"""
return get_sdkconfig_value(os.path.join(project_path, "sdkconfig"), 'CONFIG_IDF_TARGET') in supported_targets
return get_sdkconfig_value(os.path.join(project_path, 'sdkconfig'), 'CONFIG_IDF_TARGET') in supported_targets
def _guess_or_check_idf_target(args, prog_name, cache):
@@ -288,14 +289,14 @@ def _guess_or_check_idf_target(args, prog_name, cache):
"""
# Default locations of sdkconfig files.
# FIXME: they may be overridden in the project or by a CMake variable (IDF-1369).
sdkconfig_path = os.path.join(args.project_dir, "sdkconfig")
sdkconfig_defaults_path = os.path.join(args.project_dir, "sdkconfig.defaults")
sdkconfig_path = os.path.join(args.project_dir, 'sdkconfig')
sdkconfig_defaults_path = os.path.join(args.project_dir, 'sdkconfig.defaults')
# These are used to guess the target from sdkconfig, or set the default target by sdkconfig.defaults.
idf_target_from_sdkconfig = get_sdkconfig_value(sdkconfig_path, "CONFIG_IDF_TARGET")
idf_target_from_sdkconfig_defaults = get_sdkconfig_value(sdkconfig_defaults_path, "CONFIG_IDF_TARGET")
idf_target_from_env = os.environ.get("IDF_TARGET")
idf_target_from_cache = cache.get("IDF_TARGET")
idf_target_from_sdkconfig = get_sdkconfig_value(sdkconfig_path, 'CONFIG_IDF_TARGET')
idf_target_from_sdkconfig_defaults = get_sdkconfig_value(sdkconfig_defaults_path, 'CONFIG_IDF_TARGET')
idf_target_from_env = os.environ.get('IDF_TARGET')
idf_target_from_cache = cache.get('IDF_TARGET')
if not cache and not idf_target_from_env:
# CMakeCache.txt does not exist yet, and IDF_TARGET is not set in the environment.
@@ -303,7 +304,7 @@ def _guess_or_check_idf_target(args, prog_name, cache):
if guessed_target:
if args.verbose:
print("IDF_TARGET is not set, guessed '%s' from sdkconfig" % (guessed_target))
args.define_cache_entry.append("IDF_TARGET=" + guessed_target)
args.define_cache_entry.append('IDF_TARGET=' + guessed_target)
elif idf_target_from_env:
# Let's check that IDF_TARGET values are consistent
@@ -336,7 +337,7 @@ class TargetChoice(click.Choice):
def convert(self, value, param, ctx):
def normalize(str):
return str.lower().replace("-", "")
return str.lower().replace('-', '')
saved_token_normalize_func = ctx.token_normalize_func
ctx.token_normalize_func = normalize

18
tools/idf_py_actions/uf2_ext.py
View File

@@ -7,16 +7,16 @@ def action_extensions(base_actions, project_path):
run_target(target_name, args)
uf2_actions = {
"actions": {
"uf2": {
"callback": uf2_target,
"short_help": "Generate the UF2 binary with all the binaries included",
"dependencies": ["all"],
'actions': {
'uf2': {
'callback': uf2_target,
'short_help': 'Generate the UF2 binary with all the binaries included',
'dependencies': ['all'],
},
"uf2-app": {
"callback": uf2_target,
"short_help": "Generate an UF2 binary for the application only",
"dependencies": ["all"],
'uf2-app': {
'callback': uf2_target,
'short_help': 'Generate an UF2 binary for the application only',
'dependencies': ['all'],
},
}
}

86
tools/idf_size.py
View File

@@ -20,10 +20,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
from __future__ import print_function
from __future__ import unicode_literals
from __future__ import division
from future.utils import iteritems
from __future__ import division, print_function, unicode_literals
import argparse
import collections
import json
@@ -31,6 +29,8 @@ import os.path
import re
import sys
from future.utils import iteritems
GLOBAL_JSON_INDENT = 4
GLOBAL_JSON_SEPARATORS = (',', ': ')
@@ -202,8 +202,8 @@ def load_map_data(map_file):
def load_memory_config(map_file):
""" Memory Configuration section is the total size of each output section """
result = {}
scan_to_header(map_file, "Memory Configuration")
RE_MEMORY_SECTION = re.compile(r"(?P<name>[^ ]+) +0x(?P<origin>[\da-f]+) +0x(?P<length>[\da-f]+)")
scan_to_header(map_file, 'Memory Configuration')
RE_MEMORY_SECTION = re.compile(r'(?P<name>[^ ]+) +0x(?P<origin>[\da-f]+) +0x(?P<length>[\da-f]+)')
for line in map_file:
m = RE_MEMORY_SECTION.match(line)
@@ -213,13 +213,13 @@ def load_memory_config(map_file):
else:
return result # we're at the end of the Memory Configuration
section = {
"name": m.group("name"),
"origin": int(m.group("origin"), 16),
"length": int(m.group("length"), 16),
'name': m.group('name'),
'origin': int(m.group('origin'), 16),
'length': int(m.group('length'), 16),
}
if section["name"] != "*default*":
result[section["name"]] = section
raise RuntimeError("End of file while scanning memory configuration?")
if section['name'] != '*default*':
result[section['name']] = section
raise RuntimeError('End of file while scanning memory configuration?')
def detect_target_chip(map_file):
@@ -258,45 +258,45 @@ def load_sections(map_file):
information for each symbol linked into the section.
"""
# output section header, ie '.iram0.text 0x0000000040080400 0x129a5'
RE_SECTION_HEADER = re.compile(r"(?P<name>[^ ]+) +0x(?P<address>[\da-f]+) +0x(?P<size>[\da-f]+)$")
RE_SECTION_HEADER = re.compile(r'(?P<name>[^ ]+) +0x(?P<address>[\da-f]+) +0x(?P<size>[\da-f]+)$')
# source file line, ie
# 0x0000000040080400 0xa4 /home/gus/esp/32/idf/examples/get-started/hello_world/build/esp32/libesp32.a(cpu_start.o)
# cmake build system links some object files directly, not part of any archive, so make that part optional
# .xtensa.info 0x0000000000000000 0x38 CMakeFiles/hello-world.elf.dir/project_elf_src.c.obj
RE_SOURCE_LINE = re.compile(r"\s*(?P<sym_name>\S*) +0x(?P<address>[\da-f]+) +0x(?P<size>[\da-f]+) (?P<archive>.+\.a)?\(?(?P<object_file>.+\.(o|obj))\)?")
RE_SOURCE_LINE = re.compile(r'\s*(?P<sym_name>\S*) +0x(?P<address>[\da-f]+) +0x(?P<size>[\da-f]+) (?P<archive>.+\.a)?\(?(?P<object_file>.+\.(o|obj))\)?')
# Fast check to see if line is a potential source line before running the slower full regex against it
RE_PRE_FILTER = re.compile(r".*\.(o|obj)\)?")
RE_PRE_FILTER = re.compile(r'.*\.(o|obj)\)?')
# Check for lines which only contain the sym name (and rest is on following lines)
RE_SYMBOL_ONLY_LINE = re.compile(r"^ (?P<sym_name>\S*)$")
RE_SYMBOL_ONLY_LINE = re.compile(r'^ (?P<sym_name>\S*)$')
sections = {}
section = None
sym_backup = None
for line in map_file:
if line.strip() == "Cross Reference Table":
if line.strip() == 'Cross Reference Table':
# stop processing lines because we are at the next section in the map file
break
m = RE_SECTION_HEADER.match(line)
if m is not None: # start of a new section
section = {
"name": m.group("name"),
"address": int(m.group("address"), 16),
"size": int(m.group("size"), 16),
"sources": [],
'name': m.group('name'),
'address': int(m.group('address'), 16),
'size': int(m.group('size'), 16),
'sources': [],
}
sections[section["name"]] = section
sections[section['name']] = section
continue
if section is not None:
m = RE_SYMBOL_ONLY_LINE.match(line)
if m is not None:
# In some cases the section name appears on the previous line, back it up in here
sym_backup = m.group("sym_name")
sym_backup = m.group('sym_name')
continue
if not RE_PRE_FILTER.match(line):
@@ -305,21 +305,21 @@ def load_sections(map_file):
m = RE_SOURCE_LINE.match(line)
if m is not None: # input source file details=ma,e
sym_name = m.group("sym_name") if len(m.group("sym_name")) > 0 else sym_backup
archive = m.group("archive")
sym_name = m.group('sym_name') if len(m.group('sym_name')) > 0 else sym_backup
archive = m.group('archive')
if archive is None:
# optional named group "archive" was not matched, so assign a value to it
archive = "(exe)"
archive = '(exe)'
source = {
"size": int(m.group("size"), 16),
"address": int(m.group("address"), 16),
"archive": os.path.basename(archive),
"object_file": os.path.basename(m.group("object_file")),
"sym_name": sym_name,
'size': int(m.group('size'), 16),
'address': int(m.group('address'), 16),
'archive': os.path.basename(archive),
'object_file': os.path.basename(m.group('object_file')),
'sym_name': sym_name,
}
source["file"] = "%s:%s" % (source["archive"], source["object_file"])
section["sources"] += [source]
source['file'] = '%s:%s' % (source['archive'], source['object_file'])
section['sources'] += [source]
return sections
@@ -339,12 +339,12 @@ class MemRegNames(object):
def main():
parser = argparse.ArgumentParser(description="idf_size - a tool to print size information from an IDF MAP file")
parser = argparse.ArgumentParser(description='idf_size - a tool to print size information from an IDF MAP file')
parser.add_argument(
'--json',
help="Output results as JSON",
action="store_true")
help='Output results as JSON',
action='store_true')
parser.add_argument(
'map_file', help='MAP file produced by linker',
@@ -373,7 +373,7 @@ def main():
'--output-file',
type=argparse.FileType('w'),
default=sys.stdout,
help="Print output to the specified file instead of stdout")
help='Print output to the specified file instead of stdout')
args = parser.parse_args()
@@ -415,9 +415,9 @@ def main():
args.another_map_file, mem_reg_diff, memory_config_diff, sections_diff)
if args.archives:
output += get_detailed_sizes(mem_reg, sections, "archive", "Archive File", args.json, sections_diff)
output += get_detailed_sizes(mem_reg, sections, 'archive', 'Archive File', args.json, sections_diff)
if args.files:
output += get_detailed_sizes(mem_reg, sections, "file", "Object File", args.json, sections_diff)
output += get_detailed_sizes(mem_reg, sections, 'file', 'Object File', args.json, sections_diff)
if args.archive_details:
output += get_archive_symbols(mem_reg, sections, args.archive_details, args.json, sections_diff)
@@ -657,13 +657,13 @@ class StructureForDetailedSizes(object):
"""
result = {}
for _, section in iteritems(sections):
for s in section["sources"]:
for s in section['sources']:
if not s[key] in result:
result[s[key]] = {}
archive = result[s[key]]
if not section["name"] in archive:
archive[section["name"]] = 0
archive[section["name"]] += s["size"]
if not section['name'] in archive:
archive[section['name']] = 0
archive[section['name']] += s['size']
return result
@staticmethod

66
tools/idf_tools.py
View File

@@ -40,23 +40,23 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import argparse
import contextlib
import copy
import errno
import functools
import hashlib
import json
import os
import platform
import re
import shutil
import ssl
import subprocess
import sys
import argparse
import re
import platform
import hashlib
import tarfile
import zipfile
import errno
import shutil
import functools
import copy
from collections import OrderedDict, namedtuple
import ssl
import contextlib
try:
import typing # noqa: F401
@@ -64,10 +64,10 @@ except ImportError:
pass
try:
from urllib.request import urlopen
from urllib.error import ContentTooShortError
from urllib.request import urlopen
except ImportError:
from urllib import urlopen, ContentTooShortError
from urllib import ContentTooShortError, urlopen
try:
from exceptions import WindowsError
@@ -277,7 +277,7 @@ def get_file_size_sha256(filename, block_size=65536):
def report_progress(count, block_size, total_size):
percent = int(count * block_size * 100 / total_size)
percent = min(100, percent)
sys.stdout.write("\r%d%%" % percent)
sys.stdout.write('\r%d%%' % percent)
sys.stdout.flush()
@@ -319,13 +319,13 @@ def urlretrieve_ctx(url, filename, reporthook=None, data=None, context=None):
# urlopen doesn't have context argument in Python <=2.7.9
extra_urlopen_args = {}
if context:
extra_urlopen_args["context"] = context
extra_urlopen_args['context'] = context
with contextlib.closing(urlopen(url, data, **extra_urlopen_args)) as fp:
headers = fp.info()
# Just return the local path and the "headers" for file://
# URLs. No sense in performing a copy unless requested.
if url_type == "file" and not filename:
if url_type == 'file' and not filename:
return os.path.normpath(path), headers
# Handle temporary file setup.
@@ -334,7 +334,7 @@ def urlretrieve_ctx(url, filename, reporthook=None, data=None, context=None):
with tfp:
result = filename, headers
bs = 1024 * 8
size = int(headers.get("content-length", -1))
size = int(headers.get('content-length', -1))
read = 0
blocknum = 0
@@ -353,7 +353,7 @@ def urlretrieve_ctx(url, filename, reporthook=None, data=None, context=None):
if size >= 0 and read < size:
raise ContentTooShortError(
"retrieval incomplete: got only %i out of %i bytes"
'retrieval incomplete: got only %i out of %i bytes'
% (read, size), result)
return result
@@ -571,7 +571,7 @@ class IDFTool(object):
raise ToolExecError('Command {} has returned non-zero exit code ({})\n'.format(
' '.join(self._current_options.version_cmd), e.returncode))
in_str = version_cmd_result.decode("utf-8")
in_str = version_cmd_result.decode('utf-8')
match = re.search(self._current_options.version_regex, in_str)
if not match:
return UNKNOWN_VERSION
@@ -677,7 +677,7 @@ class IDFTool(object):
ctx = None
# For dl.espressif.com, add the ISRG x1 root certificate.
# This works around the issue with outdated certificate stores in some installations.
if "dl.espressif.com" in url:
if 'dl.espressif.com' in url:
try:
ctx = ssl.create_default_context()
ctx.load_verify_locations(cadata=ISRG_X1_ROOT_CERT)
@@ -687,7 +687,7 @@ class IDFTool(object):
pass
urlretrieve_ctx(url, local_temp_path, report_progress if not global_non_interactive else None, context=ctx)
sys.stdout.write("\rDone\n")
sys.stdout.write('\rDone\n')
except Exception as e:
# urlretrieve could throw different exceptions, e.g. IOError when the server is down
# Errors are ignored because the downloaded file is checked a couple of lines later.
@@ -959,7 +959,7 @@ def get_python_env_path():
version_file_path = os.path.join(global_idf_path, 'version.txt')
if os.path.exists(version_file_path):
with open(version_file_path, "r") as version_file:
with open(version_file_path, 'r') as version_file:
idf_version_str = version_file.read()
else:
try:
@@ -1209,7 +1209,7 @@ def apply_github_assets_option(tool_download_obj):
IDF_GITHUB_ASSETS is set. The github.com part of the URL will be replaced.
"""
try:
github_assets = os.environ["IDF_GITHUB_ASSETS"].strip()
github_assets = os.environ['IDF_GITHUB_ASSETS'].strip()
except KeyError:
return # no IDF_GITHUB_ASSETS
if not github_assets: # variable exists but is empty
@@ -1350,7 +1350,7 @@ def action_install_python_env(args):
info('Creating a new Python environment in {}'.format(idf_python_env_path))
try:
import virtualenv # noqa: F401
import virtualenv # noqa: F401
except ImportError:
info('Installing virtualenv')
subprocess.check_call([sys.executable, '-m', 'pip', 'install', '--user', 'virtualenv'],
@@ -1454,17 +1454,17 @@ def action_gen_doc(args):
def print_out(text):
f.write(text + '\n')
print_out(".. |zwsp| unicode:: U+200B")
print_out(" :trim:")
print_out("")
print_out('.. |zwsp| unicode:: U+200B')
print_out(' :trim:')
print_out('')
idf_gh_url = "https://github.com/espressif/esp-idf"
idf_gh_url = 'https://github.com/espressif/esp-idf'
for tool_name, tool_obj in tools_info.items():
info_url = tool_obj.options.info_url
if idf_gh_url + "/tree" in info_url:
info_url = re.sub(idf_gh_url + r"/tree/\w+/(.*)", r":idf:`\1`", info_url)
if idf_gh_url + '/tree' in info_url:
info_url = re.sub(idf_gh_url + r'/tree/\w+/(.*)', r':idf:`\1`', info_url)
license_url = "https://spdx.org/licenses/" + tool_obj.options.license
license_url = 'https://spdx.org/licenses/' + tool_obj.options.license
print_out("""
.. _tool-{name}:
@@ -1502,9 +1502,9 @@ More info: {info_url}
if install_type == IDFTool.INSTALL_NEVER:
continue
elif install_type == IDFTool.INSTALL_ALWAYS:
install_type_str = "required"
install_type_str = 'required'
elif install_type == IDFTool.INSTALL_ON_REQUEST:
install_type_str = "optional"
install_type_str = 'optional'
else:
raise NotImplementedError()
@@ -1615,7 +1615,7 @@ def main(argv):
if args.idf_path:
global_idf_path = args.idf_path
if not global_idf_path:
global_idf_path = os.path.realpath(os.path.join(os.path.dirname(__file__), ".."))
global_idf_path = os.path.realpath(os.path.join(os.path.dirname(__file__), '..'))
os.environ['IDF_PATH'] = global_idf_path
global global_idf_tools_path

123
tools/kconfig_new/confgen.py
View File

@@ -21,10 +21,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import print_function
from future.utils import iteritems
import argparse
import json
import kconfiglib
import os
import os.path
import re
@@ -32,11 +31,13 @@ import sys
import tempfile
import gen_kconfig_doc
import kconfiglib
from future.utils import iteritems
__version__ = "0.1"
__version__ = '0.1'
if "IDF_CMAKE" not in os.environ:
os.environ["IDF_CMAKE"] = ""
if 'IDF_CMAKE' not in os.environ:
os.environ['IDF_CMAKE'] = ''
class DeprecatedOptions(object):
@@ -168,7 +169,7 @@ class DeprecatedOptions(object):
def _opt_defined(opt):
if not opt.visibility:
return False
return not (opt.orig_type in (kconfiglib.BOOL, kconfiglib.TRISTATE) and opt.str_value == "n")
return not (opt.orig_type in (kconfiglib.BOOL, kconfiglib.TRISTATE) and opt.str_value == 'n')
if len(self.r_dic) > 0:
with open(path_output, 'a') as f_o:
@@ -241,7 +242,7 @@ def main():
sys.exit(1)
try:
args.env = [(name,value) for (name,value) in (e.split("=",1) for e in args.env)]
args.env = [(name,value) for (name,value) in (e.split('=',1) for e in args.env)]
except ValueError:
print("--env arguments must each contain =. To unset an environment variable, use 'ENV='")
sys.exit(1)
@@ -258,7 +259,7 @@ def main():
config.warn_assign_override = False
sdkconfig_renames = [args.sdkconfig_rename] if args.sdkconfig_rename else []
sdkconfig_renames += os.environ.get("COMPONENT_SDKCONFIG_RENAMES", "").split()
sdkconfig_renames += os.environ.get('COMPONENT_SDKCONFIG_RENAMES', '').split()
deprecated_options = DeprecatedOptions(config.config_prefix, path_rename_files=sdkconfig_renames)
if len(args.defaults) > 0:
@@ -275,13 +276,13 @@ def main():
# always load defaults first, so any items which are not defined in that config
# will have the default defined in the defaults file
for name in args.defaults:
print("Loading defaults file %s..." % name)
print('Loading defaults file %s...' % name)
if not os.path.exists(name):
raise RuntimeError("Defaults file not found: %s" % name)
raise RuntimeError('Defaults file not found: %s' % name)
try:
with tempfile.NamedTemporaryFile(prefix="confgen_tmp", delete=False) as f:
with tempfile.NamedTemporaryFile(prefix='confgen_tmp', delete=False) as f:
temp_file1 = f.name
with tempfile.NamedTemporaryFile(prefix="confgen_tmp", delete=False) as f:
with tempfile.NamedTemporaryFile(prefix='confgen_tmp', delete=False) as f:
temp_file2 = f.name
deprecated_options.replace(sdkconfig_in=name, sdkconfig_out=temp_file1)
_replace_empty_assignments(temp_file1, temp_file2)
@@ -296,7 +297,7 @@ def main():
# If config file previously exists, load it
if args.config and os.path.exists(args.config):
# ... but replace deprecated options before that
with tempfile.NamedTemporaryFile(prefix="confgen_tmp", delete=False) as f:
with tempfile.NamedTemporaryFile(prefix='confgen_tmp', delete=False) as f:
temp_file = f.name
try:
deprecated_options.replace(sdkconfig_in=args.config, sdkconfig_out=temp_file)
@@ -315,7 +316,7 @@ def main():
# Output the files specified in the arguments
for output_type, filename in args.output:
with tempfile.NamedTemporaryFile(prefix="confgen_tmp", delete=False) as f:
with tempfile.NamedTemporaryFile(prefix='confgen_tmp', delete=False) as f:
temp_file = f.name
try:
output_function = OUTPUT_FORMATS[output_type]
@@ -344,7 +345,7 @@ def write_makefile(deprecated_options, config, filename):
# Espressif IoT Development Framework (ESP-IDF) Project Makefile Configuration
#
"""
with open(filename, "w") as f:
with open(filename, 'w') as f:
tmp_dep_lines = []
f.write(CONFIG_HEADING)
@@ -355,12 +356,12 @@ def write_makefile(deprecated_options, config, filename):
try:
value = int(value)
except ValueError:
value = ""
value = ''
elif orig_type == kconfiglib.HEX:
try:
value = hex(int(value, 16)) # ensure 0x prefix
except ValueError:
value = ""
value = ''
elif orig_type == kconfiglib.STRING:
value = '"{}"'.format(kconfiglib.escape(value))
else:
@@ -400,7 +401,7 @@ def write_header(deprecated_options, config, filename):
def write_cmake(deprecated_options, config, filename):
with open(filename, "w") as f:
with open(filename, 'w') as f:
tmp_dep_list = []
write = f.write
prefix = config.config_prefix
@@ -420,8 +421,8 @@ def write_cmake(deprecated_options, config, filename):
if sym.config_string:
val = sym.str_value
if sym.orig_type in (kconfiglib.BOOL, kconfiglib.TRISTATE) and val == "n":
val = "" # write unset values as empty variables
if sym.orig_type in (kconfiglib.BOOL, kconfiglib.TRISTATE) and val == 'n':
val = '' # write unset values as empty variables
elif sym.orig_type == kconfiglib.STRING:
val = kconfiglib.escape(val)
elif sym.orig_type == kconfiglib.HEX:
@@ -436,7 +437,7 @@ def write_cmake(deprecated_options, config, filename):
for n in config.node_iter():
write_node(n)
write("set(CONFIGS_LIST {})".format(";".join(configs_list)))
write('set(CONFIGS_LIST {})'.format(';'.join(configs_list)))
if len(tmp_dep_list) > 0:
write('\n# List of deprecated options for backward compatibility\n')
@@ -454,7 +455,7 @@ def get_json_values(config):
if sym.config_string:
val = sym.str_value
if sym.type in [kconfiglib.BOOL, kconfiglib.TRISTATE]:
val = (val != "n")
val = (val != 'n')
elif sym.type == kconfiglib.HEX:
val = int(val, 16)
elif sym.type == kconfiglib.INT:
@@ -467,7 +468,7 @@ def get_json_values(config):
def write_json(deprecated_options, config, filename):
config_dict = get_json_values(config)
with open(filename, "w") as f:
with open(filename, 'w') as f:
json.dump(config_dict, f, indent=4, sort_keys=True)
@@ -491,7 +492,7 @@ def get_menu_node_id(node):
result.append(slug)
node = node.parent
result = "-".join(reversed(result))
result = '-'.join(reversed(result))
return result
@@ -502,7 +503,7 @@ def write_json_menus(deprecated_options, config, filename):
def write_node(node):
try:
json_parent = node_lookup[node.parent]["children"]
json_parent = node_lookup[node.parent]['children']
except KeyError:
assert node.parent not in node_lookup # if fails, we have a parent node with no "children" entity (ie a bug)
json_parent = result # root level node
@@ -521,16 +522,16 @@ def write_json_menus(deprecated_options, config, filename):
new_json = None
if node.item == kconfiglib.MENU or is_menuconfig:
new_json = {"type": "menu",
"title": node.prompt[0],
"depends_on": depends,
"children": [],
new_json = {'type': 'menu',
'title': node.prompt[0],
'depends_on': depends,
'children': [],
}
if is_menuconfig:
sym = node.item
new_json["name"] = sym.name
new_json["help"] = node.help
new_json["is_menuconfig"] = is_menuconfig
new_json['name'] = sym.name
new_json['help'] = node.help
new_json['is_menuconfig'] = is_menuconfig
greatest_range = None
if len(sym.ranges) > 0:
# Note: Evaluating the condition using kconfiglib's expr_value
@@ -538,7 +539,7 @@ def write_json_menus(deprecated_options, config, filename):
for min_range, max_range, cond_expr in sym.ranges:
if kconfiglib.expr_value(cond_expr):
greatest_range = [min_range, max_range]
new_json["range"] = greatest_range
new_json['range'] = greatest_range
elif isinstance(node.item, kconfiglib.Symbol):
sym = node.item
@@ -553,38 +554,38 @@ def write_json_menus(deprecated_options, config, filename):
break
new_json = {
"type": kconfiglib.TYPE_TO_STR[sym.type],
"name": sym.name,
"title": node.prompt[0] if node.prompt else None,
"depends_on": depends,
"help": node.help,
"range": greatest_range,
"children": [],
'type': kconfiglib.TYPE_TO_STR[sym.type],
'name': sym.name,
'title': node.prompt[0] if node.prompt else None,
'depends_on': depends,
'help': node.help,
'range': greatest_range,
'children': [],
}
elif isinstance(node.item, kconfiglib.Choice):
choice = node.item
new_json = {
"type": "choice",
"title": node.prompt[0],
"name": choice.name,
"depends_on": depends,
"help": node.help,
"children": []
'type': 'choice',
'title': node.prompt[0],
'name': choice.name,
'depends_on': depends,
'help': node.help,
'children': []
}
if new_json:
node_id = get_menu_node_id(node)
if node_id in existing_ids:
raise RuntimeError("Config file contains two items with the same id: %s (%s). " +
"Please rename one of these items to avoid ambiguity." % (node_id, node.prompt[0]))
new_json["id"] = node_id
raise RuntimeError('Config file contains two items with the same id: %s (%s). ' +
'Please rename one of these items to avoid ambiguity.' % (node_id, node.prompt[0]))
new_json['id'] = node_id
json_parent.append(new_json)
node_lookup[node] = new_json
for n in config.node_iter():
write_node(n)
with open(filename, "w") as f:
with open(filename, 'w') as f:
f.write(json.dumps(result, sort_keys=True, indent=4))
@@ -601,26 +602,26 @@ def write_docs(deprecated_options, config, filename):
def update_if_changed(source, destination):
with open(source, "r") as f:
with open(source, 'r') as f:
source_contents = f.read()
if os.path.exists(destination):
with open(destination, "r") as f:
with open(destination, 'r') as f:
dest_contents = f.read()
if source_contents == dest_contents:
return # nothing to update
with open(destination, "w") as f:
with open(destination, 'w') as f:
f.write(source_contents)
OUTPUT_FORMATS = {"config": write_config,
"makefile": write_makefile, # only used with make in order to generate auto.conf
"header": write_header,
"cmake": write_cmake,
"docs": write_docs,
"json": write_json,
"json_menus": write_json_menus,
OUTPUT_FORMATS = {'config': write_config,
'makefile': write_makefile, # only used with make in order to generate auto.conf
'header': write_header,
'cmake': write_cmake,
'docs': write_docs,
'json': write_json,
'json_menus': write_json_menus,
}
@@ -635,5 +636,5 @@ if __name__ == '__main__':
try:
main()
except FatalError as e:
print("A fatal error occurred: %s" % e)
print('A fatal error occurred: %s' % e)
sys.exit(2)

95
tools/kconfig_new/confserver.py
View File

@@ -4,14 +4,15 @@
# with a caller
#
from __future__ import print_function
import argparse
import confgen
import json
import kconfiglib
import os
import sys
import tempfile
import confgen
import kconfiglib
from confgen import FatalError, __version__
# Min/Max supported protocol versions
@@ -47,15 +48,15 @@ def main():
args = parser.parse_args()
if args.version < MIN_PROTOCOL_VERSION:
print("Version %d is older than minimum supported protocol version %d. Client is much older than ESP-IDF version?" %
print('Version %d is older than minimum supported protocol version %d. Client is much older than ESP-IDF version?' %
(args.version, MIN_PROTOCOL_VERSION))
if args.version > MAX_PROTOCOL_VERSION:
print("Version %d is newer than maximum supported protocol version %d. Client is newer than ESP-IDF version?" %
print('Version %d is newer than maximum supported protocol version %d. Client is newer than ESP-IDF version?' %
(args.version, MAX_PROTOCOL_VERSION))
try:
args.env = [(name,value) for (name,value) in (e.split("=",1) for e in args.env)]
args.env = [(name,value) for (name,value) in (e.split('=',1) for e in args.env)]
except ValueError:
print("--env arguments must each contain =. To unset an environment variable, use 'ENV='")
sys.exit(1)
@@ -73,7 +74,7 @@ def main():
def run_server(kconfig, sdkconfig, sdkconfig_rename, default_version=MAX_PROTOCOL_VERSION):
config = kconfiglib.Kconfig(kconfig)
sdkconfig_renames = [sdkconfig_rename] if sdkconfig_rename else []
sdkconfig_renames += os.environ.get("COMPONENT_SDKCONFIG_RENAMES", "").split()
sdkconfig_renames += os.environ.get('COMPONENT_SDKCONFIG_RENAMES', '').split()
deprecated_options = confgen.DeprecatedOptions(config.config_prefix, path_rename_files=sdkconfig_renames)
f_o = tempfile.NamedTemporaryFile(mode='w+b', delete=False)
try:
@@ -85,7 +86,7 @@ def run_server(kconfig, sdkconfig, sdkconfig_rename, default_version=MAX_PROTOCO
os.unlink(f_o.name)
config.load_config(sdkconfig)
print("Server running, waiting for requests on stdin...", file=sys.stderr)
print('Server running, waiting for requests on stdin...', file=sys.stderr)
config_dict = confgen.get_json_values(config)
ranges_dict = get_ranges(config)
@@ -94,11 +95,11 @@ def run_server(kconfig, sdkconfig, sdkconfig_rename, default_version=MAX_PROTOCO
if default_version == 1:
# V1: no 'visibility' key, send value None for any invisible item
values_dict = dict((k, v if visible_dict[k] else False) for (k,v) in config_dict.items())
json.dump({"version": 1, "values": values_dict, "ranges": ranges_dict}, sys.stdout)
json.dump({'version': 1, 'values': values_dict, 'ranges': ranges_dict}, sys.stdout)
else:
# V2 onwards: separate visibility from version
json.dump({"version": default_version, "values": config_dict, "ranges": ranges_dict, "visible": visible_dict}, sys.stdout)
print("\n")
json.dump({'version': default_version, 'values': config_dict, 'ranges': ranges_dict, 'visible': visible_dict}, sys.stdout)
print('\n')
sys.stdout.flush()
while True:
@@ -108,18 +109,18 @@ def run_server(kconfig, sdkconfig, sdkconfig_rename, default_version=MAX_PROTOCO
try:
req = json.loads(line)
except ValueError as e: # json module throws JSONDecodeError (sublcass of ValueError) on Py3 but ValueError on Py2
response = {"version": default_version, "error": ["JSON formatting error: %s" % e]}
response = {'version': default_version, 'error': ['JSON formatting error: %s' % e]}
json.dump(response, sys.stdout)
print("\n")
print('\n')
sys.stdout.flush()
continue
before = confgen.get_json_values(config)
before_ranges = get_ranges(config)
before_visible = get_visible(config)
if "load" in req: # load a new sdkconfig
if 'load' in req: # load a new sdkconfig
if req.get("version", default_version) == 1:
if req.get('version', default_version) == 1:
# for V1 protocol, send all items when loading new sdkconfig.
# (V2+ will only send changes, same as when setting an item)
before = {}
@@ -127,16 +128,16 @@ def run_server(kconfig, sdkconfig, sdkconfig_rename, default_version=MAX_PROTOCO
before_visible = {}
# if no new filename is supplied, use existing sdkconfig path, otherwise update the path
if req["load"] is None:
req["load"] = sdkconfig
if req['load'] is None:
req['load'] = sdkconfig
else:
sdkconfig = req["load"]
sdkconfig = req['load']
if "save" in req:
if req["save"] is None:
req["save"] = sdkconfig
if 'save' in req:
if req['save'] is None:
req['save'] = sdkconfig
else:
sdkconfig = req["save"]
sdkconfig = req['save']
error = handle_request(deprecated_options, config, req)
@@ -147,51 +148,51 @@ def run_server(kconfig, sdkconfig, sdkconfig_rename, default_version=MAX_PROTOCO
values_diff = diff(before, after)
ranges_diff = diff(before_ranges, after_ranges)
visible_diff = diff(before_visible, after_visible)
if req["version"] == 1:
if req['version'] == 1:
# V1 response, invisible items have value None
for k in (k for (k,v) in visible_diff.items() if not v):
values_diff[k] = None
response = {"version": 1, "values": values_diff, "ranges": ranges_diff}
response = {'version': 1, 'values': values_diff, 'ranges': ranges_diff}
else:
# V2+ response, separate visibility values
response = {"version": req["version"], "values": values_diff, "ranges": ranges_diff, "visible": visible_diff}
response = {'version': req['version'], 'values': values_diff, 'ranges': ranges_diff, 'visible': visible_diff}
if error:
for e in error:
print("Error: %s" % e, file=sys.stderr)
response["error"] = error
print('Error: %s' % e, file=sys.stderr)
response['error'] = error
json.dump(response, sys.stdout)
print("\n")
print('\n')
sys.stdout.flush()
def handle_request(deprecated_options, config, req):
if "version" not in req:
if 'version' not in req:
return ["All requests must have a 'version'"]
if req["version"] < MIN_PROTOCOL_VERSION or req["version"] > MAX_PROTOCOL_VERSION:
return ["Unsupported request version %d. Server supports versions %d-%d" % (
req["version"],
if req['version'] < MIN_PROTOCOL_VERSION or req['version'] > MAX_PROTOCOL_VERSION:
return ['Unsupported request version %d. Server supports versions %d-%d' % (
req['version'],
MIN_PROTOCOL_VERSION,
MAX_PROTOCOL_VERSION)]
error = []
if "load" in req:
print("Loading config from %s..." % req["load"], file=sys.stderr)
if 'load' in req:
print('Loading config from %s...' % req['load'], file=sys.stderr)
try:
config.load_config(req["load"])
config.load_config(req['load'])
except Exception as e:
error += ["Failed to load from %s: %s" % (req["load"], e)]
error += ['Failed to load from %s: %s' % (req['load'], e)]
if "set" in req:
handle_set(config, error, req["set"])
if 'set' in req:
handle_set(config, error, req['set'])
if "save" in req:
if 'save' in req:
try:
print("Saving config to %s..." % req["save"], file=sys.stderr)
confgen.write_config(deprecated_options, config, req["save"])
print('Saving config to %s...' % req['save'], file=sys.stderr)
confgen.write_config(deprecated_options, config, req['save'])
except Exception as e:
error += ["Failed to save to %s: %s" % (req["save"], e)]
error += ['Failed to save to %s: %s' % (req['save'], e)]
return error
@@ -199,7 +200,7 @@ def handle_request(deprecated_options, config, req):
def handle_set(config, error, to_set):
missing = [k for k in to_set if k not in config.syms]
if missing:
error.append("The following config symbol(s) were not found: %s" % (", ".join(missing)))
error.append('The following config symbol(s) were not found: %s' % (', '.join(missing)))
# replace name keys with the full config symbol for each key:
to_set = dict((config.syms[k],v) for (k,v) in to_set.items() if k not in missing)
@@ -219,21 +220,21 @@ def handle_set(config, error, to_set):
elif val is False:
sym.set_value(0)
else:
error.append("Boolean symbol %s only accepts true/false values" % sym.name)
error.append('Boolean symbol %s only accepts true/false values' % sym.name)
elif sym.type == kconfiglib.HEX:
try:
if not isinstance(val, int):
val = int(val, 16) # input can be a decimal JSON value or a string of hex digits
sym.set_value(hex(val))
except ValueError:
error.append("Hex symbol %s can accept a decimal integer or a string of hex digits, only")
error.append('Hex symbol %s can accept a decimal integer or a string of hex digits, only')
else:
sym.set_value(str(val))
print("Set %s" % sym.name)
print('Set %s' % sym.name)
del to_set[sym]
if len(to_set):
error.append("The following config symbol(s) were not visible so were not updated: %s" % (", ".join(s.name for s in to_set)))
error.append('The following config symbol(s) were not visible so were not updated: %s' % (', '.join(s.name for s in to_set)))
def diff(before, after):
@@ -320,5 +321,5 @@ if __name__ == '__main__':
try:
main()
except FatalError as e:
print("A fatal error occurred: %s" % e, file=sys.stderr)
print('A fatal error occurred: %s' % e, file=sys.stderr)
sys.exit(2)

1
tools/kconfig_new/esp-windows-curses/setup.py
View File

@@ -13,6 +13,7 @@
# limitations under the License.
import os
from setuptools import setup
setup(name='esp-windows-curses',

22
tools/kconfig_new/gen_kconfig_doc.py
View File

@@ -21,7 +21,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import print_function
import re
import kconfiglib
# Indentation to be used in the generated file
@@ -152,7 +154,7 @@ def write_docs(config, visibility, filename):
""" Note: writing .rst documentation ignores the current value
of any items. ie the --config option can be ignored.
(However at time of writing it still needs to be set to something...) """
with open(filename, "w") as f:
with open(filename, 'w') as f:
for node in config.node_iter():
write_menu_item(f, node, visibility)
@@ -170,14 +172,14 @@ def get_breadcrumbs(node):
node = node.parent
while node.parent:
if node.prompt:
result = [":ref:`%s`" % get_link_anchor(node)] + result
result = [':ref:`%s`' % get_link_anchor(node)] + result
node = node.parent
return " > ".join(result)
return ' > '.join(result)
def get_link_anchor(node):
try:
return "CONFIG_%s" % node.item.name
return 'CONFIG_%s' % node.item.name
except AttributeError:
assert(node_is_menu(node)) # only menus should have no item.name
@@ -185,9 +187,9 @@ def get_link_anchor(node):
result = []
while node.parent:
if node.prompt:
result = [re.sub(r"[^a-zA-z0-9]+", "-", node.prompt[0])] + result
result = [re.sub(r'[^a-zA-z0-9]+', '-', node.prompt[0])] + result
node = node.parent
result = "-".join(result).lower()
result = '-'.join(result).lower()
return result
@@ -206,8 +208,8 @@ def format_rest_text(text, indent):
# Format an indented text block for use with ReST
text = indent + text.replace('\n', '\n' + indent)
# Escape some characters which are inline formatting in ReST
text = text.replace("*", "\\*")
text = text.replace("_", "\\_")
text = text.replace('*', '\\*')
text = text.replace('_', '\\_')
# replace absolute links to documentation by relative ones
text = re.sub(r'https://docs.espressif.com/projects/esp-idf/\w+/\w+/(.+)\.html', r':doc:`../\1`', text)
text += '\n'
@@ -297,7 +299,7 @@ def write_menu_item(f, node, visibility):
# if no symbol name, use the prompt as the heading
title = node.prompt[0]
f.write(".. _%s:\n\n" % get_link_anchor(node))
f.write('.. _%s:\n\n' % get_link_anchor(node))
f.write('%s\n' % title)
f.write(HEADING_SYMBOLS[get_heading_level(node)] * len(title))
f.write('\n\n')
@@ -389,7 +391,7 @@ def write_menu_item(f, node, visibility):
child_list.append((child.prompt[0], get_link_anchor(child)))
child = child.next
if len(child_list) > 0:
f.write("Contains:\n\n")
f.write('Contains:\n\n')
sorted_child_list = sorted(child_list, key=lambda pair: pair[0].lower())
ref_list = ['- :ref:`{}`'.format(anchor) for _, anchor in sorted_child_list]
f.write('\n'.join(ref_list))

10
tools/kconfig_new/prepare_kconfig_files.py
View File

@@ -14,12 +14,12 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import print_function
from __future__ import unicode_literals
from io import open
from __future__ import print_function, unicode_literals
import argparse
import json
import sys
from io import open
def _prepare_source_files(env_dict):
@@ -85,9 +85,9 @@ if __name__ == '__main__':
args = parser.parse_args()
try:
env = dict([(name, value) for (name, value) in (e.split("=", 1) for e in args.env)])
env = dict([(name, value) for (name, value) in (e.split('=', 1) for e in args.env)])
except ValueError:
print("--env arguments must each contain =.")
print('--env arguments must each contain =.')
sys.exit(1)
if args.env_file is not None:

5
tools/kconfig_new/test/confgen/test_confgen.py
View File

@@ -1,6 +1,5 @@
#!/usr/bin/env python
from future.utils import iteritems
import os
import re
import subprocess
@@ -9,6 +8,8 @@ import tempfile
import textwrap
import unittest
from future.utils import iteritems
class ConfgenBaseTestCase(unittest.TestCase):
@classmethod
@@ -270,5 +271,5 @@ hex "Hex Item default prefix"
default 0x77
"""
if __name__ == "__main__":
if __name__ == '__main__':
unittest.main()

95
tools/kconfig_new/test/confserver/test_confserver.py
View File

@@ -1,5 +1,6 @@
#!/usr/bin/env python
from __future__ import print_function
import argparse
import json
import os
@@ -13,24 +14,24 @@ PROTOCOL_VERSIONS = [1, 2]
def parse_testcases(version):
with open("testcases_v%d.txt" % version, "r") as f:
with open('testcases_v%d.txt' % version, 'r') as f:
cases = [line for line in f.readlines() if len(line.strip()) > 0]
# Each 3 lines in the file should be formatted as:
# * Description of the test change
# * JSON "changes" to send to the server
# * Result JSON to expect back from the server
if len(cases) % 3 != 0:
print("Warning: testcases.txt has wrong number of non-empty lines (%d). Should be 3 lines per test case, always." % len(cases))
print('Warning: testcases.txt has wrong number of non-empty lines (%d). Should be 3 lines per test case, always.' % len(cases))
for i in range(0, len(cases), 3):
desc = cases[i]
send = cases[i + 1]
expect = cases[i + 2]
if not desc.startswith("* "):
if not desc.startswith('* '):
raise RuntimeError("Unexpected description at line %d: '%s'" % (i + 1, desc))
if not send.startswith("> "):
if not send.startswith('> '):
raise RuntimeError("Unexpected send at line %d: '%s'" % (i + 2, send))
if not expect.startswith("< "):
if not expect.startswith('< '):
raise RuntimeError("Unexpected expect at line %d: '%s'" % (i + 3, expect))
desc = desc[2:]
send = json.loads(send[2:])
@@ -45,9 +46,9 @@ def main():
try:
# set up temporary file to use as sdkconfig copy
with tempfile.NamedTemporaryFile(mode="w", delete=False) as temp_sdkconfig:
with tempfile.NamedTemporaryFile(mode='w', delete=False) as temp_sdkconfig:
temp_sdkconfig_path = os.path.join(tempfile.gettempdir(), temp_sdkconfig.name)
with open("sdkconfig") as orig:
with open('sdkconfig') as orig:
temp_sdkconfig.write(orig.read())
with tempfile.NamedTemporaryFile(delete=False) as f:
@@ -69,12 +70,12 @@ def main():
# prepare_kconfig_files.py doesn't have to be called because COMPONENT_KCONFIGS and
# COMPONENT_KCONFIGS_PROJBUILD are empty
print("Running: %s" % cmdline)
print('Running: %s' % cmdline)
p = pexpect.spawn(cmdline, timeout=30, logfile=args.logfile, echo=False, use_poll=True, maxread=1)
p.expect("Server running.+\r\n")
p.expect('Server running.+\r\n')
initial = expect_json(p)
print("Initial: %s" % initial)
print('Initial: %s' % initial)
for version in PROTOCOL_VERSIONS:
test_protocol_version(p, version)
@@ -83,7 +84,7 @@ def main():
test_invalid_json(p)
print("Done. All passed.")
print('Done. All passed.')
finally:
try:
@@ -96,89 +97,89 @@ def main():
def expect_json(p):
# run p.expect() to expect a json object back, and return it as parsed JSON
p.expect("{.+}\r\n")
p.expect('{.+}\r\n')
result = p.match.group(0).strip().decode()
print("Read raw data from server: %s" % result)
print('Read raw data from server: %s' % result)
return json.loads(result)
def send_request(p, req):
req = json.dumps(req)
print("Sending: %s" % (req))
p.send("%s\n" % req)
print('Sending: %s' % (req))
p.send('%s\n' % req)
readback = expect_json(p)
print("Read back: %s" % (json.dumps(readback)))
print('Read back: %s' % (json.dumps(readback)))
return readback
def test_protocol_version(p, version):
print("*****")
print("Testing version %d..." % version)
print('*****')
print('Testing version %d...' % version)
# reload the config from the sdkconfig file
req = {"version": version, "load": None}
req = {'version': version, 'load': None}
readback = send_request(p, req)
print("Reset response: %s" % (json.dumps(readback)))
print('Reset response: %s' % (json.dumps(readback)))
# run through each test case
cases = parse_testcases(version)
for (desc, send, expected) in cases:
print(desc)
req = {"version": version, "set": send}
req = {'version': version, 'set': send}
readback = send_request(p, req)
if readback.get("version", None) != version:
if readback.get('version', None) != version:
raise RuntimeError('Expected {"version" : %d} in response' % version)
for expect_key in expected.keys():
read_vals = readback[expect_key]
exp_vals = expected[expect_key]
if read_vals != exp_vals:
expect_diff = dict((k,v) for (k,v) in exp_vals.items() if k not in read_vals or v != read_vals[k])
raise RuntimeError("Test failed! Was expecting %s: %s" % (expect_key, json.dumps(expect_diff)))
print("OK")
print("Version %d OK" % version)
raise RuntimeError('Test failed! Was expecting %s: %s' % (expect_key, json.dumps(expect_diff)))
print('OK')
print('Version %d OK' % version)
def test_load_save(p, temp_sdkconfig_path):
print("Testing load/save...")
print('Testing load/save...')
before = os.stat(temp_sdkconfig_path).st_mtime
save_result = send_request(p, {"version": 2, "save": temp_sdkconfig_path})
print("Save result: %s" % (json.dumps(save_result)))
assert "error" not in save_result
assert len(save_result["values"]) == 0 # nothing changes when we save
assert len(save_result["ranges"]) == 0
save_result = send_request(p, {'version': 2, 'save': temp_sdkconfig_path})
print('Save result: %s' % (json.dumps(save_result)))
assert 'error' not in save_result
assert len(save_result['values']) == 0 # nothing changes when we save
assert len(save_result['ranges']) == 0
after = os.stat(temp_sdkconfig_path).st_mtime
assert after > before # something got written to disk
# Do a V2 load
load_result = send_request(p, {"version": 2, "load": temp_sdkconfig_path})
print("V2 Load result: %s" % (json.dumps(load_result)))
assert "error" not in load_result
assert len(load_result["values"]) == 0 # in V2, loading same file should return no config items
assert len(load_result["ranges"]) == 0
load_result = send_request(p, {'version': 2, 'load': temp_sdkconfig_path})
print('V2 Load result: %s' % (json.dumps(load_result)))
assert 'error' not in load_result
assert len(load_result['values']) == 0 # in V2, loading same file should return no config items
assert len(load_result['ranges']) == 0
# Do a V1 load
load_result = send_request(p, {"version": 1, "load": temp_sdkconfig_path})
print("V1 Load result: %s" % (json.dumps(load_result)))
assert "error" not in load_result
assert len(load_result["values"]) > 0 # in V1, loading same file should return all config items
assert len(load_result["ranges"]) > 0
load_result = send_request(p, {'version': 1, 'load': temp_sdkconfig_path})
print('V1 Load result: %s' % (json.dumps(load_result)))
assert 'error' not in load_result
assert len(load_result['values']) > 0 # in V1, loading same file should return all config items
assert len(load_result['ranges']) > 0
def test_invalid_json(p):
print("Testing invalid JSON formatting...")
print('Testing invalid JSON formatting...')
bad_escaping = r'{ "version" : 2, "load" : "c:\some\path\not\escaped\as\json" }'
p.send("%s\n" % bad_escaping)
p.send('%s\n' % bad_escaping)
readback = expect_json(p)
print(readback)
assert "json" in readback["error"][0].lower()
assert 'json' in readback['error'][0].lower()
not_really_json = 'Hello world!!'
p.send("%s\n" % not_really_json)
p.send('%s\n' % not_really_json)
readback = expect_json(p)
print(readback)
assert "json" in readback["error"][0].lower()
assert 'json' in readback['error'][0].lower()
if __name__ == "__main__":
if __name__ == '__main__':
main()

3
tools/kconfig_new/test/gen_kconfig_doc/test_kconfig_out.py
View File

@@ -1,5 +1,6 @@
#!/usr/bin/env python
from __future__ import unicode_literals
import io
import os
import sys
@@ -76,5 +77,5 @@ class TestDocOutput(unittest.TestCase):
self.assertNotIn('- from 100', s)
if __name__ == "__main__":
if __name__ == '__main__':
unittest.main()

5
tools/kconfig_new/test/gen_kconfig_doc/test_target_visibility.py
View File

@@ -1,9 +1,10 @@
#!/usr/bin/env python
import kconfiglib
import os
import sys
import unittest
import kconfiglib
try:
import gen_kconfig_doc
except ImportError:
@@ -107,5 +108,5 @@ class ConfigTargetVisibilityChipB(ConfigTargetVisibilityTestCase):
self.invisible('CHIPA_FEATURE_FROM_V3')
if __name__ == "__main__":
if __name__ == '__main__':
unittest.main()

Some files were not shown because too many files have changed in this diff Show More