From 7090478b21b2d4ae4f8253934e31efee9a011bbd Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Fri, 30 Oct 2026 16:50:06 +0100 Subject: [PATCH 001/250] Import the 2.9 development line (Python 3.11+, NETCONF) The changes since 2.6.32 add an SSH NETCONF adapter based on ncclient, optional OpenTelemetry instrumentation, a reworked protocol lifecycle and many fixes. The paramiko 4.0 changes of master are included in equivalent form; CHANGES.md has the details. --- .gitignore | 11 +- .travis.yml | 8 +- CHANGES.md | 124 ++ Exscript/__init__.py | 14 +- Exscript/account.py | 20 +- Exscript/contrib/__init__.py | 25 + Exscript/contrib/otel.py | 360 +++++ Exscript/emulators/__init__.py | 6 +- Exscript/emulators/command.py | 52 +- Exscript/emulators/iosemu.py | 43 +- Exscript/emulators/vdevice.py | 38 +- Exscript/host.py | 50 +- Exscript/interpreter/__init__.py | 4 +- Exscript/interpreter/append.py | 19 +- Exscript/interpreter/assign.py | 23 +- Exscript/interpreter/code.py | 44 +- Exscript/interpreter/enter.py | 8 +- Exscript/interpreter/execute.py | 131 +- Exscript/interpreter/expression.py | 31 +- Exscript/interpreter/extract.py | 73 +- Exscript/interpreter/fail.py | 26 +- Exscript/interpreter/functioncall.py | 52 +- Exscript/interpreter/ifcondition.py | 16 +- Exscript/interpreter/loop.py | 45 +- Exscript/interpreter/number.py | 5 +- Exscript/interpreter/parser.py | 15 +- Exscript/interpreter/program.py | 4 +- Exscript/interpreter/regex.py | 48 +- Exscript/interpreter/rpc.py | 75 + Exscript/interpreter/scope.py | 17 +- Exscript/interpreter/string.py | 143 +- Exscript/interpreter/template.py | 108 +- Exscript/interpreter/term.py | 23 +- Exscript/interpreter/trying.py | 11 +- Exscript/interpreter/variable.py | 10 +- Exscript/key.py | 5 +- Exscript/logger.py | 128 +- Exscript/parselib/__init__.py | 4 +- Exscript/parselib/exception.py | 17 + Exscript/parselib/lexer.py | 139 +- Exscript/parselib/token.py | 39 +- Exscript/protocols/__init__.py | 24 +- Exscript/protocols/drivers/__init__.py | 73 +- Exscript/protocols/drivers/ace.py | 22 +- Exscript/protocols/drivers/adtran.py | 24 +- Exscript/protocols/drivers/aironet.py | 13 +- Exscript/protocols/drivers/aix.py | 7 +- Exscript/protocols/drivers/arbor_peakflow.py | 13 +- Exscript/protocols/drivers/aruba.py | 23 +- Exscript/protocols/drivers/bigip.py | 14 +- Exscript/protocols/drivers/brocade.py | 21 +- Exscript/protocols/drivers/cienasaos.py | 41 +- Exscript/protocols/drivers/driver.py | 125 +- Exscript/protocols/drivers/enterasys.py | 9 +- Exscript/protocols/drivers/enterasys_wc.py | 11 +- Exscript/protocols/drivers/eos.py | 43 +- Exscript/protocols/drivers/ericsson_ban.py | 6 +- Exscript/protocols/drivers/fortios.py | 21 +- Exscript/protocols/drivers/generic.py | 2 +- Exscript/protocols/drivers/hp_pro_curve.py | 26 +- Exscript/protocols/drivers/icotera.py | 29 +- Exscript/protocols/drivers/ios.py | 27 +- Exscript/protocols/drivers/ios_xr.py | 12 +- Exscript/protocols/drivers/isam.py | 18 +- Exscript/protocols/drivers/junos.py | 13 +- Exscript/protocols/drivers/junos_erx.py | 9 +- Exscript/protocols/drivers/mrv.py | 9 +- Exscript/protocols/drivers/nxos.py | 21 +- Exscript/protocols/drivers/one_os.py | 11 +- Exscript/protocols/drivers/rios.py | 17 +- Exscript/protocols/drivers/shell.py | 3 +- Exscript/protocols/drivers/smart_edge_os.py | 12 +- Exscript/protocols/drivers/sp_vxoa.py | 9 +- Exscript/protocols/drivers/sros.py | 8 +- Exscript/protocols/drivers/vrp.py | 16 +- Exscript/protocols/drivers/vxworks.py | 12 +- Exscript/protocols/drivers/zte.py | 18 +- Exscript/protocols/drivers/zyxel.py | 10 +- Exscript/protocols/dummy.py | 80 +- Exscript/protocols/exception.py | 60 + Exscript/protocols/netconf.py | 473 ++++++ Exscript/protocols/osguesser.py | 143 +- Exscript/protocols/protocol.py | 1342 ++++++++++++----- Exscript/protocols/ssh2.py | 339 +++-- Exscript/protocols/telnet.py | 119 +- Exscript/protocols/telnetlib.py | 71 +- Exscript/queue.py | 66 +- Exscript/servers/__init__.py | 8 +- Exscript/servers/httpd.py | 37 +- Exscript/servers/server.py | 82 +- Exscript/servers/sshd.py | 74 +- Exscript/servers/telnetd.py | 50 +- Exscript/stdlib/__init__.py | 100 +- Exscript/stdlib/connection.py | 105 +- Exscript/stdlib/crypt.py | 6 +- Exscript/stdlib/list.py | 55 +- Exscript/stdlib/mysys.py | 13 +- Exscript/stdlib/string.py | 26 +- Exscript/stdlib/util.py | 7 +- Exscript/util/buffer.py | 6 +- Exscript/util/cast.py | 8 +- Exscript/util/collections.py | 15 +- Exscript/util/crypt.py | 40 +- Exscript/util/decorator.py | 3 +- Exscript/util/event.py | 5 +- Exscript/util/file.py | 55 +- Exscript/util/impl.py | 13 +- Exscript/util/interact.py | 39 +- Exscript/util/ipv4.py | 6 +- Exscript/util/ipv6.py | 1 - Exscript/util/log.py | 4 +- Exscript/util/mail.py | 17 +- Exscript/util/match.py | 12 +- Exscript/util/pidutil.py | 18 +- Exscript/util/sigint.py | 36 +- Exscript/util/start.py | 3 +- Exscript/util/syslog.py | 13 +- Exscript/util/template.py | 95 +- Exscript/util/tty.py | 6 +- Exscript/util/url.py | 33 +- Exscript/util/weakmethod.py | 3 +- Exscript/util/xml.py | 63 + Exscript/workqueue/__init__.py | 8 +- Exscript/workqueue/job.py | 7 +- Exscript/workqueue/mainloop.py | 34 +- Exscript/workqueue/pipeline.py | 5 +- Exscript/workqueue/task.py | 3 +- Exscript/workqueue/workqueue.py | 35 +- Makefile | 2 +- README.md | 62 +- demos/multiple/accounts.cfg | 2 +- demos/multiple/multiple.py | 6 +- demos/quickstart/quickstart.py | 12 +- demos/report/accounts.cfg | 4 +- demos/report/report.py | 14 +- demos/simple/simple.py | 8 +- demos/template/template.py | 6 +- requirements.txt | 5 +- scripts/exscript | 859 ++++++----- scripts/otp | 28 +- setup.py | 34 +- tests/Exscript/AccountPoolTest.py | 6 +- tests/Exscript/HostTest.py | 25 +- tests/Exscript/LogTest.py | 9 +- tests/Exscript/LogfileTest.py | 10 +- tests/Exscript/LoggerTest.py | 13 +- tests/Exscript/PrivateKeyTest.py | 13 +- tests/Exscript/QueueTest.py | 56 +- tests/Exscript/TemplateTest.py | 35 +- tests/Exscript/account_pool.cfg | 6 +- tests/Exscript/emulators/IOSEmulatorTest.py | 11 +- tests/Exscript/emulators/VirtualDeviceTest.py | 2 +- tests/Exscript/emulators/run_suite.py | 90 +- tests/Exscript/memtest.py | 6 +- tests/Exscript/memtest_host.py | 6 +- tests/Exscript/protocols/DummyTest.py | 54 +- tests/Exscript/protocols/OsGuesserTest.py | 30 +- tests/Exscript/protocols/ProtocolTest.py | 422 +++++- tests/Exscript/protocols/SSH2Test.py | 16 +- tests/Exscript/protocols/TelnetTest.py | 16 +- tests/Exscript/protocols/run_suite.py | 90 +- tests/Exscript/run_suite.py | 90 +- tests/Exscript/servers/SSHdTest.py | 11 +- tests/Exscript/servers/ServerTest.py | 9 +- tests/Exscript/servers/TelnetdTest.py | 11 +- tests/Exscript/servers/run_suite.py | 90 +- tests/Exscript/util/bufferTest.py | 10 +- tests/Exscript/util/decoratorTest.py | 15 +- tests/Exscript/util/fileTest.py | 2 +- tests/Exscript/util/ipv4Test.py | 11 +- tests/Exscript/util/mailTest.py | 4 +- tests/Exscript/util/reportTest.py | 25 +- tests/Exscript/util/run_suite.py | 90 +- tests/Exscript/util/startTest.py | 2 +- tests/Exscript/util/syslogTest.py | 5 +- tests/Exscript/util/ttyTest.py | 9 +- tests/Exscript/util/urlTest.py | 46 +- tests/Exscript/util/weakmethodTest.py | 10 +- tests/Exscript/workqueue/JobTest.py | 14 +- tests/Exscript/workqueue/MainLoopTest.py | 10 +- tests/Exscript/workqueue/PipelineTest.py | 13 +- tests/Exscript/workqueue/WorkQueueTest.py | 12 +- tests/Exscript/workqueue/run_suite.py | 90 +- tests/templates/builtinvars/test.exscript | 2 +- tests/templates/iosdummy/expected | 70 +- tests/templates/iosdummy/test.exscript | 2 + tests/templates/netconf/expected | 12 + tests/templates/netconf/pseudodev.py | 15 + tests/templates/netconf/test.exscript | 11 + tox.ini | 9 +- version.sh | 82 +- 191 files changed, 6626 insertions(+), 2742 deletions(-) create mode 100644 CHANGES.md create mode 100644 Exscript/contrib/__init__.py create mode 100644 Exscript/contrib/otel.py create mode 100644 Exscript/interpreter/rpc.py create mode 100644 Exscript/protocols/netconf.py create mode 100644 Exscript/util/xml.py create mode 100644 tests/templates/netconf/expected create mode 100644 tests/templates/netconf/pseudodev.py create mode 100644 tests/templates/netconf/test.exscript diff --git a/.gitignore b/.gitignore index a2ca59bd..277334e6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,10 +1,13 @@ -*.py[co] +.coverage +.mypy_cache +.tox +.venv *.egg-info +*.py[co] *.swp *~ -config.xml build +config.xml dist +run_suite.log unit_test.cfg -.tox -.coverage diff --git a/.travis.yml b/.travis.yml index a20f1f64..852164ed 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,9 +1,9 @@ language: python python: - - "2.7" - - "3.4" - - "3.5" - - "3.6" + - "3.11" + - "3.12" + - "3.13" + - "3.14" # command to install dependencies install: - "pip install -r requirements.txt" diff --git a/CHANGES.md b/CHANGES.md new file mode 100644 index 00000000..cacbb1e9 --- /dev/null +++ b/CHANGES.md @@ -0,0 +1,124 @@ +# Changelog + +## 3.0.0 (unreleased) + +Work in progress: the entries below cover the changes merged from the 2.9 development +line and are completed against Exscript 2.6.32 before the release. Breaking changes are +marked with **BREAKING**; a migration guide will accompany the release. + +## General + +- **Python 2.7 Support:** +- **Python 3.11+ Support:** +- **Dependencies:** +- **NETCONF Protocol:** +- **OpenTelemetry Tracing (optional):** Install the `otel` extra (`Exscript[otel]`) + and use `Exscript.contrib.otel.ExscriptInstrumentor` to instrument `Protocol` and + `Netconf` operations. It creates spans for connect, login/authentication, send, + execute, and close, and records operation durations. Span attributes may include + host names, usernames, commands, and response contents. + +## Protocols + +- **BREAKING - Default Protocol:** The default protocol changed across all relevant + entry points/methods from `telnet` to `ssh` (version 2). This change was made to + improve security and performance. This was an explicit decision as SSH2 should be + considered the best-practice nowadays. The following entry points are affected: + + - `Exscript.Host(uri, default_protocol='ssh')` + - `Exscript.protocols.prepare(host, default_protocol='ssh', ...)` + - `Exscript.protocols.connect(host, default_protocol='ssh', ...)` + - `Exscript.util.cast.to_host(host, default_protocol='ssh', ...)` + - `Exscript.util.cast.to_hosts(hosts, default_protocol='ssh', ...)` + - `Exscript.util.file.get_hosts_from_csv(filename, default_protocol='ssh', ...)` + - `Exscript.util.file.get_hosts_from_file(filename, default_protocol='ssh', ...)` + - `Exscript.util.url.Url.from_string(url, default_protocol='ssh')` + - `exscript` CLI: `-p`/`--protocol` + +### Protocol Drivers + +- **BREAKING - Graceful Logout:** `Protocol.close(force=False)` now calls the driver's + new `logout()` method before the connection is closed. The base `Driver` sends + `exit\r`, the custom `VRPDriver` sends `quit\r` and `SROSDriver` sends `logout\r`. + Additional custom drivers may override `Driver.logout(conn)` to implement a + device-specific logout sequence. The `Netconf` and `Dummy` adapters do not use the + driver, they rely on a static logout sequence instead (a `` RPC + respectively a no-op). + + This may be only breaking for the Python API, templates did not need explicit logouts + before this. Note that the remote host must have left any submode (e.g. config mode, + local shell) before the connection is closed gracefully, otherwise the logout command + may fail. + +#### Class `Exscript.protocols.drivers.Driver` + +| Old | New | Comment | +| --- | --- | --- | +| - | `netconf_device_params` | Passed to ncclient's device handler. | +| - | `netconf_capabilities_re` | NETCONF capability driver detection pattern. | +| - | `netconf_prompt_re` | Response framing with EOM (`]]>]]>`) / EOC (`\n##\n`) pattern. | +| - | `netconf_error_re` | RPC error matching with `` pattern. | +| - | `__repr__()` | Returns a readable `DriverClass('name')` string. | +| - | `check_netconf_capabilities(string)` | Checks configured server-capability patterns. | + +### Class `Exscript.protocols.Protocol` + +| Old | New | Comment | +| --- | --- | --- | +| - | `cleanup_response(response, ...)` | Static method to cleanup a response. Used by the `get_response` method. | +| `close(force=False, reconnecting=False)` | `close(force=False)` | **BREAKING:** `reconnecting` removed. `force=False` now performs a graceful, driver-driven logout before closing. | +| - | `get_response(*, strip_command, strip_chars, strip_ansi)` | Should be the first choice to retrieve a clean response. `Protocol.response` still exists with the raw received data. | + +## Emulators + +### Class `Exscript.emulators.CommandSet` + +| Old | New | Comment | +| --- | --- | --- | +| `add(command, response)` | `add(command, handler, handler_decorator=None)` | **BREAKING:** `response` renamed to `handler` for keyword callers; optional decorator added. | +| Raises `Exception` | Raises `ValueError` | More specific exception from `add_from_file()` and strict `eval()` calls on missing/non-iterable commands. | + +### Class `Exscript.emulators.VirtualDevice` + +| Old | New | Comment | +| --- | --- | --- | +| `add_commands_from_file(..., autoprompt=True)` | `add_commands_from_file(..., prompt=True)` | **BREAKING:** Keyword renamed; callers using `autoprompt=` must update. | + +## Template Engine + +### Standard Library + +The names below refer to functions callable from templates, not protocol methods. + +| Old | New | Comment | +| --- | --- | --- | +| `__response__` | `__response__` | **BREAKING:** Connection helpers now clean responses using `get_response()`, stripping command echoes where applicable, control/ANSI codes, and normalizing CR/LF. | +| - | `connection.get_protocol()` | Returns the protocol name, e.g. `ssh`, `telnet`, or `netconf`. | +| - | `connection.expect(prompt)` | Consumes the response and matched prompt. | +| `connection.wait_for(prompt)` | `connection.wait_for(prompt)` | **BREAKING:** No longer consumes the response/prompt. Use `connection.expect()` for the previous behavior. | +| `connection.execline()` | `connection.execline()` | Uses `send()` and `expect_prompt()`; custom protocol `execute()` overrides are no longer called. | +| `connection.send()`, `connection.sendline()` | `connection.send()`, `connection.sendline()` | **BREAKING:** Clear the previous `__response__` after sending rather than leaving stale data. | +| `connection.close()` | `connection.close(force=False)` | **BREAKING:** Graceful logout by default instead of forced closure; pass `true` in templates to skip logout. | +| `list.new()` | `list.new(...)` | Accepts optional initial values. | +| - | `list.sort(source)` | Returns an ascending sorted copy without modifying the source. | +| - | `list.reverse(source)` | Returns a reversed copy without modifying the source. | +| - | `string.strip(strings, ...)` | Removes leading/trailing whitespace or specified characters. | +| `sys.exec()` | `sys.exec()` | **BREAKING:** Local command output is now text rather than bytes, decoded using Python's default subprocess encoding; line endings are retained. | + +## Multi-Threading + +## Demos + +- Updated the demo scripts to match the latest Exscript version. + +## Test Suite + +### Test Servers + +- **BREAKING - Threaded Servers:** The `Server` base class and its `SSHd`/`Telnetd` + implementations now run as daemon threads rather than processes. Process-specific + APIs are no longer available; use `exit()` and `join()` for shutdown. `HTTPd` is + unaffected by this change. +- **Client Handling:** `SSHd` and `Telnetd` keep listening after a client disconnects, + allowing sequential connections. They share the supplied emulator device; concurrent + clients are not supported. diff --git a/Exscript/__init__.py b/Exscript/__init__.py index 345dd7b9..8d621bc1 100644 --- a/Exscript/__init__.py +++ b/Exscript/__init__.py @@ -23,18 +23,14 @@ """ The core module. """ -from __future__ import absolute_import -import warnings -with warnings.catch_warnings(): - warnings.filterwarnings('ignore', category=DeprecationWarning) - import paramiko -from .version import __version__ +import inspect + from .account import Account, AccountPool +from .host import Host from .key import PrivateKey +from .logger import FileLogger, Logger from .queue import Queue -from .host import Host -from .logger import Logger, FileLogger +from .version import __version__ -import inspect __all__ = [name for name, obj in list(locals().items()) if not (name.startswith('_') or inspect.ismodule(obj))] diff --git a/Exscript/account.py b/Exscript/account.py index ae897c9a..73ce3d73 100644 --- a/Exscript/account.py +++ b/Exscript/account.py @@ -23,15 +23,15 @@ """ Manages user accounts. """ -from builtins import object import multiprocessing -from collections import deque, defaultdict +from collections import defaultdict, deque + from .util.cast import to_list from .util.event import Event from .util.impl import Context -class Account(object): +class Account: """ This class represents a user account. @@ -63,7 +63,7 @@ def __init__(self, :meth:`protocols.Protocol.login` and the `*authenticate*` methods. .. warning:: - Setting `lock` to True drastically degrades performance! + Setting `needs_lock` to True drastically degrades performance! :type name: str :param name: A username. @@ -106,7 +106,7 @@ def acquire(self, signal=True): """ Locks the account. Method has no effect if the constructor argument `needs_lock` - wsa set to False. + was set to False. :type signal: bool :param signal: Whether to emit the acquired_event signal. @@ -124,7 +124,7 @@ def release(self, signal=True): """ Unlocks the account. Method has no effect if the constructor argument `needs_lock` - wsa set to False. + was set to False. :type signal: bool :param signal: Whether to emit the released_event signal. @@ -204,7 +204,7 @@ def get_key(self): return self.key -class AccountProxy(object): +class AccountProxy: """ An object that has a 1:1 relation to an account object in another @@ -363,7 +363,7 @@ def get_key(self): return self.key -class LoggerProxy(object): +class LoggerProxy: """ An object that has a 1:1 relation to a Logger object in another @@ -397,7 +397,7 @@ def log_succeeded(self, job_id): self.parent.send(('log-succeeded', (self.logger_id, job_id))) -class AccountPool(object): +class AccountPool: """ This class manages a collection of available accounts. @@ -573,7 +573,7 @@ def release_accounts(self, owner): self.unlock_cond.notify_all() -class AccountManager(object): +class AccountManager: """ Keeps track of available user accounts and assigns them to the diff --git a/Exscript/contrib/__init__.py b/Exscript/contrib/__init__.py new file mode 100644 index 00000000..746da40e --- /dev/null +++ b/Exscript/contrib/__init__.py @@ -0,0 +1,25 @@ +# +# Copyright (C) 2010-2017 Samuel Abels +# The MIT License (MIT) +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files +# (the "Software"), to deal in the Software without restriction, +# including without limitation the rights to use, copy, modify, merge, +# publish, distribute, sublicense, and/or sell copies of the Software, +# and to permit persons to whom the Software is furnished to do so, +# subject to the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" +Optional contributed extensions for Exscript. +""" diff --git a/Exscript/contrib/otel.py b/Exscript/contrib/otel.py new file mode 100644 index 00000000..0ec693be --- /dev/null +++ b/Exscript/contrib/otel.py @@ -0,0 +1,360 @@ +# +# Copyright (C) 2010-2017 Samuel Abels +# The MIT License (MIT) +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files +# (the "Software"), to deal in the Software without restriction, +# including without limitation the rights to use, copy, modify, merge, +# publish, distribute, sublicense, and/or sell copies of the Software, +# and to permit persons to whom the Software is furnished to do so, +# subject to the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" +OpenTelemetry instrumentation for Exscript. + +Automatically creates spans and metrics for ``Protocol.connect``, +``login``, ``send``, ``execute`` and ``close`` (plus the same on +``Netconf``). + +Usage:: + + from Exscript.contrib.otel import ExscriptInstrumentor + + ExscriptInstrumentor().instrument() +""" +from __future__ import annotations + +import functools +import types +from collections.abc import Callable, Collection +from timeit import default_timer + +from opentelemetry.instrumentation.instrumentor import BaseInstrumentor +from opentelemetry.metrics import Histogram, get_meter +from opentelemetry.trace import SpanKind, Tracer, get_tracer, set_span_in_context +from opentelemetry.trace.span import Span + +from Exscript.protocols.protocol import Protocol + +# TODO (Netconf): Check whether Netconf is now similar enough to Protocol. +try: + from Exscript.protocols.netconf import Netconf +except ImportError: # ncclient may not be installed + Netconf = None # type: ignore[misc,assignment] + +from Exscript.version import __version__ as _exscript_version + +# --------------------------------------------------------------------------- +# Public type aliases +# --------------------------------------------------------------------------- +_RequestHookT = Callable[[Span, Protocol], None] | None +_ResponseHookT = Callable[[Span, Protocol], None] | None + +# Sentinel attribute set on patched methods +_APPLIED_ATTR = "opentelemetry_instrumentation_exscript_applied" +# Attribute stored on the Protocol *instance* by the connect wrapper +_CONNECT_SPAN_ATTR = "opentelemetry_instrumentation_exscript_connect_span" + +# Methods to instrument — same set for Protocol and Netconf +_INSTRUMENTED_METHODS = ("connect", "login", "authenticate", "close", "send", "execute") + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- +def _get_parent_span(proto: Protocol) -> Span | None: + """Return the span started by ``connect``, if any.""" + return getattr(proto, _CONNECT_SPAN_ATTR, None) + + +def _get_protocol_attributes(proto: Protocol) -> dict: + """Collect common span attributes from a Protocol instance.""" + attrs: dict = { + "protocol.type": proto.__class__.__name__, + "protocol.host": proto.host, + "protocol.port": proto.port, + "protocol.encoding": proto.encoding, + } + # These calls may fail before connect — be defensive + try: + attrs["protocol.timeout"] = proto.get_timeout() + except Exception: # noqa: BLE001, S110 + pass + try: + attrs["protocol.connect_timeout"] = proto.get_connect_timeout() + except Exception: # noqa: BLE001, S110 + pass + try: + attrs["protocol.os"] = proto.guess_os() + except Exception: # noqa: BLE001, S110 + pass + try: + prompt = proto.get_prompt() + attrs["protocol.prompt"] = [ + getattr(p, "pattern", str(p)) for p in prompt + ] if isinstance(prompt, (list, tuple)) else getattr( + prompt, "pattern", str(prompt) + ) + except Exception: # noqa: BLE001, S110 + pass + try: + err_prompt = proto.get_error_prompt() + attrs["protocol.error_prompt"] = [ + getattr(p, "pattern", str(p)) for p in err_prompt + ] if isinstance(err_prompt, (list, tuple)) else getattr( + err_prompt, "pattern", str(err_prompt) + ) + except Exception: # noqa: BLE001, S110 + pass + try: + attrs["protocol.protocol_authenticated"] = proto.is_protocol_authenticated() + except Exception: # noqa: BLE001, S110 + pass + try: + attrs["protocol.app_authenticated"] = proto.is_app_authenticated() + except Exception: # noqa: BLE001, S110 + pass + try: + attrs["protocol.app_authorized"] = proto.is_app_authorized() + except Exception: # noqa: BLE001, S110 + pass + return attrs + + +def _span_name(protocol_name: str, action: str) -> str: + return f"{protocol_name.strip()} {action.strip()}" + + +# --------------------------------------------------------------------------- +# Generic wrapper factory +# --------------------------------------------------------------------------- +def _make_wrapper( + original, + action: str, + tracer: Tracer, + duration_histogram: Histogram, + request_hook: _RequestHookT, + response_hook: _ResponseHookT, +): + """Return an instrumented replacement for *original* method.""" + + @functools.wraps(original) + def wrapper(self, *args, **kwargs): + protocol_name = self.__class__.__name__ + name = _span_name(protocol_name, action) + + # --- span attributes --- + span_attributes: dict = {"protocol.type": protocol_name} + span_attributes.update(_get_protocol_attributes(self)) + + # action-specific attributes + if action == "connect": + hostname = kwargs.get("hostname") or (args[0] if args else None) + port = kwargs.get("port") or (args[1] if len(args) > 1 else None) + if hostname: + span_attributes["protocol.host"] = hostname + if port: + span_attributes["protocol.port"] = port + elif action in ("login", "authenticate"): + account = kwargs.get("account") or (args[0] if args else None) + app_account = kwargs.get("app_account") or (args[1] if args else None) + if account is not None: + try: + span_attributes["protocol.username"] = account.get_name() + except Exception: # noqa: BLE001, S110 + pass + if app_account is not None: + try: + span_attributes["protocol.app_username"] = app_account.get_name() + except Exception: # noqa: BLE001, S110 + pass + elif action == "close": + force = kwargs.get("force", False) or (args[0] if args else False) + span_attributes["protocol.force_close"] = force + elif action in ("send", "execute"): + data = kwargs.get("data") + command = kwargs.get("command", data) or (args[0] if args else None) + if command is not None: + span_attributes["protocol.command"] = command + + metric_labels = {"protocol.type": protocol_name} + + # --- connect uses a long-lived span (ended in close) --- + if action == "connect": + span = tracer.start_span( + name, kind=SpanKind.CLIENT, attributes=span_attributes + ) + exception = None + if callable(request_hook): + request_hook(span, self) + start_time = default_timer() + try: + result = original(self, *args, **kwargs) + except Exception as exc: # noqa: BLE001 + exception = exc + result = getattr(exc, "response", None) + finally: + elapsed = max(round((default_timer() - start_time) * 1000), 0) + + if callable(response_hook): + response_hook(span, self, result) + duration_histogram.record(elapsed, attributes=metric_labels) + + if exception is not None: + raise exception.with_traceback(exception.__traceback__) + + # Store so child spans can use it as parent + setattr(self, _CONNECT_SPAN_ATTR, span) + return result + + # --- all other actions use a child span scoped to the call --- + parent = _get_parent_span(self) + ctx = set_span_in_context(parent) if parent else None + + span_kwargs: dict = { + "kind": SpanKind.CLIENT, + "attributes": span_attributes, + } + if ctx: + span_kwargs["context"] = ctx + + with tracer.start_as_current_span(name, **span_kwargs) as span: + exception = None + if callable(request_hook): + request_hook(span, self) + start_time = default_timer() + try: + result = original(self, *args, **kwargs) + except Exception as exc: # noqa: BLE001 + exception = exc + result = getattr(exc, "response", None) + finally: + elapsed = max(round((default_timer() - start_time) * 1000), 0) + + # Capture response on execute, send does not update response by design! + if action == "execute": + try: + span.set_attribute("protocol.response", str(self.response)) + except Exception: # noqa: BLE001, S110 + pass + + if callable(response_hook): + response_hook(span, self, result) + duration_histogram.record(elapsed, attributes=metric_labels) + + if exception is not None: + raise exception.with_traceback(exception.__traceback__) + + # End the connect span when close is called + if action == "close" and parent is not None: + try: + parent.end() + except Exception: # noqa: BLE001, S110 + pass + + return result + + setattr(wrapper, _APPLIED_ATTR, True) + return wrapper + + +# --------------------------------------------------------------------------- +# Patch / un-patch logic +# --------------------------------------------------------------------------- +def _instrument_class( + cls, + tracer: Tracer, + duration_histogram: Histogram, + request_hook: _RequestHookT, + response_hook: _ResponseHookT, +): + """Monkey-patch all target methods on *cls*.""" + for method_name in _INSTRUMENTED_METHODS: + original = getattr(cls, method_name, None) + if original is None: + continue + if getattr(original, _APPLIED_ATTR, False): + continue # already instrumented + wrapped = _make_wrapper( + original, method_name, tracer, duration_histogram, + request_hook, response_hook, + ) + setattr(cls, method_name, wrapped) + + +def _uninstrument_class(cls): + """Restore original methods on *cls*.""" + for method_name in _INSTRUMENTED_METHODS: + func = getattr(cls, method_name, None) + if func is None or not getattr(func, _APPLIED_ATTR, False): + continue + setattr(cls, method_name, func.__wrapped__) + + +# --------------------------------------------------------------------------- +# Public instrumentor +# --------------------------------------------------------------------------- +class ExscriptInstrumentor(BaseInstrumentor): + """OpenTelemetry instrumentor for Exscript Protocol & Netconf. + + Usage:: + + ExscriptInstrumentor().instrument() + # optional hooks: + ExscriptInstrumentor().instrument( + request_hook=my_request_hook, + response_hook=my_response_hook, + ) + """ + + def instrumentation_dependencies(self) -> Collection[str]: + return ("exscript",) + + def _instrument(self, **kwargs): + tracer_provider = kwargs.get("tracer_provider") + tracer = get_tracer( + __name__, _exscript_version, tracer_provider, + ) + meter_provider = kwargs.get("meter_provider") + meter = get_meter(__name__, _exscript_version, meter_provider) + duration_histogram = meter.create_histogram( + name="exscript.client.duration", + unit="ms", + description="Duration of Exscript protocol operations", + ) + request_hook = kwargs.get("request_hook") + response_hook = kwargs.get("response_hook") + + _instrument_class( + Protocol, tracer, duration_histogram, request_hook, response_hook, + ) + if Netconf is not None: + _instrument_class( + Netconf, tracer, duration_histogram, request_hook, response_hook, + ) + + def _uninstrument(self, **kwargs): + _uninstrument_class(Protocol) + if Netconf is not None: + _uninstrument_class(Netconf) + + @staticmethod + def uninstrument_protocol(protocol): + """Disable instrumentation on a specific *protocol* instance.""" + for method_name in _INSTRUMENTED_METHODS: + func = getattr(protocol, method_name, None) + if func is None or not getattr(func, _APPLIED_ATTR, False): + continue + original = types.MethodType(func.__wrapped__, protocol) + setattr(protocol, method_name, original) diff --git a/Exscript/emulators/__init__.py b/Exscript/emulators/__init__.py index dda7b9bd..248535b5 100644 --- a/Exscript/emulators/__init__.py +++ b/Exscript/emulators/__init__.py @@ -23,11 +23,11 @@ """ Emulating a device for testing your scripts. """ -from __future__ import absolute_import +import inspect + from .command import CommandSet -from .vdevice import VirtualDevice from .iosemu import IOSEmulator +from .vdevice import VirtualDevice -import inspect __all__ = [name for name, obj in list(locals().items()) if not (name.startswith('_') or inspect.ismodule(obj))] diff --git a/Exscript/emulators/command.py b/Exscript/emulators/command.py index c4543983..c75035b4 100644 --- a/Exscript/emulators/command.py +++ b/Exscript/emulators/command.py @@ -23,13 +23,11 @@ """ Defines the behavior of commands by mapping commands to functions. """ -from past.builtins import execfile -from builtins import str -from builtins import object import re +from collections.abc import Iterable -class CommandSet(object): +class CommandSet: """ A set of commands to be used by the Dummy adapter. @@ -49,9 +47,9 @@ def __deepcopy__(self, memo): cmdset.response_list.append((regex, response)) return cmdset - def add(self, command, response): + def add(self, command, handler, handler_decorator=None): """ - Register a command/response pair. + Register a command and response handler pair. The command may be either a string (which is then automatically compiled into a regular expression), or a pre-compiled regular @@ -64,36 +62,45 @@ def add(self, command, response): :type command: str|regex :param command: A string or a compiled regular expression. - :type response: function|str - :param response: A response, or a response handler. + :type handler: function|str + :param handler: A response, or a response handler. + :type handler_decorator: function + :param handler_decorator: A function that is used to decorate + the response or response handler. """ command = re.compile(command) - self.response_list.append((command, response)) + if handler_decorator: + handler = handler_decorator(handler) + self.response_list.append((command, handler)) def add_from_file(self, filename, handler_decorator=None): """ - Wrapper around add() that reads the handlers from the + Wrapper around add() that reads the response handlers from the file with the given name. The file is a Python script containing a list named 'commands' of tuples that map command names to - handlers. + response handlers. :type filename: str :param filename: The name of the file containing the tuples. :type handler_decorator: function :param handler_decorator: A function that is used to decorate - each of the handlers in the file. + each of the responses or response handlers in the file. + :raises ValueError: Raised when the file does not define a variable + named "commands" or if "commands" is not iterable. """ args = {} - execfile(filename, args) + with open(filename, "r", encoding="utf-8") as fin: + source = fin.read() + code = compile(source, filename, mode="exec") + exec(code, args) # noqa: S102 + commands = args.get('commands') if commands is None: - raise Exception(filename + ' has no variable named "commands"') - elif not hasattr(commands, '__iter__'): - raise Exception(filename + ': "commands" is not iterable') - for key, handler in commands: - if handler_decorator: - handler = handler_decorator(handler) - self.add(key, handler) + raise ValueError(filename + ': no variable named "commands"') + elif not isinstance(commands, Iterable): + raise ValueError(filename + ': variable "commands" is not iterable') + for command, response in commands: + self.add(command, response, handler_decorator) def eval(self, command): """ @@ -104,16 +111,17 @@ def eval(self, command): :param command: The command that is evaluated. :rtype: str or None :return: The response, if one was defined. + :raises ValueError: Raised when the command is undefined (strict mode only). """ for cmd, response in self.response_list: if not cmd.match(command): continue if response is None: return None - elif hasattr(response, '__call__'): + elif callable(response): return response(command) else: return response if self.strict: - raise Exception('Undefined command: ' + repr(command)) + raise ValueError('Undefined command: ' + repr(command)) return None diff --git a/Exscript/emulators/iosemu.py b/Exscript/emulators/iosemu.py index f6438efa..70d2ad30 100644 --- a/Exscript/emulators/iosemu.py +++ b/Exscript/emulators/iosemu.py @@ -1,7 +1,7 @@ -# +# # Copyright (C) 2010-2017 Samuel Abels # The MIT License (MIT) -# +# # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files # (the "Software"), to deal in the Software without restriction, @@ -9,10 +9,10 @@ # publish, distribute, sublicense, and/or sell copies of the Software, # and to permit persons to whom the Software is furnished to do so, # subject to the following conditions: -# +# # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. -# +# # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. @@ -21,12 +21,11 @@ # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ -Cisco IOS emulator. +Cisco IOS emulator used for internal testing. """ -from __future__ import unicode_literals -#from builtins import str import re -from Exscript.emulators import VirtualDevice + +from .vdevice import VirtualDevice iosbanner = ''' Connected to %s. @@ -36,7 +35,7 @@ Unauthorized access prohibited! -%s line 2 +%s line 2 at '%s' port '/dev/vty0' from '12.34.56.78' @@ -62,7 +61,7 @@ def show_diag(data): ROM Monitor version 18.0 Fabric Downloader version used 7.1 (ROM version is 7.1) Primary clock is CSC 1 - Board is analyzed + Board is analyzed Board State is Line Card Enabled (IOS RUN ) Insertion time: 00:00:30 (36w1d ago) Processor Memory size: 1073741824 bytes @@ -73,7 +72,7 @@ def show_diag(data): commands = ( ('show version', """ -Cisco Internetwork Operating System Software +Cisco Internetwork Operating System Software IOS (tm) GS Software (C12KPRP-P-M), Version 12.0(32)SY6c, RELEASE SOFTWARE (fc3) Technical Support: http://www.cisco.com/techsupport Copyright (c) 1986-2008 by cisco Systems, Inc. @@ -83,7 +82,7 @@ def show_diag(data): ROM: System Bootstrap, Version 12.0(20040128:214555) [assafb-PRP1P_20040101 1.8dev(2.83)] DEVELOPMENT SOFTWARE BOOTLDR: GS Software (C12KPRP-P-M), Version 12.0(32)SY6c, RELEASE SOFTWARE (fc3) - S-EA1 uptime is 36 weeks, 1 day, 15 hours, 9 minutes + dummy uptime is 36 weeks, 1 day, 15 hours, 9 minutes Uptime for this control processor is 36 weeks, 1 day, 14 hours, 30 minutes System returned to ROM by reload at 03:32:54 MET Mon Feb 16 2009 System restarted at 03:25:22 MET Tue Mar 10 2009 @@ -116,21 +115,21 @@ def show_diag(data): 250880K bytes of ATA PCMCIA card at slot 0 (Sector size 512 bytes). 65536K bytes of Flash internal SIMM (Sector size 256K). Configuration register is 0x2102 -""".lstrip()), +"""), (r'sh\S* ip int\S* brie\S*', """ Interface IP-Address OK? Method Status Protocol -Ethernet0 10.108.00.5 YES NVRAM up up -Ethernet1 unassigned YES unset administratively down down -Loopback0 10.108.200.5 YES NVRAM up up -Serial0 10.108.100.5 YES NVRAM up up -Serial1 10.108.40.5 YES NVRAM up up -Serial2 10.108.100.5 YES manual up up -Serial3 unassigned YES unset administratively down down -""".lstrip()), +Ethernet0 10.108.00.5 YES NVRAM up up +Ethernet1 unassigned YES unset administratively down down +Loopback0 10.108.200.5 YES NVRAM up up +Serial0 10.108.100.5 YES NVRAM up up +Serial1 10.108.40.5 YES NVRAM up up +Serial2 10.108.100.5 YES manual up up +Serial3 unassigned YES unset administratively down down +"""), ('show interface.*', """ -FastEthernet0/2 is administratively down, line protocol is down +FastEthernet0/2 is administratively down, line protocol is down Hardware is i82545, address is 0001.c9f4.c418 (bia 0001.c9f4.c418) MTU 1500 bytes, BW 100000 Kbit, DLY 100 usec, rely 255/255, load 1/255 Encapsulation ARPA, loopback not set diff --git a/Exscript/emulators/vdevice.py b/Exscript/emulators/vdevice.py index 96fa5fad..918b8d10 100644 --- a/Exscript/emulators/vdevice.py +++ b/Exscript/emulators/vdevice.py @@ -21,16 +21,12 @@ # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ -Defines the behavior of a device, as needed by :class:`Exscript.servers`. +Defines the behavior of a virtual device, as needed by :class:`Exscript.servers`. """ -from __future__ import absolute_import, unicode_literals -from builtins import range -from builtins import object -from builtins import str from . import CommandSet -class VirtualDevice(object): +class VirtualDevice: """ An object that emulates a remote device. @@ -53,7 +49,6 @@ def __init__(self, """ :type hostname: str :param hostname: The hostname, used for the prompt. - :param echo: bool :keyword echo: whether to echo the command in a response. :param login_type: int @@ -89,10 +84,10 @@ def _get_prompt(self): self.logged_in = True return self.prompt else: - raise Exception('invalid prompt stage') + raise RuntimeError('invalid prompt stage') def _create_autoprompt_handler(self, handler): - if hasattr(handler, '__call__'): + if callable(handler): return lambda x: handler(x) + '\n' + self._get_prompt() else: return lambda x: handler + '\n' + self._get_prompt() @@ -133,30 +128,24 @@ def add_command(self, command, handler, prompt=True): :type handler: function|str :param handler: A string, or a response handler. :type prompt: bool - :param prompt: Whether to show a prompt after completing the command. + :param prompt: Whether to append a prompt to the response. """ - if prompt: - thehandler = self._create_autoprompt_handler(handler) - else: - thehandler = handler - self.commands.add(command, thehandler) + deco = self._create_autoprompt_handler if prompt else None + self.commands.add(command, handler, deco) - def add_commands_from_file(self, filename, autoprompt=True): + def add_commands_from_file(self, filename, prompt=True): """ - Wrapper around add_command_handler that reads the handlers from the + Wrapper around add_command that reads the response handlers from the file with the given name. The file is a Python script containing a list named 'commands' of tuples that map command names to - handlers. + response handlers. :type filename: str :param filename: The name of the file containing the tuples. - :type autoprompt: bool - :param autoprompt: Whether to append a prompt to each response. + :type prompt: bool + :param prompt: Whether to append a prompt to each response. """ - if autoprompt: - deco = self._create_autoprompt_handler - else: - deco = None + deco = self._create_autoprompt_handler if prompt else None self.commands.add_from_file(filename, deco) def init(self): @@ -194,4 +183,5 @@ def do(self, command): response = self.commands.eval(command) if response is None: return echo + '\n' + self._get_prompt() + # Response or its handler was wrapped to add the prompt automatically! return echo + response diff --git a/Exscript/host.py b/Exscript/host.py index dfb487c1..29367806 100644 --- a/Exscript/host.py +++ b/Exscript/host.py @@ -23,21 +23,13 @@ """ Representing a device to connect with. """ -from __future__ import absolute_import -from builtins import str -from builtins import object from .account import Account from .util.cast import to_list -from .util.ipv4 import is_ip, clean_ip +from .util.ipv4 import clean_ip, is_ip from .util.url import Url -def _is_ip(string): - # Adds IPv6 support. - return ':' in string or is_ip(string) - - -class Host(object): +class Host: """ Represents a device on which to open a connection. @@ -50,7 +42,7 @@ class Host(object): 'tcp_port', 'options') - def __init__(self, uri, default_protocol='telnet'): + def __init__(self, uri, default_protocol='ssh'): """ Constructor. The given uri is passed to Host.set_uri(). The default_protocol argument defines the protocol that is used @@ -206,9 +198,8 @@ def set_protocol(self, protocol): supported: - telnet: Telnet - - ssh1: SSH version 1 + - ssh: SSH version 2 - ssh2: SSH version 2 - - ssh: Automatically selects the best supported SSH version - dummy: A virtual device that accepts any command, but that does not respond anything useful. - pseudo: A virtual device that loads a file with the given @@ -216,7 +207,7 @@ def set_protocol(self, protocol): information on how the virtual device shall respond to commands. For more information please refer to the documentation of - protocols.Dummy.load_command_handler_from_file(). + emulators.VirtualDevice.add_commands_from_file(). :type protocol: string :param protocol: The protocol name. @@ -237,15 +228,40 @@ def set_option(self, name, value): Defines a (possibly protocol-specific) option for the host. Possible options include: - verify_fingerprint: bool + banner_timeout: int # SSH2 only + connect_rate_limit: int + connect_timeout: int + debug: int + disabled_algorithms: dict[str, list[str]] # SSH2 only + driver: Driver()|str + encoding: str + preferred_authentications: list[str]|None # SSH2 only + termtype: str + timeout: int + verify_fingerprint: bool # SSH2 only + + Please refer to the Protocol constructor for detailed descriptions. :type name: str :param name: The option name. :type value: object :param value: The option value. """ - if name not in ('debug', 'verify_fingerprint', 'driver'): - raise TypeError('No such option: ' + repr(name)) + valid_options = ( + 'banner_timeout', + 'connect_rate_limit', + 'connect_timeout', + 'debug', + 'disabled_algorithms', + 'driver', + 'encoding', + 'preferred_authentications', + 'termtype', + 'timeout', + 'verify_fingerprint', + ) + if name not in valid_options: + raise TypeError('Invalid option: ' + repr(name)) if self.options is None: self.options = {} self.options[name] = value diff --git a/Exscript/interpreter/__init__.py b/Exscript/interpreter/__init__.py index aa441084..b64a7bc0 100644 --- a/Exscript/interpreter/__init__.py +++ b/Exscript/interpreter/__init__.py @@ -20,9 +20,9 @@ # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -from __future__ import absolute_import +import inspect + from .parser import Parser -import inspect __all__ = [name for name, obj in list(locals().items()) if not (name.startswith('_') or inspect.ismodule(obj))] diff --git a/Exscript/interpreter/append.py b/Exscript/interpreter/append.py index 3e2808ea..b608b771 100644 --- a/Exscript/interpreter/append.py +++ b/Exscript/interpreter/append.py @@ -20,23 +20,29 @@ # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -from __future__ import print_function, absolute_import from ..parselib import Token -from .term import Term +from .expression import Expression class Append(Token): def __init__(self, lexer, parser, parent): - Token.__init__(self, 'Append', lexer, parser, parent) + Token.__init__(self, self.__class__.__name__, lexer, parser, parent) # First expect an expression. lexer.expect(self, 'keyword', 'append') lexer.expect(self, 'whitespace') - self.expr = Term(lexer, parser, parent) + # The later expression token skips all whitespaces and newlines, so we must + # fail on newlines here to keep the first expression term on the first line. + if lexer.current_is('newline'): + ttype, token = lexer.token() + self.mark_start() + self.mark_end(distance=len(token)) + msg = 'Expected expression but got %s %s' % (ttype, repr(token)) + lexer.syntax_error(msg, self) + self.expr = Expression(lexer, parser, parent) # Expect "to" keyword. - lexer.expect(self, 'whitespace') lexer.expect(self, 'keyword', 'to') # Expect a variable name. @@ -45,7 +51,8 @@ def __init__(self, lexer, parser, parent): lexer.expect(self, 'varname') self.parent.define(**{self.varname: []}) - self.mark_end() + # End of statement. + self.mark_end(rstrip=True) def value(self, context): existing = self.parent.get(self.varname) diff --git a/Exscript/interpreter/assign.py b/Exscript/interpreter/assign.py index 8539a352..e63f90b3 100644 --- a/Exscript/interpreter/assign.py +++ b/Exscript/interpreter/assign.py @@ -20,7 +20,6 @@ # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -from __future__ import print_function, absolute_import from ..parselib import Token from .expression import Expression @@ -28,7 +27,7 @@ class Assign(Token): def __init__(self, lexer, parser, parent): - Token.__init__(self, 'Assign', lexer, parser, parent) + Token.__init__(self, self.__class__.__name__, lexer, parser, parent) # Extract the variable name. _, self.varname = lexer.token() @@ -37,17 +36,31 @@ def __init__(self, lexer, parser, parent): lexer.expect(self, 'assign') lexer.expect(self, 'whitespace') - if self.varname.startswith('__'): - msg = 'Assignment to internal variable ' + self.varname + # The later expression token skips all whitespaces and newlines, so we must + # fail on newlines here to keep the first expression term on the first line. + if lexer.current_is('newline'): + ttype, token = lexer.token() + self.mark_start() + self.mark_end(distance=len(token)) + msg = 'Expected expression but got %s %s' % (ttype, repr(token)) + lexer.syntax_error(msg, self) + + if self.varname.startswith('__') and self.varname.endswith('__'): + # The start mark still points at the beginning of the varname. + self.mark_end(distance=len(self.varname)) + msg = 'Assignment to internal variable %s' % repr(self.varname) lexer.syntax_error(msg, self) self.expression = Expression(lexer, parser, parent) self.parent.define(**{self.varname: None}) + # End of statement. + self.mark_end(rstrip=True) + def dump(self, indent=0): print((' ' * indent) + self.name, self.varname, 'start') self.expression.dump(indent + 1) - print((' ' * indent) + self.name, self.varname, 'start') + print((' ' * indent) + self.name, self.varname, 'end') def value(self, context): result = self.expression.value(context) diff --git a/Exscript/interpreter/code.py b/Exscript/interpreter/code.py index 8414cd2b..7475c7b3 100644 --- a/Exscript/interpreter/code.py +++ b/Exscript/interpreter/code.py @@ -20,10 +20,10 @@ # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -from __future__ import absolute_import import re + import Exscript.interpreter.template -from .scope import Scope + from .append import Append from .assign import Assign from .enter import Enter @@ -32,8 +32,14 @@ from .functioncall import FunctionCall from .ifcondition import IfCondition from .loop import Loop -from .trying import Try +from .scope import Scope from .string import varname_re +from .trying import Try + +# IDEA: Add support for 'None' in the template engine? +# Right now a regex group that does not capture anything might inject a none +# value into a variable. We would need to test this (e.g. in addition) and then +# check if there is some kind of None handling/comparison. If not, we should add it. varname = varname_re.pattern keywords = ['append', @@ -66,6 +72,7 @@ grammar = ( ('escaped_data', r'\\.'), + ('line_comment', r'(?:^|(?<=[\r\n{]))[ \t]*#[^\r\n]*(?:[\r\n]|$)'), ('regex_delimiter', r'/'), ('string_delimiter', r'"'), ('open_curly_bracket', r'{'), @@ -74,15 +81,15 @@ ('comma', r','), ('whitespace', r'[ \t]+'), ('keyword', r'\b(?:' + '|'.join(keywords) + r')\b'), - ('assign', r'='), - ('octal_number', r'0\d+'), - ('hex_number', r'0x(?:\w\w)+'), ('comparison', r'\b(?:' + '|'.join(operators) + r')\b'), ('arithmetic_operator', r'(?:\*|\+|-|%|\.)'), ('logical_operator', r'\b(?:and|or|not)\b'), + ('assign', r'='), + ('hex_number', r'\b0x[0-9a-fA-F]+\b'), + ('octal_number', r'\b0o?[0-7]+\b'), + ('number', r'\b\d+\b'), ('open_function_call', varname + r'(?:\.' + varname + r')*\('), ('varname', varname), - ('number', r'\d+'), ('newline', r'[\r\n]'), ('raw_data', r'[^\r\n{}]+') ) @@ -95,15 +102,15 @@ class Code(Scope): def __init__(self, lexer, parser, parent): - Scope.__init__(self, 'Code', lexer, parser, parent) + Scope.__init__(self, self.__class__.__name__, lexer, parser, parent) lexer.set_grammar(grammar_c) + lexer.next_if('open_curly_bracket') while True: - lexer.skip(['whitespace', 'newline']) + lexer.skip(['line_comment', 'whitespace', 'newline']) if lexer.next_if('close_curly_bracket'): if isinstance(parent, Exscript.interpreter.template.Template): break - self.add( - Exscript.interpreter.template.Template(lexer, parser, self)) + self.add(Exscript.interpreter.template.Template(lexer, parser, self)) elif lexer.current_is('keyword', 'append'): self.add(Append(lexer, parser, self)) elif lexer.current_is('keyword', 'extract'): @@ -122,15 +129,24 @@ def __init__(self, lexer, parser, parent): self.add(Enter(lexer, parser, self)) elif lexer.current_is('keyword', 'else'): if not isinstance(parent, Code): - lexer.syntax_error('"end" without a scope start', self) + self.mark_start() + self.mark_end(distance=len('else')) + lexer.syntax_error("Unexpected 'else' without a scope", self) break elif lexer.next_if('keyword', 'end'): if not isinstance(parent, Code): - lexer.syntax_error('"end" without a scope start', self) + self.mark_start(lexer.last_char) + self.mark_end(distance=len('end')) + lexer.syntax_error("Unexpected 'end' without a scope", self) lexer.skip(['whitespace', 'newline']) break elif lexer.current_is('open_function_call'): self.add(FunctionCall(lexer, parser, self)) else: - lexer.syntax_error('Unexpected %s "%s"' % lexer.token(), self) + ttype, token = lexer.token() + self.mark_start() + self.mark_end(distance=len(token)) + lexer.syntax_error('Unexpected %s %s' % (ttype, repr(token)), self) lexer.restore_grammar() + # End of statement. + self.mark_end(rstrip=True) diff --git a/Exscript/interpreter/enter.py b/Exscript/interpreter/enter.py index 90664f35..c5f48fcb 100644 --- a/Exscript/interpreter/enter.py +++ b/Exscript/interpreter/enter.py @@ -20,7 +20,6 @@ # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -from __future__ import print_function, absolute_import from ..parselib import Token from .execute import Execute @@ -28,12 +27,11 @@ class Enter(Token): def __init__(self, lexer, parser, parent): - Token.__init__(self, 'Enter', lexer, parser, parent) - + Token.__init__(self, self.__class__.__name__, lexer, parser, parent) lexer.expect(self, 'keyword', 'enter') - lexer.skip(['whitespace', 'newline']) - self.execute = Execute(lexer, parser, parent, '') + # End of statement. + self.mark_end(rstrip=True) def value(self, context): return self.execute.value(context) diff --git a/Exscript/interpreter/execute.py b/Exscript/interpreter/execute.py index a8de0cc8..f9237a43 100644 --- a/Exscript/interpreter/execute.py +++ b/Exscript/interpreter/execute.py @@ -20,54 +20,133 @@ # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -from __future__ import print_function, absolute_import +import re +import textwrap + +from lxml.etree import XMLSyntaxError + from ..parselib import Token -from .string import String, string_re +from ..util.xml import validate_xml +from .string import String, varstring_re + +grammar = { + 'leading_blank_lines': r'^(?:[ \t]*[\r\n])+', + 'trailing_blank_lines': r'(?:[\r\n][ \t]*)+$', + 'line_comment': r'(?:^|(?<=[\r\n]))[ \t]*#[^\r\n]*(?:[\r\n]|$)', +} +leading_whitespace_re = re.compile(r'^[ \t\r\n]*') +line_comment_re = re.compile(grammar['line_comment']) +strip_re = re.compile('|'.join(grammar.values())) class Execute(String): - def __init__(self, lexer, parser, parent, command): - Token.__init__(self, 'Execute', lexer, parser, parent) - self.string = command + def __init__(self, lexer, parser, parent, command, escape_offset=0): + Token.__init__(self, self.__class__.__name__, lexer, parser, parent) + self.delimiter = '' # Command strings are not delimited in Execute tokens. + self.string = command # Original command will be evaluated at runtime. self.no_prompt = parser.no_prompt self.strip_command = parser.strip_command - # The lexer has parsed the command, including a newline. - # Make the debugger point to the beginning of the command. - self.start -= len(command) + 1 - self.mark_end(self.start + len(command)) + # The lexer has parsed the command and consumed the trailing command delimiter + # (newline, open curly bracket, RPC tag). We want the debugger point to the + # beginning of the command. The offset is expected to be the length of the + # additionally consumed command delimiter before entering this Execute token. + # Any offset caused by prior escape evaluation is also taken into account. + command = command.lstrip() + delimiter_offset = lexer.current_char - lexer.last_char + self.mark_start(self.start - len(command) - delimiter_offset - escape_offset) + self.mark_end(distance=len(command.strip()) + escape_offset) # Make sure that any variables specified in the command are declared. - string_re.sub(self.variable_test_cb, command) - self.parent.define(__response__=[]) + # Once more we use the left-stripped command to align with the start mark. + # NOTE: The substitution test might also fire in line comments with variables. + # This is prevented by making the line comments "invisible" while preserving + # the overall command length/structure for proper error tracking. + command = line_comment_re.sub(self.invisible_sub_cb, command) + varstring_re.sub(self.variable_test_cb, command) + + # Validate XML for RPCBody tokens. We need to use the original command + # string for exact reporting on UnicodeEncodeError and XMLSyntaxError. + # Additionally we need to replace all remaining variable chars with "_" + # to create an XML tree as valid as possible. The XML body will be + # tested once more at runtime by the protocol adapter (evaluated vars). + if type(parent).__name__ == 'RPCBody': + # The start mark points to the first non-whitespace character of the + # RPC body. We will need the original start position of the command. + original_start = self.start - len( + '' + leading_whitespace_re.search(self.string).group(0) + ) + + # Prepare full RPC command to eliminate most false-positives. + rpc_command = f'{self.string}' + rpc_command = line_comment_re.sub(self.invisible_sub_cb, rpc_command) + has_vars = bool(varstring_re.search(rpc_command)) + rpc_command = varstring_re.sub(self.variable_sigil_sub_cb, rpc_command) + + # Now we can validate the prepared RPC command. + msg = ( + "XML validation of the RPC command has failed. The error was " + "caused by:\n" + ) + try: + validate_xml(rpc_command, silent=False) + except UnicodeEncodeError as exc: + # UnicodeEncodeError position (start, end) comes with a valid + # absolute character offset. Simply adjust the start/end marks. + self.mark_start(original_start + exc.start) + self.mark_end(distance=(exc.end - exc.start)) + msg += f'{type(exc).__name__}: {exc}' + lexer.syntax_error(msg, self) + except XMLSyntaxError as exc: + # XMLSyntaxError position (line, offset) needs to be converted + # to an absolute character offset. Then adjust the start/end marks. + lines = rpc_command.splitlines(keepends=True) + exc_start = len(''.join(lines[:exc.lineno - 1])) + exc_start += exc.offset + # We assume the end positions are either both set or not! + exc_end = len(''.join(lines[:(exc.end_lineno or exc.lineno) - 1])) + exc_end += exc.end_offset or exc.offset + 1 + self.mark_start(original_start + exc_start) + self.mark_end(distance=(exc_end - exc_start)) + if has_vars: + msg = ( + "XML validation of the RPC command has failed. This could " + "be a real syntax error or might depend on the final variable " + "substitution at runtime. The warning was caused by:\n" + ) + msg += f'{type(exc).__name__}: {exc}' + lexer.syntax_warning(msg, self) + else: + msg += f'{type(exc).__name__}: {exc}' + lexer.syntax_error(msg, self) + + self.parent.define(__response__=['']) def value(self, context): - if not self.parent.is_defined('__connection__'): - error = 'Undefined variable "__connection__"' + connvar = "__connection__" + if not self.parent.is_defined(connvar): + error = 'Undefined variable %s' % repr(connvar) self.lexer.runtime_error(error, self) - conn = self.parent.get('__connection__') + conn = self.parent.get(connvar) + + # Prepare the command for execution by stripping unwanted substrings. + # The dedent of common indentations is useful for multiline commands. + command = strip_re.sub('', self.string) + command = textwrap.dedent(command) # Substitute variables in the command for values. - command = string_re.sub(self.variable_sub_cb, self.string) - command = command.lstrip() + command = varstring_re.sub(self.variable_sub_cb, command) # Execute the command. if self.no_prompt: conn.send(command + '\r') - response = '' else: conn.execute(command) - response = conn.response.replace('\r\n', '\n') - response = response.replace('\r', '\n').split('\n') - - if self.strip_command: - response = response[1:] - if len(response) == 0: - response = [''] - + response = conn.get_response(strip_command=self.strip_command) + response = response.split('\n') self.parent.define(__response__=response) return 1 def dump(self, indent=0): - print((' ' * indent) + self.name, self.string) + print((' ' * indent) + self.name, repr(self.string)) diff --git a/Exscript/interpreter/expression.py b/Exscript/interpreter/expression.py index ab25e531..39c4ef22 100644 --- a/Exscript/interpreter/expression.py +++ b/Exscript/interpreter/expression.py @@ -20,9 +20,8 @@ # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -from __future__ import print_function, absolute_import, division -from builtins import str import Exscript.interpreter.term + from ..parselib import Token @@ -34,7 +33,7 @@ def __init__(self, lexer, parser, parent, parent_node=None): # messages). lexer.skip(['whitespace', 'newline']) - Token.__init__(self, 'ExpressionNode', lexer, parser, parent) + Token.__init__(self, self.__class__.__name__, lexer, parser, parent) self.lft = None self.rgt = None self.op = None @@ -46,14 +45,14 @@ def __init__(self, lexer, parser, parent, parent_node=None): if not lexer.current_is('logical_operator', 'not'): self.lft = Exscript.interpreter.term.Term(lexer, parser, parent) - # The expression may end already (a single term is also an - # expression). + # The expression may end already (a single term is also an expression). lexer.skip(['whitespace', 'newline']) if not lexer.current_is('arithmetic_operator') and \ not lexer.current_is('logical_operator') and \ not lexer.current_is('comparison') and \ not lexer.current_is('regex_delimiter'): - self.mark_end() + # Early end of statement. + self.mark_end(rstrip=True) return # Expect the operator. @@ -62,13 +61,15 @@ def __init__(self, lexer, parser, parent, parent_node=None): not lexer.next_if('logical_operator') and \ not lexer.next_if('comparison') and \ not lexer.next_if('regex_delimiter'): - self.mark_end() - msg = 'Expected operator but got %s' % self.op_type + self.mark_start() + self.mark_end(distance=len(self.op)) + msg = 'Expected operator but got %s %s' % (self.op_type, repr(self.op)) lexer.syntax_error(msg, self) # Expect the second term. self.rgt = ExpressionNode(lexer, parser, parent, self) - self.mark_end() + # End of statement. + self.mark_end(rstrip=True) def priority(self): if self.op is None: @@ -90,7 +91,7 @@ def priority(self): elif self.op_type == 'logical_operator': return 1 else: - raise Exception('Invalid operator.') + raise Exception('Invalid operator: ' + repr(self.op)) def value(self, context): # Special behavior where we only have one term. @@ -181,20 +182,22 @@ def dump(self, indent=0): print((' ' * (indent + 1)) + 'Operator', self.op) if self.rgt is not None: self.rgt.dump(indent + 1) - print((' ' * indent) + self.name, self.op, 'end.') + print((' ' * indent) + self.name, self.op, 'end') class Expression(Token): def __init__(self, lexer, parser, parent): - Token.__init__(self, 'Expression', lexer, parser, parent) + Token.__init__(self, self.__class__.__name__, lexer, parser, parent) # Parse the expression. self.root = ExpressionNode(lexer, parser, parent) # Reorder the tree according to the operator priorities. self.prioritize(self.root) - self.mark_end() + + # End of statement. + self.mark_end(rstrip=True) def prioritize(self, start, prio=1): # print("Prioritizing from", start.op, "with prio", prio, (start.lft,) @@ -253,4 +256,4 @@ def value(self, context): def dump(self, indent=0): print((' ' * indent) + self.name, 'start') self.root.dump(indent + 1) - print((' ' * indent) + self.name, 'end.') + print((' ' * indent) + self.name, 'end') diff --git a/Exscript/interpreter/extract.py b/Exscript/interpreter/extract.py index 14f10e1a..e8181670 100644 --- a/Exscript/interpreter/extract.py +++ b/Exscript/interpreter/extract.py @@ -20,7 +20,6 @@ # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -from __future__ import print_function, absolute_import from ..parselib import Token from .regex import Regex from .term import Term @@ -29,42 +28,58 @@ class Extract(Token): def __init__(self, lexer, parser, parent): - Token.__init__(self, 'Extract', lexer, parser, parent) + Token.__init__(self, self.__class__.__name__, lexer, parser, parent) self.varnames = [] self.variables = {} self.append = False self.source = None if parser.no_prompt: + self.mark_end(distance=len('extract')) msg = "'extract' keyword does not work with --no-prompt" lexer.syntax_error(msg, self) # First expect a regular expression. lexer.expect(self, 'keyword', 'extract') lexer.expect(self, 'whitespace') + _regex_start = lexer.current_char # Helps with later Regex errors. self.regex = Regex(lexer, parser, parent) # Expect "as" keyword. - lexer.expect(self, 'whitespace') + lexer.expect(self, ['whitespace', 'newline']) + lexer.skip(['whitespace', 'newline']) if lexer.next_if('keyword', 'as'): self.append = False elif lexer.next_if('keyword', 'into'): self.append = True else: - _, token = lexer.token() - msg = 'Expected "as" or "into" but got %s' % token + ttype, token = lexer.token() + self.mark_start() + self.mark_end(distance=len(token)) + msg = "Expected 'as' or 'into' but got %s %s" % (ttype, repr(token)) lexer.syntax_error(msg, self) # Expect a list of variable names. - while 1: + while True: # Variable name. lexer.expect(self, 'whitespace') - _, token = lexer.token() + _, varname = lexer.token() lexer.expect(self, 'varname') - if token in self.variables: - lexer.syntax_error('Duplicate variable name %s', self) - self.varnames.append(token) - self.variables[token] = [] + + if varname.startswith('__') and varname.endswith('__'): + self.mark_start(lexer.last_char) + self.mark_end(distance=len(varname)) + msg = 'Extraction to internal variable %s' % repr(varname) + lexer.syntax_error(msg, self) + + if varname in self.variables: + self.mark_start(lexer.last_char) + self.mark_end(distance=len(varname)) + msg = 'Duplicate variable name %s' % repr(varname) + lexer.syntax_error(msg, self) + + self.varnames.append(varname) + self.variables[varname] = [] # Comma. if lexer.next_if('comma'): @@ -72,17 +87,21 @@ def __init__(self, lexer, parser, parent): break self.parent.define(**self.variables) - if len(self.varnames) != self.regex.n_groups: - count = (len(self.varnames), self.regex.n_groups) - error = '%s variables, but regex has %s groups' % count - lexer.syntax_error(error, self) + if len(self.varnames) != self.regex.groups: + count = (len(self.varnames), self.regex.groups) + self.mark_start(_regex_start) + self.mark_end(rstrip=True) + msg = 'Extraction to %s variable(s), but regex has %s capture group(s)' % count + lexer.syntax_error(msg, self) # Handle the "from" keyword. - lexer.skip('whitespace') + lexer.skip(['whitespace', 'newline']) if lexer.next_if('keyword', 'from'): lexer.expect(self, 'whitespace') self.source = Term(lexer, parser, parent) - self.mark_end() + + # End of statement. + self.mark_end(rstrip=True) def extract(self, context): # Re-initialize the variable content, because this method @@ -103,8 +122,7 @@ def extract(self, context): if match is None: continue - # If there was a match, store the extracted substrings in our - # list variables. + # If there was a match, store the extracted substrings in our variables. i = 0 for varname in self.varnames: i += 1 @@ -112,9 +130,10 @@ def extract(self, context): value = match.group(i) except IndexError: # This happens if the user provided a regex with less - # groups in it than the number of variables. - msg = 'Extract: %s variables, but regular expression' % i - msg += '\ncontains only %s groups.' % (i - 1) + # groups in it than the number of variables. This should have + # been caught already during parsing (comparing group count). + msg = 'Extraction to variable #%s has no corresponding' % i + msg += ' regex capture group (index out of range).' self.lexer.runtime_error(msg, self) self.variables[varname].append(value) @@ -130,6 +149,10 @@ def value(self, context): def dump(self, indent=0): mode = self.append and 'into' or 'as' - source = self.source is not None and self.source or 'buffer' - print((' ' * indent) + self.name, self.regex.string, end=' ') - print(mode, self.varnames, "from", source) + source = self.source is not None and self.source or '__response__' + print( + (' ' * indent) + self.name, + self.regex.delimiter + self.regex.string + self.regex.delimiter, + end=' ', + ) + print(mode, ', '.join(self.varnames), "from", source) diff --git a/Exscript/interpreter/fail.py b/Exscript/interpreter/fail.py index 55377c85..cd4ad0bc 100644 --- a/Exscript/interpreter/fail.py +++ b/Exscript/interpreter/fail.py @@ -20,32 +20,37 @@ # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -from __future__ import print_function, absolute_import from ..parselib import Token -from .expression import Expression from .exception import FailException +from .expression import Expression class Fail(Token): def __init__(self, lexer, parser, parent): - Token.__init__(self, 'Fail', lexer, parser, parent) + Token.__init__(self, self.__class__.__name__, lexer, parser, parent) self.expression = None # "fail" keyword. lexer.expect(self, 'keyword', 'fail') lexer.expect(self, 'whitespace') + # The later expression token skips all whitespaces and newlines, so we must + # fail on newlines here to keep the first expression term on the first line. + if lexer.current_is('newline'): + ttype, token = lexer.token() + self.mark_start() + self.mark_end(distance=len(token)) + msg = 'Expected expression but got %s %s' % (ttype, repr(token)) + lexer.syntax_error(msg, self) self.msg = Expression(lexer, parser, parent) - # 'If' keyword with an expression. - # token = lexer.token() + # "if" keyword with an expression. if lexer.next_if('keyword', 'if'): lexer.expect(self, 'whitespace') self.expression = Expression(lexer, parser, parent) - # End of expression. - self.mark_end() - lexer.skip(['whitespace', 'newline']) + # End of statement. + self.mark_end(rstrip=True) def value(self, context): if self.expression is None or self.expression.value(context)[0]: @@ -55,5 +60,6 @@ def value(self, context): def dump(self, indent=0): print((' ' * indent) + self.name, 'start') self.msg.dump(indent + 1) - self.expression.dump(indent + 1) - print((' ' * indent) + self.name, 'end.') + if self.expression: + self.expression.dump(indent + 1) + print((' ' * indent) + self.name, 'end') diff --git a/Exscript/interpreter/functioncall.py b/Exscript/interpreter/functioncall.py index 1111a5aa..86b78e1e 100644 --- a/Exscript/interpreter/functioncall.py +++ b/Exscript/interpreter/functioncall.py @@ -20,16 +20,16 @@ # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -from __future__ import print_function, absolute_import +import Exscript.interpreter.expression + from ..parselib import Token -from .expression import Expression from .exception import PermissionError class FunctionCall(Token): def __init__(self, lexer, parser, parent): - Token.__init__(self, 'FunctionCall', lexer, parser, parent) + Token.__init__(self, self.__class__.__name__, lexer, parser, parent) self.funcname = None self.arguments = [] @@ -39,35 +39,55 @@ def __init__(self, lexer, parser, parent): self.funcname = token[:-1] function = self.parent.get(self.funcname) if function is None: - lexer.syntax_error('Undefined function %s' % self.funcname, self) + self.mark_end(distance=len(self.funcname)) + lexer.syntax_error('Undefined function %s' % repr(self.funcname), self) + + if parser.secure_only and not hasattr(function, '_is_secure'): + self.mark_end(distance=len(self.funcname)) + msg = 'Use of insecure function %s is not permitted' % repr(self.funcname) + lexer.error(msg, self, PermissionError) # Parse the argument list. _, token = lexer.token() - while 1: + while True: if lexer.next_if('close_bracket'): break - self.arguments.append(Expression(lexer, parser, parent)) + self.arguments.append( + Exscript.interpreter.expression.Expression(lexer, parser, parent) + ) ttype, token = lexer.token() if not lexer.next_if('comma') and not lexer.current_is('close_bracket'): - error = 'Expected separator or argument list end but got %s' - lexer.syntax_error(error % ttype, self) + self.mark_start() + self.mark_end(distance=len(token)) + msg = 'Expected comma or closing bracket but got %s %s' + lexer.syntax_error(msg % (ttype, repr(token)), self) - if parser.secure_only and not hasattr(function, '_is_secure'): - msg = 'Use of insecure function %s is not permitted' % self.funcname - lexer.error(msg, self, PermissionError) - - self.mark_end() + # End of statement. + self.mark_end(rstrip=True) def dump(self, indent=0): print((' ' * indent) + self.name, self.funcname, 'start') for argument in self.arguments: argument.dump(indent + 1) - print((' ' * indent) + self.name, self.funcname, 'end.') + print((' ' * indent) + self.name, self.funcname, 'end') def value(self, context): argument_values = [arg.value(context) for arg in self.arguments] function = self.parent.get(self.funcname) if function is None: + self.mark_end(distance=len(self.funcname)) + self.lexer.runtime_error( + 'Undefined function %s' % repr(self.funcname), self + ) + try: + result = function(self.parent, *argument_values) + except Exception as e: + self.mark_end(distance=len(self.funcname)) self.lexer.runtime_error( - 'Undefined function %s' % self.funcname, self) - return function(self.parent, *argument_values) + '%s while executing function %s: %s' % ( + type(e).__name__, repr(self.funcname), e + ), + self, + ) + else: + return result diff --git a/Exscript/interpreter/ifcondition.py b/Exscript/interpreter/ifcondition.py index 817959a8..8edb7d81 100644 --- a/Exscript/interpreter/ifcondition.py +++ b/Exscript/interpreter/ifcondition.py @@ -20,22 +20,21 @@ # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -from __future__ import print_function, absolute_import -from ..parselib import Token import Exscript.interpreter.code + +from ..parselib import Token from .expression import Expression class IfCondition(Token): def __init__(self, lexer, parser, parent): - Token.__init__(self, 'If-condition', lexer, parser, parent) + Token.__init__(self, self.__class__.__name__, lexer, parser, parent) # Expect an expression. lexer.expect(self, 'keyword', 'if') lexer.expect(self, 'whitespace') self.expression = Expression(lexer, parser, parent) - self.mark_end() # Body of the if block. self.if_block = Exscript.interpreter.code.Code(lexer, parser, parent) @@ -45,6 +44,8 @@ def __init__(self, lexer, parser, parent): # If there is no "else" statement, just return. lexer.skip(['whitespace', 'newline']) if not lexer.next_if('keyword', 'else'): + # End of statement. + self.mark_end(rstrip=True) return # If the "else" statement is followed by an "if" (=elif), @@ -52,11 +53,16 @@ def __init__(self, lexer, parser, parent): lexer.skip(['whitespace', 'newline']) if lexer.current_is('keyword', 'if'): self.else_block = IfCondition(lexer, parser, parent) + # End of statement. + self.mark_end(rstrip=True) return # There was no "elif", so we handle a normal "else" condition here. self.else_block = Exscript.interpreter.code.Code(lexer, parser, parent) + # End of statement. + self.mark_end(rstrip=True) + def value(self, context): if self.expression.value(context)[0]: self.if_block.value(context) @@ -70,4 +76,4 @@ def dump(self, indent=0): self.if_block.dump(indent + 1) if self.else_block is not None: self.else_block.dump(indent + 1) - print((' ' * indent) + self.name, 'end.') + print((' ' * indent) + self.name, 'end') diff --git a/Exscript/interpreter/loop.py b/Exscript/interpreter/loop.py index b05e27ab..aaa2994e 100644 --- a/Exscript/interpreter/loop.py +++ b/Exscript/interpreter/loop.py @@ -20,19 +20,17 @@ # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -from __future__ import print_function -from __future__ import absolute_import -from builtins import range import Exscript.interpreter.code + from ..parselib import Token -from .term import Term from .expression import Expression +from .term import Term class Loop(Token): def __init__(self, lexer, parser, parent): - Token.__init__(self, 'Loop', lexer, parser, parent) + Token.__init__(self, self.__class__.__name__, lexer, parser, parent) self.during = None self.until = None self.thefrom = None @@ -40,7 +38,7 @@ def __init__(self, lexer, parser, parent): self.list_variables = [] self.iter_varnames = [] - # Expect one ore more lists. + # Expect one ore more lists for the default loop. lexer.expect(self, 'keyword', 'loop') lexer.expect(self, 'whitespace') if not lexer.current_is('keyword', 'while') and \ @@ -56,7 +54,7 @@ def __init__(self, lexer, parser, parent): # Expect the "as" keyword. lexer.expect(self, 'keyword', 'as') - # The iterator variable. + # The loop variable. lexer.next_if('whitespace') _, iter_varname = lexer.token() lexer.expect(self, 'varname') @@ -72,12 +70,13 @@ def __init__(self, lexer, parser, parent): lexer.skip(['whitespace', 'newline']) if len(self.iter_varnames) != len(self.list_variables): - error = '%s lists, but only %s iterators in loop' % (len(self.iter_varnames), - len(self.list_variables)) - lexer.syntax_error(error, self) + self.mark_end(rstrip=True) + count = (len(self.list_variables), len(self.iter_varnames)) + msg = 'Loop of %s list(s), but %s loop variable(s) defined' % count + lexer.syntax_error(msg, self) # Check if this is a "from ... to ..." loop. - if lexer.next_if('keyword', 'from'): + elif lexer.next_if('keyword', 'from'): lexer.expect(self, 'whitespace') self.thefrom = Expression(lexer, parser, parent) lexer.next_if('whitespace') @@ -102,18 +101,18 @@ def __init__(self, lexer, parser, parent): lexer.next_if('whitespace') # Check if this is an "until" loop. - if lexer.next_if('keyword', 'until'): + elif lexer.next_if('keyword', 'until'): lexer.expect(self, 'whitespace') self.until = Expression(lexer, parser, parent) lexer.next_if('whitespace') - # End of statement. - self.mark_end() - # Body of the loop block. lexer.skip(['whitespace', 'newline']) self.block = Exscript.interpreter.code.Code(lexer, parser, parent) + # End of statement. + self.mark_end(rstrip=True) + def value(self, context): if len(self.list_variables) == 0 and not self.thefrom: # If this is a "while" loop, iterate as long as the condition is @@ -136,7 +135,7 @@ def value(self, context): lists = [list(range(start, stop))] else: lists = [var.value(context) for var in self.list_variables] - vars = self.iter_varnames + varnames = self.iter_varnames # Make sure that all lists have the same length. for thelist in lists: @@ -148,7 +147,7 @@ def value(self, context): for i in range(len(lists[0])): f = 0 for thelist in lists: - self.block.define(**{vars[f]: [thelist[i]]}) + self.block.define(**{varnames[f]: [thelist[i]]}) f += 1 if self.until is not None and self.until.value(context)[0]: break @@ -159,10 +158,14 @@ def value(self, context): def dump(self, indent=0): print((' ' * indent) + self.name, end=' ') - print(self.list_variables, 'as', self.iter_varnames, 'start') + print(self.list_variables, 'as', ', '.join(self.iter_varnames), 'start') if self.during is not None: - self.during.dump(indent + 1) + print((' ' * (indent + 1)) + 'While start') + self.during.dump(indent + 2) + print((' ' * (indent + 1)) + 'While end') if self.until is not None: - self.until.dump(indent + 1) + print((' ' * (indent + 1)) + 'Until start') + self.until.dump(indent + 2) + print((' ' * (indent + 1)) + 'Until end') self.block.dump(indent + 1) - print((' ' * indent) + self.name, 'end.') + print((' ' * indent) + self.name, 'end') diff --git a/Exscript/interpreter/number.py b/Exscript/interpreter/number.py index d7563bc7..5181f251 100644 --- a/Exscript/interpreter/number.py +++ b/Exscript/interpreter/number.py @@ -20,11 +20,8 @@ # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -from __future__ import print_function -from builtins import object - -class Number(object): +class Number: def __init__(self, number): self.number = int(number) diff --git a/Exscript/interpreter/parser.py b/Exscript/interpreter/parser.py index 9c813597..3a916175 100644 --- a/Exscript/interpreter/parser.py +++ b/Exscript/interpreter/parser.py @@ -20,25 +20,26 @@ # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -from __future__ import absolute_import -from builtins import object import copy +from collections.abc import Callable, Iterable + from ..parselib import Lexer from .program import Program -class Parser(object): +class Parser: def __init__(self, **kwargs): - self.no_prompt = kwargs.get('no_prompt', False) + self.no_escaping = kwargs.get('no_escaping', False) + self.no_prompt = kwargs.get('no_prompt', False) self.strip_command = kwargs.get('strip_command', True) - self.secure_only = kwargs.get('secure', False) - self.debug = kwargs.get('debug', 0) + self.secure_only = kwargs.get('secure', False) + self.debug = kwargs.get('debug', 0) self.variables = {} def define(self, **kwargs): for key, value in list(kwargs.items()): - if hasattr(value, '__iter__') or hasattr(value, '__call__'): + if not isinstance(value, str) and isinstance(value, Iterable) or isinstance(value, Callable): self.variables[key] = value else: self.variables[key] = [value] diff --git a/Exscript/interpreter/program.py b/Exscript/interpreter/program.py index 8a23ab91..d0894c8e 100644 --- a/Exscript/interpreter/program.py +++ b/Exscript/interpreter/program.py @@ -20,10 +20,10 @@ # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -from __future__ import absolute_import import copy -from .template import Template + from .scope import Scope +from .template import Template class Program(Scope): diff --git a/Exscript/interpreter/regex.py b/Exscript/interpreter/regex.py index 1300a4bd..b044bbd3 100644 --- a/Exscript/interpreter/regex.py +++ b/Exscript/interpreter/regex.py @@ -20,17 +20,13 @@ # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -from __future__ import print_function, absolute_import import re -from .string import String -# Matches any opening parenthesis that is neither preceded by a backslash -# nor has a "?:" or "?<" appended. -bracket_re = re.compile(r'(?'), + ('close_rpc_tag', r'<\/rpc>'), + ('newline', r'[\r\n]'), + ('raw_data_before_tag', r'[^\r\n]+?(?=<\/?rpc>)'), + ('raw_data', r'[^\r\n]+'), +) + +grammar_c = [] +for thetype, regex in grammar: + grammar_c.append((thetype, re.compile(regex))) + + +class RPCBody(Scope): + + def __init__(self, lexer, parser, parent): + Scope.__init__(self, self.__class__.__name__, lexer, parser, parent) + lexer.set_grammar(grammar_c) + lexer.expect(self, 'open_rpc_tag') + buffer = '' + while True: + if lexer.next_if('close_rpc_tag'): + if buffer.strip() != '': + self.add(Execute(lexer, parser, self, buffer)) + buffer = '' + break + # Line comments in multiline RPCs are handled later by the Execute + # token, so we have to collect them too. + elif lexer.current_is( + ['line_comment', 'newline', 'raw_data_before_tag', 'raw_data'] + ): + buffer += lexer.token()[1] + next(lexer) + elif lexer.current_is('EOF'): + self.mark_start() + lexer.syntax_error("Expected '' but got EOF", self) + else: + ttype, token = lexer.token() + self.mark_start() + self.mark_end(distance=len(token)) + lexer.syntax_error('Unexpected %s %s' % (ttype, repr(token)), self) + lexer.restore_grammar() + # End of statement. + self.mark_end(rstrip=True) + + def execute(self): + return self.value(self) diff --git a/Exscript/interpreter/scope.py b/Exscript/interpreter/scope.py index 97663c6d..0a7b606f 100644 --- a/Exscript/interpreter/scope.py +++ b/Exscript/interpreter/scope.py @@ -20,8 +20,8 @@ # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -from __future__ import print_function, absolute_import -from copy import deepcopy +import copy + from ..parselib import Token @@ -34,7 +34,10 @@ def __init__(self, name, lexer, parser, parent=None, *args, **kwargs): self.exit_requested = 0 for key in self.variables: if key.find('.') < 0 and not key.startswith('_'): - assert type(self.variables[key]) == type([]) + if type(self.variables[key]) is type([]): + continue + msg = '%s: variable %s must be a list, but got %s' + raise TypeError(msg % (self.name, key, type(self.variables[key]))) def exit_request(self): self.exit_requested = 1 @@ -44,7 +47,7 @@ def define(self, **kwargs): return self.parent.define(**kwargs) for key in kwargs: if key.find('.') >= 0 or key.startswith('_') \ - or type(kwargs[key]) == type([]): + or type(kwargs[key]) is type([]): self.variables[key] = kwargs[key] else: self.variables[key] = [kwargs[key]] @@ -79,7 +82,7 @@ def copy_public_vars(self): """ vars = self.get_vars() vars = dict([k for k in list(vars.items()) if not k[0].startswith('_')]) - return deepcopy(vars) + return copy.deepcopy(vars) def get(self, name, default=None): if name in self.variables: @@ -100,8 +103,8 @@ def dump(self, indent=0): child.dump(indent + 1) print((' ' * indent) + self.name, 'end') - def dump1(self): + def dump_vars(self): if self.parent is not None: - self.parent.dump1() + self.parent.dump_vars() return print("Scope:", self.variables) diff --git a/Exscript/interpreter/string.py b/Exscript/interpreter/string.py index 3976b903..3837edf7 100644 --- a/Exscript/interpreter/string.py +++ b/Exscript/interpreter/string.py @@ -20,17 +20,27 @@ # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -from __future__ import print_function, absolute_import -from builtins import next -from builtins import str +import ast import re + from ..parselib import Token -varname_re = re.compile(r'((?:__)?[a-zA-Z][\w_]*(?:__)?)') -string_re = re.compile(r'(\\?)\$([\w_]*)') +varname_re = re.compile(r'(__[a-zA-Z][\w_]*__|[a-zA-Z][\w_]*)') +varstring_re = re.compile(r'(\\?)\$([\w_]*)') + +escaped_grammar = [ + # Unicode range: 0x00000000 to 0x0010FFFF + ('escaped_unicode_32bit', r'\\U[0-9a-fA-F]{8}'), + ('escaped_unicode_16bit', r'\\u[0-9a-fA-F]{4}'), + ('escaped_hex', r'\\x[0-9a-fA-F]{2}'), + ('escaped_octal', r'\\[0-7]{3}'), + ('escaped_extended_controls', r'\\[0abefv]'), + ('escaped_common_controls', r'\\[nrt]'), +] -grammar = [ - ('escaped_data', r'\\.'), +grammar = escaped_grammar + [ + ('escaped_data', r'\\.'), + ('newline', r'[\r\n]'), ] grammar_c = [] @@ -40,59 +50,85 @@ class String(Token): - def __init__(self, lexer, parser, parent): + def __init__(self, lexer, parser, parent, purpose='string'): Token.__init__(self, self.__class__.__name__, lexer, parser, parent) # Create a grammar depending on the delimiting character. - tok_type, delimiter = lexer.token() - escaped_delimiter = '\\' + delimiter - data = r'[^\r\n\\' + escaped_delimiter + ']+' + self.delimiter = lexer.token()[1] + escaped_delimiter = '\\' + self.delimiter + data_re = re.compile(r'[^\r\n\\' + escaped_delimiter + ']+') delimiter_re = re.compile(escaped_delimiter) - data_re = re.compile(data) grammar_with_delim = grammar_c[:] - grammar_with_delim.append(('string_data', data_re)) - grammar_with_delim.append(('string_delimiter', delimiter_re)) + grammar_with_delim.append((purpose + '_data', data_re)) + grammar_with_delim.append((purpose + '_delimiter', delimiter_re)) lexer.set_grammar(grammar_with_delim) # Begin parsing the string. - lexer.expect(self, 'string_delimiter') self.string = '' - while 1: - if lexer.current_is('string_data'): + lexer.expect(self, purpose + '_delimiter') + while True: + if lexer.current_is(purpose + '_data'): self.string += lexer.token()[1] next(lexer) - elif lexer.current_is('escaped_data'): - self.string += self._escape(lexer.token()[1]) + elif lexer.current_is([ + 'escaped_unicode_32bit', + 'escaped_unicode_16bit', + 'escaped_hex', + 'escaped_octal', + 'escaped_extended_controls', + 'escaped_common_controls', + 'escaped_data', + ]): + self.string += self.eval_escape(*lexer.token()) next(lexer) - elif lexer.next_if('string_delimiter'): + elif lexer.next_if(purpose + '_delimiter'): break else: - ttype = lexer.token()[0] - lexer.syntax_error('Expected string but got %s' % ttype, self) + ttype, token = lexer.token() + self.mark_start() + self.mark_end(distance=len(token)) + msg = 'Expected %s but got %s %s' % (purpose, ttype, repr(token)) + lexer.syntax_error(msg, self) # Make sure that any variables specified in the command are declared. - string_re.sub(self.variable_test_cb, self.string) - self.mark_end() + varstring_re.sub(self.variable_test_cb, self.string) lexer.restore_grammar() - - def _escape(self, token): - char = token[1] - if char == 'n': - return '\n' - elif char == 'r': - return '\r' - elif char == '$': # Escaping is done later, in variable_sub_cb. - return token - return char - - def _variable_error(self, field, msg): - self.start += self.string.find(field) - self.end = self.start + len(field) + # End of statement. + self.mark_end(rstrip=True) + + def eval_escape(self, ttype, token): + # Supporting various types of escape sequences. The easiest way is to + # evaluate them as strings with the builtin ast module. + if not self.parser.no_escaping and ttype in ( + 'escaped_unicode_32bit', + 'escaped_unicode_16bit', + 'escaped_hex', + 'escaped_octal', + 'escaped_extended_controls', + ): + if token == '\\e': # Invalid sequence in Python. + return '\x1b' + return ast.literal_eval('"' + token + '"') + if ttype == 'escaped_common_controls': + return ast.literal_eval('"' + token + '"') + # Simple escape sequence with one character. + if len(token) == 2: + char = token[1] + if char == '$': # This is handled later, in variable_sub_cb. + return token + # Simply unescape anything else. + return token[1:] + + def _variable_error(self, field, field_start, msg): + # Narrow down the marks on the variable that caused the error. + self.mark_start(self.start + len(self.delimiter) + field_start) + self.mark_end(distance=len(field)) self.lexer.runtime_error(msg, self) # Tokens that include variables in a string may use this callback to # substitute the variable against its value. def variable_sub_cb(self, match): + field_start = match.start(0) field = match.group(0) escape = match.group(1) varname = match.group(2) @@ -105,15 +141,15 @@ def variable_sub_cb(self, match): return '$' if not varname_re.match(varname): msg = '%s is not a variable name' % repr(varname) - self._variable_error(field, msg) + self._variable_error(field, field_start, msg) # Check the variable value. if value is None: msg = 'Undefined variable %s' % repr(varname) - self._variable_error(field, msg) + self._variable_error(field, field_start, msg) elif hasattr(value, 'func_name'): msg = '%s is a function, not a variable name' % repr(varname) - self._variable_error(field, msg) + self._variable_error(field, field_start, msg) elif isinstance(value, list): if len(value) > 0: value = '\n'.join([str(v) for v in value]) @@ -127,8 +163,29 @@ def variable_test_cb(self, match): self.variable_sub_cb(match) return match.group(0) + # Tokens that include variables in a string may use this callback to mask + # the sigil ($, \$) with underscores while keeping the variable name intact. + def variable_sigil_sub_cb(self, match): + # Replace only the sigil ($, \$) with underscores; keep the variable name. + return '_' * (len(match.group(0)) - len(match.group(2))) + match.group(2) + + # Tokens that need to make any string invisible to the lexer, like comments, + # while keeping its length, may use this callback. + def invisible_sub_cb(self, match, char=' '): + lines = match.group(0).splitlines(keepends=True) + for i, line in enumerate(lines): + length = len(line) + # Preserve the line-breaks! + if line.endswith('\r\n'): + lines[i] = (char * (length - 2)) + line[-2:] + elif line.endswith(('\r', '\n')): + lines[i] = (char * (length - 1)) + line[-1] + else: + lines[i] = char * length + return ''.join(lines) + def value(self, context): - return [string_re.sub(self.variable_sub_cb, self.string)] + return [varstring_re.sub(self.variable_sub_cb, self.string)] def dump(self, indent=0): - print((' ' * indent) + 'String "' + self.string + '"') + print((' ' * indent) + self.name, repr(self.string)) diff --git a/Exscript/interpreter/template.py b/Exscript/interpreter/template.py index e9983f7e..cbf3c97c 100644 --- a/Exscript/interpreter/template.py +++ b/Exscript/interpreter/template.py @@ -20,20 +20,35 @@ # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -from __future__ import print_function, absolute_import -from builtins import next +import ast import re -from .scope import Scope + from .code import Code from .execute import Execute +from .rpc import RPCBody +from .scope import Scope + +escaped_grammar = [ + # Unicode range: 0x00000000 to 0x0010FFFF + ('escaped_unicode_32bit', r'\\U[0-9a-fA-F]{8}'), + ('escaped_unicode_16bit', r'\\u[0-9a-fA-F]{4}'), + ('escaped_hex', r'\\x[0-9a-fA-F]{2}'), + ('escaped_octal', r'\\[0-7]{3}'), + ('escaped_extended_controls', r'\\[0abefv]'), + ('escaped_common_controls', r'\\[nrt]'), +] -grammar = ( +grammar = escaped_grammar + [ ('escaped_data', r'\\.'), + ('line_comment', r'(?:^|(?<=[\r\n}]))[ \t]*#[^\r\n]*(?:[\r\n]|$)'), ('open_curly_bracket', '{'), ('close_curly_bracket', '}'), + ('open_rpc_tag', r''), + ('close_rpc_tag', r'<\/rpc>'), ('newline', r'[\r\n]'), - ('raw_data', r'[^\r\n{}\\]+') -) + ('raw_data_before_tag', r'[^\r\n{}\\]+?(?=<\/?rpc>)'), + ('raw_data', r'[^\r\n{}\\]+'), +] grammar_c = [] for thetype, regex in grammar: @@ -43,44 +58,83 @@ class Template(Scope): def __init__(self, lexer, parser, parent, *args, **kwargs): - Scope.__init__(self, 'Template', lexer, parser, parent, **kwargs) + Scope.__init__(self, self.__class__.__name__, lexer, parser, parent, **kwargs) lexer.set_grammar(grammar_c) - # print("Opening Scope:", lexer.token()) buffer = '' - while 1: + escape_offset = 0 + while True: + lexer.skip(['line_comment']) if self.exit_requested or lexer.current_is('EOF'): break - elif lexer.next_if('open_curly_bracket'): + elif lexer.current_is('open_curly_bracket'): if buffer.strip() != '': - self.add(Execute(lexer, parser, self, buffer)) + self.add(Execute(lexer, parser, self, buffer, escape_offset)) buffer = '' + escape_offset = 0 if isinstance(parent, Code): + # Need to consume the bracket if the Code token is not called + # due to nesting in IfCondition, Loop and Try tokens. + next(lexer) break self.add(Code(lexer, parser, self)) - elif lexer.current_is('raw_data'): - if lexer.token()[1].lstrip().startswith('#'): - while not lexer.current_is('newline'): - next(lexer) - continue + elif lexer.current_is('open_rpc_tag'): + if buffer.strip() != '': + self.add(Execute(lexer, parser, self, buffer, escape_offset)) + buffer = '' + escape_offset = 0 + self.add(RPCBody(lexer, parser, self)) + elif lexer.current_is(['raw_data_before_tag', 'raw_data']): buffer += lexer.token()[1] next(lexer) - elif lexer.current_is('escaped_data'): - token = lexer.token()[1] - if token[1] == '$': - # An escaped $ is handled by the Execute() token, so - # we do not strip the \ here. - buffer += token - else: - buffer += token[1] + elif lexer.current_is([ + 'escaped_unicode_32bit', + 'escaped_unicode_16bit', + 'escaped_hex', + 'escaped_octal', + 'escaped_extended_controls', + 'escaped_common_controls', + 'escaped_data', + ]): + ttype, token = lexer.token() + result = self.eval_escape(ttype, token) + buffer += result + escape_offset += len(token) - len(result) next(lexer) elif lexer.next_if('newline'): if buffer.strip() != '': - self.add(Execute(lexer, parser, self, buffer)) + self.add(Execute(lexer, parser, self, buffer, escape_offset)) buffer = '' + escape_offset = 0 else: - ttype = lexer.token()[0] - lexer.syntax_error('Unexpected %s' % ttype, self) + ttype, token = lexer.token() + self.mark_start() + self.mark_end(distance=len(token)) + lexer.syntax_error('Unexpected %s %s' % (ttype, repr(token)), self) lexer.restore_grammar() + # End of statement. + self.mark_end(rstrip=True) + + def eval_escape(self, ttype, token): + # Supporting various types of escape sequences. The easiest way is to + # evaluate them as strings with the builtin ast module. + if not self.parser.no_escaping and ttype in ( + 'escaped_unicode_32bit', + 'escaped_unicode_16bit', + 'escaped_hex', + 'escaped_octal', + 'escaped_extended_controls', + 'escaped_common_controls', + ): + if token == '\\e': # Invalid sequence in Python. + return '\x1b' + return ast.literal_eval('"' + token + '"') + # Simple escape sequence with one character. + if len(token) == 2: + char = token[1] + if char == '$': # This is handled later by the Execute token. + return token + # Simply unescape anything else. + return token[1:] def execute(self): return self.value(self) diff --git a/Exscript/interpreter/term.py b/Exscript/interpreter/term.py index 612d5978..5307886e 100644 --- a/Exscript/interpreter/term.py +++ b/Exscript/interpreter/term.py @@ -20,19 +20,18 @@ # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -from __future__ import print_function, absolute_import from ..parselib import Token -from .variable import Variable -from .number import Number from .functioncall import FunctionCall -from .string import String +from .number import Number from .regex import Regex +from .string import String +from .variable import Variable class Term(Token): def __init__(self, lexer, parser, parent): - Token.__init__(self, 'Term', lexer, parser, parent) + Token.__init__(self, self.__class__.__name__, lexer, parser, parent) self.term = None self.lft = None self.rgt = None @@ -42,7 +41,9 @@ def __init__(self, lexer, parser, parent): ttype, token = lexer.token() if lexer.current_is('varname'): if not parent.is_defined(token): - lexer.error('Undeclared variable %s' % token, self, ValueError) + self.mark_start() + self.mark_end(distance=len(token)) + lexer.error('Undefined variable %s' % repr(token), self, NameError) self.term = Variable(lexer, parser, parent) elif lexer.current_is('open_function_call'): self.term = FunctionCall(lexer, parser, parent) @@ -55,14 +56,18 @@ def __init__(self, lexer, parser, parent): elif lexer.next_if('keyword', 'true'): self.term = Number(1) elif lexer.next_if('octal_number'): - self.term = Number(int(token[1:], 8)) + start = 2 if "o" in token else 1 + self.term = Number(int(token[start:], 8)) elif lexer.next_if('hex_number'): self.term = Number(int(token[2:], 16)) elif lexer.current_is('regex_delimiter'): self.term = Regex(lexer, parser, parent) else: - lexer.syntax_error('Expected term but got %s' % ttype, self) - self.mark_end() + self.mark_start() + self.mark_end(distance=len(token)) + lexer.syntax_error('Expected term but got %s %s' % (ttype, repr(token)), self) + # End of statement. + self.mark_end(rstrip=True) def priority(self): return 6 diff --git a/Exscript/interpreter/trying.py b/Exscript/interpreter/trying.py index 67e44d64..302dc5c7 100644 --- a/Exscript/interpreter/trying.py +++ b/Exscript/interpreter/trying.py @@ -20,8 +20,8 @@ # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -from __future__ import print_function, absolute_import import Exscript.interpreter.code + from ..protocols.exception import ProtocolException from .scope import Scope @@ -29,17 +29,16 @@ class Try(Scope): def __init__(self, lexer, parser, parent): - Scope.__init__(self, 'Try', lexer, parser, parent) - - lexer.next_if('whitespace') + Scope.__init__(self, self.__class__.__name__, lexer, parser, parent) lexer.expect(self, 'keyword', 'try') - lexer.skip(['whitespace', 'newline']) self.block = Exscript.interpreter.code.Code(lexer, parser, parent) + # End of statement. + self.mark_end(rstrip=True) def value(self, context): try: self.block.value(context) - except ProtocolException as e: + except ProtocolException: return 1 return 1 diff --git a/Exscript/interpreter/variable.py b/Exscript/interpreter/variable.py index 092c9efc..3deed11a 100644 --- a/Exscript/interpreter/variable.py +++ b/Exscript/interpreter/variable.py @@ -20,24 +20,24 @@ # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -from __future__ import print_function, absolute_import from ..parselib import Token class Variable(Token): def __init__(self, lexer, parser, parent): - Token.__init__(self, 'Variable', lexer, parser, parent) + Token.__init__(self, self.__class__.__name__, lexer, parser, parent) self.varname = lexer.token()[1] lexer.expect(self, 'varname') - self.mark_end() + # End of statement. + self.mark_end(rstrip=True) def value(self, context): val = self.parent.get(self.varname) if val is None: - msg = 'Undefined variable %s' % self.varname + msg = 'Undefined variable %s' % repr(self.varname) self.lexer.runtime_error(msg, self) return val def dump(self, indent=0): - print((' ' * indent) + 'Variable', self.varname, '.') + print((' ' * indent) + self.name, self.varname) diff --git a/Exscript/key.py b/Exscript/key.py index 761e8435..a9a73443 100644 --- a/Exscript/key.py +++ b/Exscript/key.py @@ -23,12 +23,11 @@ """ Represents a private key. """ -from builtins import object -from paramiko import RSAKey, ECDSAKey, Ed25519Key +from paramiko import ECDSAKey, Ed25519Key, RSAKey from paramiko.ssh_exception import SSHException -class PrivateKey(object): +class PrivateKey: """ Represents a cryptographic key, and may be used to authenticate diff --git a/Exscript/logger.py b/Exscript/logger.py index e917f716..77281e00 100644 --- a/Exscript/logger.py +++ b/Exscript/logger.py @@ -23,24 +23,103 @@ """ Logging to memory. """ -from __future__ import print_function, absolute_import, unicode_literals -from future import standard_library -standard_library.install_aliases() -from builtins import filter -from builtins import str -from builtins import object -import os + import errno +import logging +import os +import threading +import time import weakref +from collections import defaultdict from io import StringIO from itertools import chain -from collections import defaultdict + from .util.impl import format_exception -logger_registry = weakref.WeakValueDictionary() # Map id(logger) to Logger. +# ----------------------------------------------------------------------------- +# Python Logging + +LOGGING_FORMAT = "%(levelname)s: %(name)s (%(threadName)s) %(message)s" +LOGGING_LEVEL = logging.DEBUG + + +def getConsoleLogger(name, *, level=LOGGING_LEVEL, fmt=LOGGING_FORMAT, utc=True): + """Get the console logger with the given name. + + This function is not related to the host logging mechanics of Exscript. + Instead it configures the logging for the particular Python logger instance. + """ + logger = logging.getLogger(name) + + for hdlr in logger.handlers: + if type(hdlr) is logging.StreamHandler: + return logger + + _fmt = logging.Formatter(fmt) + if utc: + _fmt.converter = time.gmtime + + _console = logging.StreamHandler() + _console.setFormatter(_fmt) + _console.setLevel(level) + + logger.addHandler(_console) + logger.setLevel(logging.DEBUG) + logger.propagate = False + return logger + + +def setupLogging(*, package=False, parser=False, protocol=False, init=getConsoleLogger): + """Setup the package logging. + + By default the setup adds a console/terminal handler to make the logs visible. + The relevant loggers get filtered according to the boolean flags. + """ + + class _LogFilter: + + def __init__(self, name, enabled): + self.name = name + self.enabled = enabled + + def filter(self, record): + if record.name == self.name: + return self.enabled + if record.name.startswith(self.name + "."): + return self.enabled + return False + + states = { + "Exscript": package, + "Exscript.interpreter.parser": parser, + "Exscript.parselib.lexer": parser, + "Exscript.protocols": protocol, + "Exscript.servers": protocol, + # Netconf Client + "ncclient.capabilities": protocol, + "ncclient.devices": protocol, + "ncclient.manager": protocol, + "ncclient.operations": protocol, + "ncclient.transport.session": protocol, + "ncclient.transport.ssh": protocol, + # Paramiko + "paramiko.transport": protocol, + } + + for name, enabled in states.items(): + logger = init(name) + filter = _LogFilter(name, enabled) + for hdlr in logger.handlers: + hdlr.addFilter(filter) + + +# ----------------------------------------------------------------------------- +# Exscript Logging + +logger_registry = weakref.WeakValueDictionary() # Map id(logger) to Logger. -class Log(object): +class Log: def __init__(self, name): self.name = name @@ -161,7 +240,7 @@ def succeeded(self): Log.succeeded(self) -class Logger(object): +class Logger: """ A QueueListener that implements logging for the queue. @@ -174,6 +253,7 @@ def __init__(self): decorator to send messages to the logger. """ logger_registry[id(self)] = self + self._lock = threading.Lock() self.logs = defaultdict(list) self.started = 0 self.success = 0 @@ -212,7 +292,8 @@ def add_log(self, job_id, name, attempt): log = Log(name) log.started() self.logs[job_id].append(log) - self.started += 1 + with self._lock: + self.started += 1 return log def log(self, job_id, message): @@ -224,15 +305,17 @@ def log(self, job_id, message): def log_aborted(self, job_id, exc_info): log = self._get_log(job_id) log.aborted(exc_info) - self.failed += 1 + with self._lock: + self.failed += 1 def log_succeeded(self, job_id): log = self._get_log(job_id) log.succeeded() - self.success += 1 + with self._lock: + self.success += 1 -class LoggerProxy(object): +class LoggerProxy: """ An object that has a 1:1 relation to a Logger object in another @@ -248,22 +331,27 @@ def __init__(self, parent, logger_id): """ self.parent = parent self.logger_id = logger_id + self._lock = threading.Lock() def add_log(self, job_id, name, attempt): - self.parent.send(('log-add', (self.logger_id, job_id, name, attempt))) - response = self.parent.recv() + with self._lock: + self.parent.send(('log-add', (self.logger_id, job_id, name, attempt))) + response = self.parent.recv() if isinstance(response, Exception): raise response return response def log(self, job_id, message): - self.parent.send(('log-message', (self.logger_id, job_id, message))) + with self._lock: + self.parent.send(('log-message', (self.logger_id, job_id, message))) def log_aborted(self, job_id, exc_info): - self.parent.send(('log-aborted', (self.logger_id, job_id, exc_info))) + with self._lock: + self.parent.send(('log-aborted', (self.logger_id, job_id, exc_info))) def log_succeeded(self, job_id): - self.parent.send(('log-succeeded', (self.logger_id, job_id))) + with self._lock: + self.parent.send(('log-succeeded', (self.logger_id, job_id))) class FileLogger(Logger): diff --git a/Exscript/parselib/__init__.py b/Exscript/parselib/__init__.py index fcd39ed4..48a7245e 100644 --- a/Exscript/parselib/__init__.py +++ b/Exscript/parselib/__init__.py @@ -20,10 +20,10 @@ # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -from __future__ import absolute_import +import inspect + from .lexer import Lexer from .token import Token -import inspect __all__ = [name for name, obj in list(locals().items()) if not (name.startswith('_') or inspect.ismodule(obj))] diff --git a/Exscript/parselib/exception.py b/Exscript/parselib/exception.py index c7e33d1c..6620ec0e 100644 --- a/Exscript/parselib/exception.py +++ b/Exscript/parselib/exception.py @@ -30,6 +30,14 @@ class LexerException(Exception): pass +class LexerWarning(UserWarning): + + """ + Base class for minor warnings. + """ + pass + + class CompileError(LexerException): """ @@ -39,6 +47,15 @@ class CompileError(LexerException): pass +class CompileWarning(LexerWarning): + + """ + Raised during the compilation procedure if the template contained + a syntax error, that does not prevent compilation but should be noted. + """ + pass + + class ExecuteError(LexerException): """ diff --git a/Exscript/parselib/lexer.py b/Exscript/parselib/lexer.py index 49999c57..91a9c9f8 100644 --- a/Exscript/parselib/lexer.py +++ b/Exscript/parselib/lexer.py @@ -20,13 +20,14 @@ # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -from __future__ import print_function -from builtins import next -from builtins import object -from .exception import LexerException, CompileError, ExecuteError +import re +import warnings +from types import SimpleNamespace +from .exception import CompileError, CompileWarning, ExecuteError, LexerException -class Lexer(object): + +class Lexer: def __init__(self, parser_cls, *args, **kwargs): """ @@ -61,21 +62,30 @@ def match(self): self.token_buffer = (token_type, match.group(0)) # print("Match:", self.token_buffer) return - end = self.input.find('\n', self.current_char + 2) - error = 'Invalid syntax: %s' % repr(self.input[self.current_char:end]) - self.syntax_error(error) + # The match failed so we raise an invalid syntax error. We create a fake + # sender object with marks to reuse the existing error handling code. + sender = SimpleNamespace(start=self.current_char, end=self.current_char + 1) + # Match 1..50 chars (eager) up to the last space, newline or EOF to + # prevent cutoffs in the middle of a word/line. + regex = re.compile(r'.{1,50}(?=[ \t\r\n]|$)', flags=re.S) + context = regex.match(self.input, sender.start).group(0) + error = 'Invalid syntax: %s' % repr(context) + self.syntax_error(error, sender) def _get_line_number_from_char(self, char): return self.input[:char].count('\n') + 1 - def _get_current_line_number(self): + def get_current_line_number(self): return self._get_line_number_from_char(self.current_char) def _get_line(self, number): return self.input.split('\n')[number - 1] + def _get_lines(self, first, last): + return self.input.split('\n')[first - 1 : last] + def get_current_line(self): - line = self._get_current_line_number() + line = self.get_current_line_number() return self._get_line(line) def _get_line_from_char(self, char): @@ -89,22 +99,55 @@ def _get_line_position_from_char(self, char): break line_start -= 1 line_end = self.input.find('\n', char) + if line_end == -1: + line_end = self.input_length return line_start, line_end def _error(self, exc_cls, error, sender=None): if not sender: + if issubclass(exc_cls, Warning): + warnings.warn(exc_cls('\n' + error), stacklevel=3) + return raise exc_cls('\n' + error) - start, end = self._get_line_position_from_char(sender.end) - line_number = self._get_line_number_from_char(sender.end) - line = self._get_line(line_number) - offset = sender.start - start - token_len = sender.end - sender.start - output = line + '\n' - if token_len <= 1: + lineno_first = self._get_line_number_from_char(sender.start) + lineno_last = self._get_line_number_from_char(sender.end) + # When the sender marks a range, that ends with a newline character, the + # end mark would point to an additional irrelevant line, so we fix that. + if self.input[sender.start:sender.end].endswith('\n'): + lineno_last -= 1 + lines = self._get_lines(lineno_first, lineno_last) + + # Determine the offset and width of the offending range. + if len(lines) <= 1: + # Use the sender's start/end marks for single-line ranges. The sender + # is responsible to set the correct marks (no adjustments here). + lineno = str(lineno_first) + offset = sender.start - self._get_line_position_from_char(sender.start)[0] + token_length = sender.end - sender.start + else: + # Multiline ranges are more complicated so the min. offset on the left and + # max. width to the right need to be calculated from all non-empty lines. + lineno = '%s-%s' % (lineno_first, lineno_last) + offset = self.input_length # reasonable high starting value + token_length = 1 # reasonable low starting value + whitespace_re = re.compile(r'[ \t]*') + for ln in lines: + if ln.strip() == '': + continue + offset = min(offset, len(whitespace_re.match(ln).group(0))) + token_length = max(token_length, len(ln.rstrip())) + token_length -= offset + + # Build the error string with the offending line(s). + output = '\n'.join(lines) + '\n' + if token_length <= 1: output += (' ' * offset) + '^\n' else: - output += (' ' * offset) + "'" + ('-' * (token_len - 2)) + "'\n" - output += '%s in %s:%s' % (error, self.filename, line_number) + output += (' ' * offset) + ('^' * token_length) + '\n' + output += '%s in %s:%s' % (error, self.filename, lineno) + if issubclass(exc_cls, Warning): + warnings.warn(exc_cls('\n' + output), stacklevel=3) + return raise exc_cls('\n' + output) def error(self, error, sender=None, exc_cls=LexerException): @@ -113,61 +156,65 @@ def error(self, error, sender=None, exc_cls=LexerException): def syntax_error(self, error, sender=None): self._error(CompileError, error, sender) + def syntax_warning(self, warning, sender=None): + self._error(CompileWarning, warning, sender) + def runtime_error(self, error, sender=None): self._error(ExecuteError, error, sender) - def forward(self, chars=1): + def _forward(self, chars=1): self.last_char = self.current_char self.current_char += chars self.token_buffer = None def __next__(self): if self.token_buffer: - self.forward(len(self.token_buffer[1])) + if self.token_buffer[0] == 'EOF': + msg = "Unexpected EOF while iterating tokens." + raise StopIteration(msg) + self._forward(len(self.token_buffer[1])) def next_if(self, types, token=None): - if token is not None: - if self.current_is(types, token): - next(self) - return 1 - return 0 - - if type(types) != type([]): + if type(types) is not type([]): types = [types] - for t in types: - if self.current_is(t, token): - next(self) - return 1 + if self.current_is(types, token): + next(self) + return 1 return 0 def skip(self, types, token=None): while self.next_if(types, token): pass - def expect(self, sender, type, token=None): + def expect(self, sender, types, token=None): cur_type, cur_token = self.token() - if self.next_if(type, token): + if type(types) is not type([]): + types = [types] + if self.next_if(types, token): return if token: - error = 'Expected "%s" but got %s %s' - error = error % (token, cur_type, repr(cur_token)) + error = 'Expected %s but got %s %s' + error = error % (repr(token), cur_type, repr(cur_token)) + elif len(types) > 1: + error = 'Expected one of %s but got %s %s' + error = error % (', '.join(types), cur_type, repr(cur_token)) else: - error = 'Expected %s but got %s (%s)' - error = error % (type, cur_type, repr(cur_token)) + error = 'Expected %s but got %s %s' + error = error % (types[0], cur_type, repr(cur_token)) # In this case we do not point to the token that raised the error, # but to the actual position of the lexer. sender.start = self.current_char - sender.end = self.current_char + 1 + sender.end = self.current_char + len(cur_token) self.syntax_error(error, sender) - def current_is(self, type, token=None): + def current_is(self, types, token=None): if self.token_buffer is None: self.match() - if self.token_buffer[0] != type: + if type(types) is not type([]): + types = [types] + if self.token_buffer[0] not in types: return 0 - if token is None: - return 1 - if self.token_buffer[1] == token: + if token is None or self.token_buffer[1] == token: return 1 return 0 @@ -187,7 +234,9 @@ def parse(self, string, filename=None): self.token_buffer = None self.grammar = [] compiled = self.parser_cls(self, *self.parser_cls_args) - if self.debug > 3: + if self.debug >= 5: + # IDEA: Consider to switch compiled.dump() from print() to logging. + # The related code is located in Exscript.interpreter.*.dump() methods. compiled.dump() return compiled diff --git a/Exscript/parselib/token.py b/Exscript/parselib/token.py index fe53b186..0de03d22 100644 --- a/Exscript/parselib/token.py +++ b/Exscript/parselib/token.py @@ -20,17 +20,15 @@ # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -from __future__ import print_function -from builtins import object -class Token(object): +class Token: """ Abstract base class for all tokens. """ - class Iterator(object): + class Iterator: """ A tree iterator that walks through all tokens. @@ -95,13 +93,34 @@ def value(self, context): for child in self.get_children(): child.value(context) - def mark_start(self): - self.start = self.lexer.current_char + def mark_start(self, char=None): + if char is None: + char = self.lexer.current_char + # EOF will point behind the last character index (length of input)! + self.start = max(0, min(char, self.lexer.input_length)) if self.start >= self.end: - self.end = self.start + 1 - - def mark_end(self, char=None): - self.end = char and char or self.lexer.current_char + self.mark_end(distance=1) + + def mark_end(self, char=None, distance=None, rstrip=False): + if char is not None and distance is not None: + msg = 'must not specify both arguments "char" and "distance"' + raise ValueError(msg) + if distance is not None: + char = self.start + distance + elif char is None: + char = self.lexer.current_char + # The rstrip flag emulates the str.rstrip block per char to rewind the end mark + # to the last non-whitespace character. This is useful when the end mark is set + # at the end of a Token init, that finally skipped trailing whitespaces. + if rstrip: + if isinstance(rstrip, str): + isspace = lambda c: c in rstrip # noqa: E731 + else: + isspace = lambda c: c.isspace() # noqa: E731 + while char > 0 and isspace(self.lexer.input[char - 1:char]): + char -= 1 + # EOF will point behind the last character index (length of input)! + self.end = max(self.start, min(char, self.lexer.input_length)) def __iter__(self): """ diff --git a/Exscript/protocols/__init__.py b/Exscript/protocols/__init__.py index 5fdb2255..7d85ed72 100644 --- a/Exscript/protocols/__init__.py +++ b/Exscript/protocols/__init__.py @@ -20,20 +20,23 @@ # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -from __future__ import absolute_import from .. import Account from ..util.cast import to_host from ..util.url import Url +from .dummy import Dummy +from .netconf import Netconf from .protocol import Protocol -from .telnet import Telnet from .ssh2 import SSH2 -from .dummy import Dummy +from .telnet import Telnet -protocol_map = {'dummy': Dummy, - 'pseudo': Dummy, - 'telnet': Telnet, - 'ssh': SSH2, - 'ssh2': SSH2} +protocol_map = { + 'dummy': Dummy, + 'pseudo': Dummy, + 'telnet': Telnet, + 'ssh': SSH2, + 'ssh2': SSH2, + 'netconf': Netconf, +} def get_protocol_from_name(name): @@ -66,7 +69,7 @@ def create_protocol(name, **kwargs): return cls(**kwargs) -def prepare(host, default_protocol='telnet', **kwargs): +def prepare(host, default_protocol='ssh', **kwargs): """ Creates an instance of the protocol by either parsing the given URL-formatted hostname using :class:`Exscript.util.url`, or according to @@ -90,7 +93,7 @@ def prepare(host, default_protocol='telnet', **kwargs): return conn -def connect(host, default_protocol='telnet', **kwargs): +def connect(host, default_protocol='ssh', **kwargs): """ Like :class:`prepare()`, but also connects to the host by calling :class:`Protocol.connect()`. If the URL or host contain any login info, this @@ -114,5 +117,6 @@ def connect(host, default_protocol='telnet', **kwargs): return conn import inspect + __all__ = [name for name, obj in list(locals().items()) if not (name.startswith('_') or inspect.ismodule(obj))] diff --git a/Exscript/protocols/drivers/__init__.py b/Exscript/protocols/drivers/__init__.py index 006c5f52..ce19cabf 100644 --- a/Exscript/protocols/drivers/__init__.py +++ b/Exscript/protocols/drivers/__init__.py @@ -21,41 +21,42 @@ # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. import inspect -from Exscript.protocols.drivers.driver import Driver -from Exscript.protocols.drivers.adtran import AdtranDriver -from Exscript.protocols.drivers.aironet import AironetDriver -from Exscript.protocols.drivers.aix import AIXDriver -from Exscript.protocols.drivers.arbor_peakflow import ArborPeakflowDriver -from Exscript.protocols.drivers.brocade import BrocadeDriver -from Exscript.protocols.drivers.enterasys import EnterasysDriver -from Exscript.protocols.drivers.generic import GenericDriver -from Exscript.protocols.drivers.hp_pro_curve import HPProCurveDriver -from Exscript.protocols.drivers.ios import IOSDriver -from Exscript.protocols.drivers.nxos import NXOSDriver -from Exscript.protocols.drivers.ios_xr import IOSXRDriver -from Exscript.protocols.drivers.ace import ACEDriver -from Exscript.protocols.drivers.junos import JunOSDriver -from Exscript.protocols.drivers.junos_erx import JunOSERXDriver -from Exscript.protocols.drivers.mrv import MRVDriver -from Exscript.protocols.drivers.one_os import OneOSDriver -from Exscript.protocols.drivers.shell import ShellDriver -from Exscript.protocols.drivers.smart_edge_os import SmartEdgeOSDriver -from Exscript.protocols.drivers.vrp import VRPDriver -from Exscript.protocols.drivers.sros import SROSDriver -from Exscript.protocols.drivers.aruba import ArubaDriver -from Exscript.protocols.drivers.enterasys_wc import EnterasysWCDriver -from Exscript.protocols.drivers.fortios import FortiOSDriver -from Exscript.protocols.drivers.bigip import BigIPDriver -from Exscript.protocols.drivers.isam import IsamDriver -from Exscript.protocols.drivers.zte import ZteDriver -from Exscript.protocols.drivers.vxworks import VxworksDriver -from Exscript.protocols.drivers.ericsson_ban import EricssonBanDriver -from Exscript.protocols.drivers.rios import RIOSDriver -from Exscript.protocols.drivers.eos import EOSDriver -from Exscript.protocols.drivers.cienasaos import CienaSAOSDriver -from Exscript.protocols.drivers.icotera import IcoteraDriver -from Exscript.protocols.drivers.zyxel import ZyxelDriver -from Exscript.protocols.drivers.sp_vxoa import SilverPeakVXOADriver + +from .ace import ACEDriver as ACEDriver +from .adtran import AdtranDriver as AdtranDriver +from .aironet import AironetDriver as AironetDriver +from .aix import AIXDriver as AIXDriver +from .arbor_peakflow import ArborPeakflowDriver as ArborPeakflowDriver +from .aruba import ArubaDriver as ArubaDriver +from .bigip import BigIPDriver as BigIPDriver +from .brocade import BrocadeDriver as BrocadeDriver +from .cienasaos import CienaSAOSDriver as CienaSAOSDriver +from .driver import Driver +from .enterasys import EnterasysDriver as EnterasysDriver +from .enterasys_wc import EnterasysWCDriver as EnterasysWCDriver +from .eos import EOSDriver as EOSDriver +from .ericsson_ban import EricssonBanDriver as EricssonBanDriver +from .fortios import FortiOSDriver as FortiOSDriver +from .generic import GenericDriver as GenericDriver +from .hp_pro_curve import HPProCurveDriver as HPProCurveDriver +from .icotera import IcoteraDriver as IcoteraDriver +from .ios import IOSDriver as IOSDriver +from .ios_xr import IOSXRDriver as IOSXRDriver +from .isam import IsamDriver as IsamDriver +from .junos import JunOSDriver as JunOSDriver +from .junos_erx import JunOSERXDriver as JunOSERXDriver +from .mrv import MRVDriver as MRVDriver +from .nxos import NXOSDriver as NXOSDriver +from .one_os import OneOSDriver as OneOSDriver +from .rios import RIOSDriver as RIOSDriver +from .shell import ShellDriver as ShellDriver +from .smart_edge_os import SmartEdgeOSDriver as SmartEdgeOSDriver +from .sp_vxoa import SilverPeakVXOADriver as SilverPeakVXOADriver +from .sros import SROSDriver as SROSDriver +from .vrp import VRPDriver as VRPDriver +from .vxworks import VxworksDriver as VxworksDriver +from .zte import ZteDriver as ZteDriver +from .zyxel import ZyxelDriver as ZyxelDriver driver_classes = [] drivers = [] @@ -63,7 +64,7 @@ def isdriver(o): - return inspect.isclass(o) and issubclass(o, Driver) and not o is Driver + return inspect.isclass(o) and issubclass(o, Driver) and o is not Driver def add_driver(cls): driver = cls() diff --git a/Exscript/protocols/drivers/ace.py b/Exscript/protocols/drivers/ace.py index ef0aea8c..0030342e 100644 --- a/Exscript/protocols/drivers/ace.py +++ b/Exscript/protocols/drivers/ace.py @@ -21,20 +21,22 @@ # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ -A driver for Cisco Application Control Engine (ACE) +A driver for Cisco Application Control Engine (ACE). """ import re -from Exscript.protocols.drivers.driver import Driver + +from .driver import Driver _user_re = [re.compile(r'user ?name: ?$', re.I)] -_password_re = [re.compile(r'(?:[\r\n]Password: ?|last resort password:)$')] -_tacacs_re = re.compile(r'[\r\n]s\/key[\S ]+\r?%s' % _password_re[0].pattern) -_prompt_re = [re.compile(r'[\r\n][\-\w+\.:/]+(?:\([^\)]+\))?[>#] ?$')] -_error_re = [re.compile(r'%Error'), - re.compile(r'invalid input', re.I), - re.compile(r'(?:incomplete|ambiguous) command', re.I), - re.compile(r'connection timed out', re.I), - re.compile(r'[^\r\n]+ not found', re.I)] +_password_re = [re.compile(r'(?:(?:\r\n|[\r\n])Password: ?|last resort password:)$')] +_prompt_re = [re.compile(r'(?:\r\n|[\r\n])[\-\w+\.:/]+(?:\([^\)]+\))?[>#] ?$')] +_error_re = [ + re.compile(r'%Error'), + re.compile(r'invalid input', re.I), + re.compile(r'(?:incomplete|ambiguous) command', re.I), + re.compile(r'connection timed out', re.I), + re.compile(r'[^\r\n]+ not found', re.I), +] class ACEDriver(Driver): diff --git a/Exscript/protocols/drivers/adtran.py b/Exscript/protocols/drivers/adtran.py index bec6b31b..b3a7c59b 100644 --- a/Exscript/protocols/drivers/adtran.py +++ b/Exscript/protocols/drivers/adtran.py @@ -1,11 +1,31 @@ +# +# Copyright (C) 2010-2017 Samuel Abels +# The MIT License (MIT) +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files +# (the "Software"), to deal in the Software without restriction, +# including without limitation the rights to use, copy, modify, merge, +# publish, distribute, sublicense, and/or sell copies of the Software, +# and to permit persons to whom the Software is furnished to do so, +# subject to the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ A driver for the Adtran DSLAMs. The DSLAMs only report "OpenSSH" in their SSH remote protocol id and have no SSH banner so no possibility for check_*_for_os(). - """ -from __future__ import absolute_import from .driver import Driver diff --git a/Exscript/protocols/drivers/aironet.py b/Exscript/protocols/drivers/aironet.py index 9956f129..a4ad3de9 100644 --- a/Exscript/protocols/drivers/aironet.py +++ b/Exscript/protocols/drivers/aironet.py @@ -21,19 +21,14 @@ # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ -A driver for Cisco Aironet Wireless Controllers +A driver for Cisco Aironet Wireless Controllers. """ import re -from Exscript.protocols.drivers.driver import Driver + +from .driver import Driver _user_re = [re.compile(r'User:\s$', re.I)] -_password_re = [re.compile(r'(?:[\r\n]Password: ?|last resort password:)$')] -_prompt_re = [ - re.compile(r'[\r\n](?:\([^\)]+\))\s>$')] -_error_re = [re.compile(r'Incorrect\susage', re.I), - re.compile(r'Incorrect\sinput', re.I), - re.compile(r'connection timed out', re.I), - re.compile(r'[^\r\n]+ not found', re.I)] +_prompt_re = [re.compile(r'(?:\r\n|[\r\n])(?:\([^\)]+\))\s>$')] class AironetDriver(Driver): diff --git a/Exscript/protocols/drivers/aix.py b/Exscript/protocols/drivers/aix.py index 2befdb22..7eeacacc 100644 --- a/Exscript/protocols/drivers/aix.py +++ b/Exscript/protocols/drivers/aix.py @@ -24,10 +24,11 @@ A driver for AIX. """ import re -from Exscript.protocols.drivers.driver import Driver -_user_re = [re.compile(r'[\r\n]Authorized Login: $')] -_password_re = [re.compile(r'[\r\n]\w+\'s Password: $')] +from .driver import Driver + +_user_re = [re.compile(r'(?:\r\n|[\r\n])Authorized Login: $')] +_password_re = [re.compile(r'(?:\r\n|[\r\n])\w+\'s Password: $')] _aix_re = re.compile(r'\bAIX\b') diff --git a/Exscript/protocols/drivers/arbor_peakflow.py b/Exscript/protocols/drivers/arbor_peakflow.py index 08db9d5c..d7ab14f0 100644 --- a/Exscript/protocols/drivers/arbor_peakflow.py +++ b/Exscript/protocols/drivers/arbor_peakflow.py @@ -24,14 +24,17 @@ A driver for Peakflow SP by Arbor Networks. """ import re -from Exscript.protocols.drivers.driver import Driver + +from .driver import Driver _user_re = [re.compile(r'(user|login): $', re.I)] _password_re = [re.compile(r'Password: $')] -_prompt_re = [re.compile(r'[\r\n][\-\w+\._]+@[\-\w+\._~]+:(?:/\w*)+[#%] $'), # arbos prompt - re.compile(r'Exit anyway\? \[.\] $'), # confirm on exit - re.compile(r'[\r\n]DIAG\>'), # up to 8.0 shell - re.compile(r'[\-\w+\._~()]+:.*#')] # 8.3 shell +_prompt_re = [ + re.compile(r'(?:\r\n|[\r\n])[\-\w+\._]+@[\-\w+\._~]+:(?:/\w*)+[#%] $'), # arbos prompt + re.compile(r'Exit anyway\? \[.\] $'), # confirm on exit + re.compile(r'(?:\r\n|[\r\n])DIAG\>'), # up to 8.0 shell + re.compile(r'[\-\w+\._~()]+:.*#'), # 8.3 shell +] _os_re = re.compile(r'\b(arbo[rs]|peakflow)\b', re.I) diff --git a/Exscript/protocols/drivers/aruba.py b/Exscript/protocols/drivers/aruba.py index 96d9ea0e..cfbda7ad 100644 --- a/Exscript/protocols/drivers/aruba.py +++ b/Exscript/protocols/drivers/aruba.py @@ -21,21 +21,22 @@ # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ -A driver for Aruba controllers +A driver for Aruba controllers. """ import re -from Exscript.protocols.drivers.driver import Driver + +from .driver import Driver _user_re = [re.compile(r'user ?name: ?$', re.I)] -_password_re = [re.compile(r'(?:[\r\n]Password: ?|last resort password:)$')] -_tacacs_re = re.compile(r'[\r\n]s\/key[\S ]+\r?%s' % _password_re[0].pattern) -_prompt_re = [re.compile(r'[\r\n]\(\w+\) [>#] ?$')] -_error_re = [re.compile(r'%Error'), - re.compile(r'invalid input', re.I), - re.compile(r'(?:incomplete|ambiguous) command', re.I), - re.compile(r'connection timed out', re.I), - re.compile(r'[^\r\n]+ not found', re.I)] -_aruba_prompt_re = [re.compile(r'\(Aruba\d?\d?\d?\d?\) >')] +_password_re = [re.compile(r'(?:(?:\r\n|[\r\n])Password: ?|last resort password:)$')] +_prompt_re = [re.compile(r'(?:\r\n|[\r\n])\(\w+\) [>#] ?$')] +_error_re = [ + re.compile(r'%Error'), + re.compile(r'invalid input', re.I), + re.compile(r'(?:incomplete|ambiguous) command', re.I), + re.compile(r'connection timed out', re.I), + re.compile(r'[^\r\n]+ not found', re.I), +] class ArubaDriver(Driver): diff --git a/Exscript/protocols/drivers/bigip.py b/Exscript/protocols/drivers/bigip.py index 80888d9f..f60d55ad 100644 --- a/Exscript/protocols/drivers/bigip.py +++ b/Exscript/protocols/drivers/bigip.py @@ -21,17 +21,19 @@ # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ -A driver for F5 Big-IP system (TMSH SHELL) +A driver for F5 Big-IP system (TMSH SHELL). """ -from __future__ import unicode_literals import re -from Exscript.protocols.drivers.driver import Driver + +from .driver import Driver _user_re = [re.compile(r'user ?name: ?$', re.I)] -_password_re = [re.compile(r'(?:[\r\n]Password: ?|last resort password:)$')] +_password_re = [re.compile(r'(?:(?:\r\n|[\r\n])Password: ?|last resort password:)$')] _prompt_re = [re.compile(r'\S+@(\(.*?\)){1,4}\(tmos.*?\)#\s?')] -_error_re = [re.compile(r'Syntax Error', re.I), - re.compile(r'connection timed out', re.I)] +_error_re = [ + re.compile(r'Syntax Error', re.I), + re.compile(r'connection timed out', re.I), +] class BigIPDriver(Driver): diff --git a/Exscript/protocols/drivers/brocade.py b/Exscript/protocols/drivers/brocade.py index 005da722..514196ac 100644 --- a/Exscript/protocols/drivers/brocade.py +++ b/Exscript/protocols/drivers/brocade.py @@ -25,18 +25,21 @@ """ import re -from Exscript.protocols.drivers.driver import Driver -_user_re = [re.compile(r'[\r\n](Please Enter Login Name: |User Name:)$')] -_password_re = [re.compile(r'[\r\n](Please Enter Password: |Password:)$')] +from .driver import Driver + +_user_re = [re.compile(r'(?:\r\n|[\r\n])(Please Enter Login Name: |User Name:)$')] +_password_re = [re.compile(r'(?:\r\n|[\r\n])(Please Enter Password: |Password:)$')] _warning = r'(?:Warning: \d+ user\(s\) already in config mode\.)' -_prompt = r'[\r\n]?(telnet|SSH)@[\-\w+\.:]+(?:\([\-\/\w]+\))?[>#]$' +_prompt = r'(?:\r\n|[\r\n])?(telnet|SSH)@[\-\w+\.:]+(?:\([\-\/\w]+\))?[>#]$' _prompt_re = [re.compile(_warning + r'?' + _prompt)] -_error_re = [re.compile(r'%Error'), - re.compile(r'Invalid input', re.I), - re.compile(r'(?:incomplete|ambiguous) command', re.I), - re.compile(r'connection timed out', re.I), - re.compile(r'[^\r\n]+ not found', re.I)] +_error_re = [ + re.compile(r'%Error'), + re.compile(r'Invalid input', re.I), + re.compile(r'(?:incomplete|ambiguous) command', re.I), + re.compile(r'connection timed out', re.I), + re.compile(r'[^\r\n]+ not found', re.I), +] class BrocadeDriver(Driver): diff --git a/Exscript/protocols/drivers/cienasaos.py b/Exscript/protocols/drivers/cienasaos.py index 6fc8a8bc..caad173a 100644 --- a/Exscript/protocols/drivers/cienasaos.py +++ b/Exscript/protocols/drivers/cienasaos.py @@ -1,17 +1,42 @@ +# +# Copyright (C) 2010-2017 Samuel Abels +# The MIT License (MIT) +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files +# (the "Software"), to deal in the Software without restriction, +# including without limitation the rights to use, copy, modify, merge, +# publish, distribute, sublicense, and/or sell copies of the Software, +# and to permit persons to whom the Software is furnished to do so, +# subject to the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ -A driver for Ciena SAOS carrier ethernet devices +A driver for Ciena SAOS carrier Ethernet devices. """ import re -from Exscript.protocols.drivers.driver import Driver + +from .driver import Driver _user_re = [re.compile(r'[^:]* login: ?$', re.I)] _password_re = [re.compile(r'Password: ?$')] -_prompt_re = [re.compile(r'[\r\n][\-\w+\.:/]+[>#] ?$')] -_error_re = [re.compile(r'SHELL PARSER FAILURE'), - re.compile(r'invalid input', re.I), - re.compile(r'(?:incomplete|ambiguous) command', re.I), - re.compile(r'connection timed out', re.I), - re.compile(r'[^\r\n]+ not found', re.I)] +_prompt_re = [re.compile(r'(?:\r\n|[\r\n])[\-\w+\.:/]+[>#] ?$')] +_error_re = [ + re.compile(r'SHELL PARSER FAILURE'), + re.compile(r'invalid input', re.I), + re.compile(r'(?:incomplete|ambiguous) command', re.I), + re.compile(r'connection timed out', re.I), + re.compile(r'[^\r\n]+ not found', re.I), +] class CienaSAOSDriver(Driver): diff --git a/Exscript/protocols/drivers/driver.py b/Exscript/protocols/drivers/driver.py index 66104681..ba926532 100644 --- a/Exscript/protocols/drivers/driver.py +++ b/Exscript/protocols/drivers/driver.py @@ -23,62 +23,76 @@ """ Base class for all drivers. """ -from builtins import object import re import string _flags = re.I _printable = re.escape(string.printable) _unprintable = r'[^' + _printable + r']' -_unprintable_re = re.compile(_unprintable) _ignore = r'[\x1b\x07\x00]' -_nl = r'[\r\n]' +_nl = r'(?:\r\n|[\r\n])' _prompt_start = _nl + r'(?:' + _unprintable + r'*|' + _ignore + '*)' -_prompt_chars = r'[\-\w\(\)@:~]' _filename = r'(?:[\w\+\-\._]+)' _path = r'(?:(?:' + _filename + r')?(?:/' + _filename + r')*/?)' _any_path = r'(?:' + _path + r'|~' + _path + r'?)' _host = r'(?:[\w+\-\.]+)' _user = r'(?:[\w+\-]+)' _user_host = r'(?:(?:' + _user + r'\@)?' + _host + r')' -_prompt_re = [re.compile(_prompt_start - + r'[\[\<]?' - + r'\w+' - + _user_host + r'?' - + r':?' - + _any_path + r'?' - + r'[: ]?' - + _any_path + r'?' - + r'(?:\(' + _filename + r'\))?' - + r'[\]\-]?' - + r'[#>%\$\]] ?' - + _unprintable + r'*' - + r'\Z', _flags)] +_prompt_re = [ + re.compile(_prompt_start + + r'[\[\<]?' + + r'\w+' + + _user_host + r'?' + + r':?' + + _any_path + r'?' + + r'[: ]?' + + _any_path + r'?' + + r'(?:\(' + _filename + r'\))?' + + r'[\]\-]?' + + r'[#>%\$\]] ?' + + _unprintable + r'*' + + r'\Z', _flags), +] _user_re = [re.compile(r'(user ?name|user|login): *$', _flags)] _pass_re = [re.compile(r'password:? *$', _flags)] -_errors = [r'error', - r'invalid', - r'incomplete', - r'unrecognized', - r'unknown command', - r'connection timed out', - r'[^\r\n]+ not found'] +_errors = [ + r'error', + r'invalid', + r'incomplete', + r'unrecognized', + r'unknown command', + r'connection timed out', + r'[^\r\n]+ not found', +] _error_re = [re.compile(r'^%?\s*(?:' + '|'.join(_errors) + r')', _flags)] -_login_fail = [r'bad secrets', - r'denied', - r'invalid', - r'too short', - r'incorrect', - r'connection timed out', - r'failed', - r'failure'] -_login_fail_re = [re.compile(_nl - + r'[^\r\n]*' - + r'(?:' + '|'.join(_login_fail) + r')', _flags)] - - -class Driver(object): +_login_fail = [ + r'bad secrets', + r'denied', + r'invalid', + r'too short', + r'incorrect', + r'connection timed out', + r'failed', + r'failure', +] +_login_fail_re = [ + re.compile(_nl + + r'[^\r\n]*' + + r'(?:' + '|'.join(_login_fail) + r')', _flags), +] +_netconf_prompt_re = [ + re.compile(r'(?<=>)' # any XML closing tag (generic) + + r'[ \t\r\n]*' + + r'(?:]]>]]>|\n##\n)' # 1.0 EOM or 1.1 EOC sequence + + r'[ \t\r\n]*' + + _unprintable + r'*' + + r'\Z', _flags), +] +_netconf_error_re = [re.compile(r'<(?:[\w.\-]+:)?rpc-error[\s/>]', _flags)] + + +class Driver: def __init__(self, name): self.name = name @@ -87,7 +101,23 @@ def __init__(self, name): self.prompt_re = _prompt_re self.error_re = _error_re self.login_error_re = _login_fail_re + # SSH2 related: self.reconnect_between_auth_methods = False + # Netconf related: + self.netconf_device_params = {} # see ncclient documentation + self.netconf_capabilities_re = [] # used to determine vendor/OS driver + self.netconf_prompt_re = _netconf_prompt_re + self.netconf_error_re = _netconf_error_re + + def __repr__(self): + return f"{type(self).__name__}('{self.name}')" + + def supports_os_guesser(self): + return ( + self.check_protocol_for_os.__code__ is not Driver.check_protocol_for_os.__code__ or + self.check_head_for_os.__code__ is not Driver.check_head_for_os.__code__ or + self.check_response_for_os.__code__ is not Driver.check_response_for_os.__code__ + ) def check_protocol_for_os(self, string): return 0 @@ -95,6 +125,17 @@ def check_protocol_for_os(self, string): def _check_protocol(self, string): return self.name, self.check_protocol_for_os(string) + def check_netconf_capabilities(self, string): + if not self.netconf_capabilities_re: + return 0 + for regex in self.netconf_capabilities_re: + if regex.search(string): + return 100 + return 0 + + def _check_netconf(self, string): + return self.name, self.check_netconf_capabilities(string) + def check_head_for_os(self, string): return 0 @@ -107,9 +148,6 @@ def check_response_for_os(self, string): def _check_response(self, string): return self.name, self.check_response_for_os(string) - def supports_os_guesser(self): - return (not self.check_head_for_os.__code__ is Driver.check_head_for_os.__code__) - def clean_response_for_re_match(self, response): return response, '' @@ -117,7 +155,10 @@ def init_terminal(self, conn): pass def supports_auto_authorize(self): - return self.__class__.auto_authorize != Driver.auto_authorize + return self.__class__.auto_authorize is not Driver.auto_authorize def auto_authorize(self, conn, account, flush, bailout): conn.app_authorize(account, flush, bailout) + + def logout(self, conn): + conn.send('exit\r') diff --git a/Exscript/protocols/drivers/enterasys.py b/Exscript/protocols/drivers/enterasys.py index e3255fd1..d9775aa0 100644 --- a/Exscript/protocols/drivers/enterasys.py +++ b/Exscript/protocols/drivers/enterasys.py @@ -24,11 +24,12 @@ A driver for Enterasys devices. """ import re -from Exscript.protocols.drivers.driver import Driver -_user_re = [re.compile(r'[\r\n]Username: $')] -_password_re = [re.compile(r'[\r\n]Password: $')] -_prompt_re = [re.compile(r'[\r\n][\-\w+\.]+(?:\([^\)]+\))?-?[>#] ?$')] +from .driver import Driver + +_user_re = [re.compile(r'(?:\r\n|[\r\n])Username: $')] +_password_re = [re.compile(r'(?:\r\n|[\r\n])Password: $')] +_prompt_re = [re.compile(r'(?:\r\n|[\r\n])[\-\w+\.]+(?:\([^\)]+\))?-?[>#] ?$')] _enterasys_re = re.compile(r'\benterasys\b', re.I) diff --git a/Exscript/protocols/drivers/enterasys_wc.py b/Exscript/protocols/drivers/enterasys_wc.py index 6e15277d..9f33b637 100644 --- a/Exscript/protocols/drivers/enterasys_wc.py +++ b/Exscript/protocols/drivers/enterasys_wc.py @@ -26,13 +26,14 @@ Created using a C5110 device. """ import re -from Exscript.protocols.drivers.driver import Driver -_user_re = [re.compile(r'[\r\n]Username: $')] -_password_re = [re.compile(r'[\r\n]Password: $')] +from .driver import Driver + +_user_re = [re.compile(r'(?:\r\n|[\r\n])Username: $')] +_password_re = [re.compile(r'(?:\r\n|[\r\n])Password: $')] _prompt_re = [ - re.compile(r'[\r\n][a-zA-Z0-9-_ .]+# $'), # base prompt - re.compile(r'[\r\n][a-zA-Z0-9-_ .]+(?::[A-Za-z0-9_.-]+)*# $') + re.compile(r'(?:\r\n|[\r\n])[a-zA-Z0-9-_ .]+# $'), # base prompt + re.compile(r'(?:\r\n|[\r\n])[a-zA-Z0-9-_ .]+(?::[A-Za-z0-9_.-]+)*# $'), ] _hwc_re = re.compile(r'Enterasys Wireless Convergence OS', re.I) diff --git a/Exscript/protocols/drivers/eos.py b/Exscript/protocols/drivers/eos.py index be728b22..9fc290a0 100644 --- a/Exscript/protocols/drivers/eos.py +++ b/Exscript/protocols/drivers/eos.py @@ -24,10 +24,11 @@ A driver for Arista EOS. """ import re -from Exscript.protocols.drivers.driver import Driver + +from .driver import Driver # Match following prompt pattern variations: -# NOTE: Upon first login, EOS does not return \r\n before prompt +# Important: Upon first login, EOS does not return \r\n before prompt. # hostname(s1)> # hostname(s1)# # hostname(s1)(config)# @@ -46,23 +47,27 @@ _user_re = [re.compile(r'user ?name: ?$', re.I), re.compile(r' login: ?$', re.I)] -_password_re = [re.compile(r'[\r\n]Password: ?$', re.I)] -_prompt_re = [re.compile(r'[\r\n]?[\-\w+\.:/]+(?:\([^\)]+\)){,3}[>#] ?$')] -_error_re = [re.compile(r'% ?Error'), - re.compile(r'% ?Invalid input', re.I), - re.compile(r'% ?(?:Incomplete|Ambiguous) command', re.I), - re.compile(r'connection timed out', re.I)] -_login_fail = [r'[Bb]ad secrets?', - r'denied', - r'invalid', - r'too short', - r'incorrect', - r'connection timed out', - r'failed', - r'failure'] -_login_fail_re = [re.compile(r'[\r\n]' - + r'[^\r\n]*' - + r'(?:' + '|'.join(_login_fail) + r')', re.I)] +_password_re = [re.compile(r'(?:\r\n|[\r\n])Password: ?$', re.I)] +_prompt_re = [re.compile(r'(?:\r\n|[\r\n])?[\-\w+\.:/]+(?:\([^\)]+\)){,3}[>#] ?$')] +_error_re = [ + re.compile(r'% ?Error'), + re.compile(r'% ?Invalid input', re.I), + re.compile(r'% ?(?:Incomplete|Ambiguous) command', re.I), + re.compile(r'connection timed out', re.I), +] +_login_fail = [ + r'[Bb]ad secrets?', + r'denied', + r'invalid', + r'too short', + r'incorrect', + r'connection timed out', + r'failed', + r'failure', +] +_login_fail_re = [ + re.compile(r'(?:\r\n|[\r\n])[^\r\n]*(?:' + '|'.join(_login_fail) + r')', re.I), +] class EOSDriver(Driver): diff --git a/Exscript/protocols/drivers/ericsson_ban.py b/Exscript/protocols/drivers/ericsson_ban.py index 87077057..57d243fa 100644 --- a/Exscript/protocols/drivers/ericsson_ban.py +++ b/Exscript/protocols/drivers/ericsson_ban.py @@ -24,12 +24,14 @@ A driver for devices running Ericsson's Broadband Access Nodes (BAN) OS """ import re -from Exscript.protocols.drivers.driver import Driver + +from .driver import Driver _user_re = [re.compile(r'user:', re.I)] _password_re = [re.compile(r'pass:', re.I)] _prompt_re = [ - re.compile(r'[\r\n][\-\w+\.]+(?:\([^\)]+\))?[%#] ?$|(?:\(y/n\)\[n\])')] + re.compile(r'(?:\r\n|[\r\n])[\-\w+\.]+(?:\([^\)]+\))?[%#] ?$|(?:\(y/n\)\[n\])'), +] _error_re = [re.compile(r'\(error\)')] _ban_re = re.compile(r"BLM\d+ - Broadband Loop Multiplexer", re.I) diff --git a/Exscript/protocols/drivers/fortios.py b/Exscript/protocols/drivers/fortios.py index 0989c996..ea72e487 100644 --- a/Exscript/protocols/drivers/fortios.py +++ b/Exscript/protocols/drivers/fortios.py @@ -26,21 +26,22 @@ Created using a Fortigate device and FortiOS 5.0. """ import re -from Exscript.protocols.drivers.driver import Driver -_user_re = [re.compile(r'[\r\n]Username: $')] -_password_re = [re.compile(r'[\r\n]Password: $')] +from .driver import Driver + +_user_re = [re.compile(r'(?:\r\n|[\r\n])Username: $')] +_password_re = [re.compile(r'(?:\r\n|[\r\n])Password: $')] _prompt_re = [ - re.compile(r'^[a-zA-Z0-9-_ .]+ (?:\$|#) $'), # first prompt - re.compile(r'[\r\n][a-zA-Z0-9-_ .]+ (?:\([A-Za-z0-9_/.-]+\) )?(?:\$|#) $') + re.compile(r'(?:\r\n|[\r\n])?[a-zA-Z0-9-_ .]+ (?:\([A-Za-z0-9_/.-]+\) )?(?:\$|#) $'), ] _fortios_re = [ - _prompt_re[0], # first prompt - re.compile(r'^[a-zA-Z0-9-_.]+@[a-zA-Z0-9-_.]+\'s password:') + _prompt_re[0], + re.compile(r'^[a-zA-Z0-9-_.]+@[a-zA-Z0-9-_.]+\'s password:'), +] +_error_re = [ + re.compile(r'^Command fail.'), + re.compile(r'^object check operator error'), ] -_error_re = [re.compile(r'^Command fail.'), - re.compile(r'^object check operator error') - ] # example errors: # invalid netmask. diff --git a/Exscript/protocols/drivers/generic.py b/Exscript/protocols/drivers/generic.py index a326e73f..a7b54de1 100644 --- a/Exscript/protocols/drivers/generic.py +++ b/Exscript/protocols/drivers/generic.py @@ -23,7 +23,7 @@ """ The default driver that is used when the OS is not recognized. """ -from Exscript.protocols.drivers.driver import Driver +from .driver import Driver class GenericDriver(Driver): diff --git a/Exscript/protocols/drivers/hp_pro_curve.py b/Exscript/protocols/drivers/hp_pro_curve.py index f5d87d4a..42a52997 100644 --- a/Exscript/protocols/drivers/hp_pro_curve.py +++ b/Exscript/protocols/drivers/hp_pro_curve.py @@ -24,17 +24,22 @@ A driver for HP ProCurve switches. """ import re -from Exscript.protocols.drivers.driver import Driver -_user_re = [re.compile(r'[\r\n]Username: ?$')] -_password_re = [re.compile(r'[\r\n]Password: ?$')] -_prompt_re = [re.compile(r'[\r\n][\-\w+\.:/]+[>#] ?$')] +from .driver import Driver + +_user_re = [re.compile(r'(?:\r\n|[\r\n])Username: ?$')] +_password_re = [re.compile(r'(?:\r\n|[\r\n])Password: ?$')] +_prompt_re = [re.compile(r'(?:\r\n|[\r\n])[\-\w+\.:/]+[>#] ?$')] _error_re = [re.compile(r'(?:invalid|incomplete|ambiguous) input:', re.I)] -_login_fail_re = [re.compile(r'[\r\n]invalid password', re.I), - re.compile(r'unable to verify password', re.I), - re.compile(r'unable to login', re.I)] -_clean_res_re = [(re.compile(r'\x1bE'), "\r\n"), - (re.compile(r'(?:\x1b\[|\x9b)[\x30-\x3f]*[\x40-\x7e]'), "")] +_login_fail_re = [ + re.compile(r'(?:\r\n|[\r\n])invalid password', re.I), + re.compile(r'unable to verify password', re.I), + re.compile(r'unable to login', re.I), +] +_clean_res_re = [ + (re.compile(r'\x1bE'), "\r\n"), + (re.compile(r'(?:\x1b\[|\x9b)[\x30-\x3f]*[\x40-\x7e]'), ""), +] class HPProCurveDriver(Driver): @@ -46,7 +51,6 @@ def __init__(self): self.prompt_re = _prompt_re self.error_re = _error_re self.login_error_re = _login_fail_re - self.clean_res_re = _clean_res_re def check_head_for_os(self, string): if 'ProCurve' in string: @@ -59,7 +63,7 @@ def clean_response_for_re_match(self, response): start = response[:10].find('\x1b') if start != -1: response = response[start:] - for regexp, sub in self.clean_res_re: + for regexp, sub in _clean_res_re: response = regexp.subn(sub, response)[0] i = response.find('\x1b') if i > -1: diff --git a/Exscript/protocols/drivers/icotera.py b/Exscript/protocols/drivers/icotera.py index f0c31edc..9e0e0449 100644 --- a/Exscript/protocols/drivers/icotera.py +++ b/Exscript/protocols/drivers/icotera.py @@ -1,8 +1,31 @@ +# +# Copyright (C) 2010-2017 Samuel Abels +# The MIT License (MIT) +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files +# (the "Software"), to deal in the Software without restriction, +# including without limitation the rights to use, copy, modify, merge, +# publish, distribute, sublicense, and/or sell copies of the Software, +# and to permit persons to whom the Software is furnished to do so, +# subject to the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ A driver for Icotera CPE """ import re -from Exscript.protocols.drivers import Driver + +from .driver import Driver class IcoteraDriver(Driver): @@ -13,11 +36,11 @@ def __init__(self): """ Driver.__init__(self, 'icotera') self.user_re = [re.compile(r'user ?name: ?$', re.I)] - self.password_re = [re.compile(r'(?:[\r\n]Password: ?|last resort password:)$')] + self.password_re = [re.compile(r'(?:(?:\r\n|[\r\n])Password: ?|last resort password:)$')] self.prompt_re = [re.compile(r'.*?>\s*$')] self.error_re = [re.compile(r'ERROR')] def check_head_for_os(self, string): if 'ICOTERA' in string: return 80 - return 0 \ No newline at end of file + return 0 diff --git a/Exscript/protocols/drivers/ios.py b/Exscript/protocols/drivers/ios.py index f955592d..e7badcbc 100644 --- a/Exscript/protocols/drivers/ios.py +++ b/Exscript/protocols/drivers/ios.py @@ -23,21 +23,22 @@ """ A driver for Cisco IOS (not IOS XR). """ -from __future__ import absolute_import import re + from ..exception import InvalidCommandException from .driver import Driver - _user_re = [re.compile(r'user ?name: ?$', re.I)] -_password_re = [re.compile(r'(?:[\r\n][Pp]assword: ?|last resort password:)$')] -_tacacs_re = re.compile(r'(?:[\r\n]s|^s)\/key[\S ]+\r?%s' % _password_re[0].pattern) -_prompt_re = [re.compile(r'[\r\n][\-\w+\.:/]+(?:\([^\)]+\))?[>#] ?$')] -_error_re = [re.compile(r'%Error'), - re.compile(r'invalid input', re.I), - re.compile(r'(?:incomplete|ambiguous) command', re.I), - re.compile(r'connection timed out', re.I), - re.compile(r'[^\r\n]+ not found', re.I)] +_password_re = [re.compile(r'(?:(?:\r\n|[\r\n])[Pp]assword: ?|last resort password:)$')] +_tacacs_re = re.compile(r'(?:(?:\r\n|[\r\n])s|^s)\/key[\S ]+\r?%s' % _password_re[0].pattern) +_prompt_re = [re.compile(r'(?:\r\n|[\r\n])[\-\w+\.:/]+(?:\([^\)]+\))?[>#] ?$')] +_error_re = [ + re.compile(r'%Error'), + re.compile(r'invalid input', re.I), + re.compile(r'(?:incomplete|ambiguous) command', re.I), + re.compile(r'connection timed out', re.I), + re.compile(r'[^\r\n]+ not found', re.I), +] class IOSDriver(Driver): @@ -48,9 +49,9 @@ def __init__(self): self.password_re = _password_re self.prompt_re = _prompt_re self.error_re = _error_re - # Some Cisco IOS devices (e.g. WS-C3750) do not accept further login - # attempts after failing one. So in this hack, we - # re-connect after each attempt... + # Some Cisco IOS devices (e.g. WS-C3750) do not accept further SSH login + # attempts after failing one, but also do not disconnect each time. So + # in this hack, we re-connect after each attempt... self.reconnect_between_auth_methods = True def check_protocol_for_os(self, string): diff --git a/Exscript/protocols/drivers/ios_xr.py b/Exscript/protocols/drivers/ios_xr.py index 0a9c2750..0d519950 100644 --- a/Exscript/protocols/drivers/ios_xr.py +++ b/Exscript/protocols/drivers/ios_xr.py @@ -24,12 +24,15 @@ A driver for Cisco IOS XR. """ import re -from Exscript.protocols.drivers.driver import Driver -_user_re = [re.compile(r'[\r\n]Username: $')] -_password_re = [re.compile(r'[\r\n]Password: $')] +from .driver import Driver + +_user_re = [re.compile(r'(?:\r\n|[\r\n])Username: $')] +_password_re = [re.compile(r'(?:\r\n|[\r\n])Password: $')] _prompt_re = [ - re.compile(r'[\r\n\x00]RP/\d+/(?:RS?P)?\d+\/CPU\d+:[^#]+(?:\([^\)]+\))?#$')] + re.compile(r'(?:\r\n\x00?|[\r\n\x00])RP/\d+/(?:RS?P)?\d+\/CPU\d+:[^#]+(?:\([^\)]+\))?#$'), +] +_netconf_caps_re = [re.compile(r"Cisco-IOS-XR", re.I)] class IOSXRDriver(Driver): @@ -39,6 +42,7 @@ def __init__(self): self.user_re = _user_re self.password_re = _password_re self.prompt_re = _prompt_re + self.netconf_capabilities_re = _netconf_caps_re def check_response_for_os(self, string): if _prompt_re[0].search(string): diff --git a/Exscript/protocols/drivers/isam.py b/Exscript/protocols/drivers/isam.py index b0d13f37..783dff07 100644 --- a/Exscript/protocols/drivers/isam.py +++ b/Exscript/protocols/drivers/isam.py @@ -24,16 +24,20 @@ A driver for devices running ISAM (runs on Alcatel ISAM). """ import re -from Exscript.protocols.drivers.driver import Driver + +from .driver import Driver _user_re = [re.compile(r'login: ?', re.I)] -_password_re = [re.compile(r'[\r\n]password: ?', re.I)] -_prompt_re = [re.compile(r'[\r\n][\- +\d+\w+\.]+(?:\([^\)]+\))?[>#] ?')] -_error_re = [re.compile(r'%Error'), - re.compile(r'invalid token', re.I), - re.compile(r"command is not complete")] +_password_re = [re.compile(r'(?:\r\n|[\r\n])password: ?', re.I)] +_prompt_re = [re.compile(r'(?:\r\n|[\r\n])[\- +\d+\w+\.]+(?:\([^\)]+\))?[>#] ?')] +_error_re = [ + re.compile(r'%Error'), + re.compile(r'invalid token', re.I), + re.compile(r"command is not complete"), +] _isam_re = re.compile( - r"last login : \d{1,2}/\d{1,2}/\d{1,2} \d{1,2}:\d{1,2}:\d{1,2}") + r"last login : \d{1,2}/\d{1,2}/\d{1,2} \d{1,2}:\d{1,2}:\d{1,2}", +) class IsamDriver(Driver): diff --git a/Exscript/protocols/drivers/junos.py b/Exscript/protocols/drivers/junos.py index bade4b71..42447d50 100644 --- a/Exscript/protocols/drivers/junos.py +++ b/Exscript/protocols/drivers/junos.py @@ -24,7 +24,8 @@ A driver for devices running JunOS (by Juniper). """ import re -from Exscript.protocols.drivers.driver import Driver + +from .driver import Driver # JunOS prompt examples: # sab@DD-EA3> @@ -48,17 +49,18 @@ # pheller@fw1> # -_user_re = [re.compile(r'[\r\n](?:[\-\w+]+ )?login: $')] -_password_re = [re.compile(r'[\r\n](Local )?[Pp]assword: ?$')] +_user_re = [re.compile(r'(?:\r\n|[\r\n])(?:[\-\w+]+ )?login: $')] +_password_re = [re.compile(r'(?:\r\n|[\r\n])(Local )?[Pp]assword: ?$')] _mb = r'(?:\{master(?::\d+)?\}|\{backup(?::\d+)?\})' _ps = r'(?:\{primary:node\d+\}|\{secondary:node\d+\})' _re_re = r'(?:' + _mb + r'|' + _ps + r')' _edit = r'(?:\[edit[^\]\r\n]*\])' -_prefix = r'(?:[\r\n]+' + _re_re + r'?' + _edit + r'?)' -_prompt = r'[\r\n]+[\w\-\.]+@[\-\w+\.:]+[%>#] $' +_prefix = r'(?:(?:\r\n|[\r\n])+' + _re_re + r'?' + _edit + r'?)' +_prompt = r'(?:\r\n|[\r\n])+[\w\-\.]+@[\-\w+\.:]+[%>#] $' _prompt_re = [re.compile(_prefix + r'?' + _prompt)] _error_re = [re.compile(r'^(unknown|invalid|error|syntax error)', re.I)] _junos_re = re.compile(r'\bjunos\b', re.I) +_netconf_caps_re = [re.compile(r"juniper\.net", re.I)] class JunOSDriver(Driver): @@ -69,6 +71,7 @@ def __init__(self): self.password_re = _password_re self.prompt_re = _prompt_re self.error_re = _error_re + self.netconf_capabilities_re = _netconf_caps_re def check_head_for_os(self, string): if _junos_re.search(string): diff --git a/Exscript/protocols/drivers/junos_erx.py b/Exscript/protocols/drivers/junos_erx.py index 945aa97e..f8241a38 100644 --- a/Exscript/protocols/drivers/junos_erx.py +++ b/Exscript/protocols/drivers/junos_erx.py @@ -24,11 +24,12 @@ A driver for devices running Juniper ERX OS. """ import re -from Exscript.protocols.drivers.driver import Driver -from Exscript.protocols.drivers.ios import _prompt_re -_user_re = [re.compile(r'[\r\n]User: $')] -_password_re = [re.compile(r'[\r\n](Telnet password:|Password:) $')] +from .driver import Driver + +_user_re = [re.compile(r'(?:\r\n|[\r\n])User: $')] +_password_re = [re.compile(r'(?:\r\n|[\r\n])(Telnet password:|Password:) $')] +_prompt_re = [re.compile(r'(?:\r\n|[\r\n])[\-\w+\.:/]+(?:\([^\)]+\))?[>#] ?$')] _junos_re = re.compile(r'\bJuniper Networks\b', re.I) diff --git a/Exscript/protocols/drivers/mrv.py b/Exscript/protocols/drivers/mrv.py index d61fa632..3a7e418b 100644 --- a/Exscript/protocols/drivers/mrv.py +++ b/Exscript/protocols/drivers/mrv.py @@ -24,12 +24,13 @@ A driver for MRV Linux. """ import re -from Exscript.protocols.drivers.driver import Driver -_user_re = [re.compile(r'[\r\n]Authorized Login: $')] -_password_re = [re.compile(r'[\r\n]Password: $')] +from .driver import Driver + +_user_re = [re.compile(r'(?:\r\n|[\r\n])Authorized Login: $')] +_password_re = [re.compile(r'(?:\r\n|[\r\n])Password: $')] _mrv_re = re.compile(r'\bMRV\b') -_prompt_re = [re.compile(r'[\r\n]+[^$\r\n]+\$ ?$')] +_prompt_re = [re.compile(r'(?:\r\n|[\r\n])+[^$\r\n]+\$ ?$')] class MRVDriver(Driver): diff --git a/Exscript/protocols/drivers/nxos.py b/Exscript/protocols/drivers/nxos.py index ad09d5a8..d7792eac 100644 --- a/Exscript/protocols/drivers/nxos.py +++ b/Exscript/protocols/drivers/nxos.py @@ -21,19 +21,22 @@ # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ -A driver for Cisco Nexus OS (NXOS) +A driver for Cisco Nexus OS (NXOS). """ import re -from Exscript.protocols.drivers.driver import Driver + +from .driver import Driver _user_re = [re.compile(r'user ?name: ?$', re.I)] -_password_re = [re.compile(r'(?:[\r\n]Password: ?|last resort password:)$')] -_prompt_re = [re.compile(r'[\r\n\x00][\-\w+\.:/]+(?:\([^\)]+\))?[>#] ?$')] -_error_re = [re.compile(r'%Error'), - re.compile(r'invalid input', re.I), - re.compile(r'(?:incomplete|ambiguous) command', re.I), - re.compile(r'connection timed out', re.I), - re.compile(r'[^\r\n]+ not found', re.I)] +_password_re = [re.compile(r'(?:(?:\r\n|[\r\n])Password: ?|last resort password:)$')] +_prompt_re = [re.compile(r'(?:\r\n\x00?|[\r\n\x00])[\-\w+\.:/]+(?:\([^\)]+\))?[>#] ?$')] +_error_re = [ + re.compile(r'%Error'), + re.compile(r'invalid input', re.I), + re.compile(r'(?:incomplete|ambiguous) command', re.I), + re.compile(r'connection timed out', re.I), + re.compile(r'[^\r\n]+ not found', re.I), +] class NXOSDriver(Driver): diff --git a/Exscript/protocols/drivers/one_os.py b/Exscript/protocols/drivers/one_os.py index fb7787e2..8cc9c0ba 100644 --- a/Exscript/protocols/drivers/one_os.py +++ b/Exscript/protocols/drivers/one_os.py @@ -24,12 +24,13 @@ A driver for OneOS (OneAccess). """ import re -from Exscript.protocols.drivers.driver import Driver -_user_re = [re.compile(r'[\r\n]Username: ?$')] -_password_re = [re.compile(r'[\r\n]Password: ?$')] +from .driver import Driver + +_user_re = [re.compile(r'(?:\r\n|[\r\n])Username: ?$')] +_password_re = [re.compile(r'(?:\r\n|[\r\n])Password: ?$')] _first_prompt_re = re.compile(r'\r?\n\r?\n[\-\w+\.]+[>#]$') -_prompt_re = [re.compile(r'[\r\n][\-\w+\.]+(?:\([^\)]+\))?[>#] ?$')] +_prompt_re = [re.compile(r'(?:\r\n|[\r\n])[\-\w+\.]+(?:\([^\)]+\))?[>#] ?$')] class OneOSDriver(Driver): @@ -50,7 +51,7 @@ def init_terminal(self, conn): # TERMINAL WIDTH ONEOS5 try: conn.execute('stty columns 255') - except Exception: + except Exception: # noqa: BLE001, S110 pass def auto_authorize(self, conn, account, flush, bailout): diff --git a/Exscript/protocols/drivers/rios.py b/Exscript/protocols/drivers/rios.py index 21b4d5b0..1c53be1c 100644 --- a/Exscript/protocols/drivers/rios.py +++ b/Exscript/protocols/drivers/rios.py @@ -21,17 +21,20 @@ # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ -A driver for Riverbed RIOS +A driver for Riverbed RIOS. """ import re -from Exscript.protocols.drivers.driver import Driver + +from .driver import Driver _user_re = [re.compile(r'user ?name: ?$', re.I)] -_password_re = [re.compile(r'(?:[\r\n]Password: ?|last resort password:)$')] -_prompt_re = [re.compile(r'[\r\n][\-\w+\.:/]+\s?(?:\([^\)]+\))?\s+[>#]\s+$')] -_error_re = [re.compile(r'%Error'), - re.compile(r'% Unrecognized', re.I), - re.compile(r'% Inactive', re.I)] +_password_re = [re.compile(r'(?:(?:\r\n|[\r\n])Password: ?|last resort password:)$')] +_prompt_re = [re.compile(r'(?:\r\n|[\r\n])[\-\w+\.:/]+\s?(?:\([^\)]+\))?\s+[>#]\s+$')] +_error_re = [ + re.compile(r'%Error'), + re.compile(r'% Unrecognized', re.I), + re.compile(r'% Inactive', re.I), +] class RIOSDriver(Driver): diff --git a/Exscript/protocols/drivers/shell.py b/Exscript/protocols/drivers/shell.py index 585a0c76..0fb9926a 100644 --- a/Exscript/protocols/drivers/shell.py +++ b/Exscript/protocols/drivers/shell.py @@ -24,7 +24,8 @@ A generic shell driver that handles unknown unix shells. """ import re -from Exscript.protocols.drivers.driver import Driver + +from .driver import Driver _user_re = [re.compile(r'(user|login): $', re.I)] _password_re = [re.compile(r'Password: ?$')] diff --git a/Exscript/protocols/drivers/smart_edge_os.py b/Exscript/protocols/drivers/smart_edge_os.py index 2d7a55f1..5065ed1d 100644 --- a/Exscript/protocols/drivers/smart_edge_os.py +++ b/Exscript/protocols/drivers/smart_edge_os.py @@ -24,13 +24,15 @@ A driver for Redback Smart Edge OS. """ import re -from Exscript.protocols.drivers.driver import Driver -_user_re = [re.compile(r'[\r\n]login: ')] -_password_re = [re.compile(r'[\r\n]Password: $')] +from .driver import Driver + +_user_re = [re.compile(r'(?:\r\n|[\r\n])login: ')] +_password_re = [re.compile(r'(?:\r\n|[\r\n])Password: $')] _prompt_re = [ - re.compile(r'[\r\n]\x00?\[\w+\][\-\w+\.]+(?:\([^\)]+\))?[>#] ?$')] -_model_re = re.compile(r'[\r\n][^\r\n]+-se800[\r\n]') + re.compile(r'(?:\r\n|[\r\n])\x00?\[\w+\][\-\w+\.]+(?:\([^\)]+\))?[>#] ?$'), +] +_model_re = re.compile(r'(?:\r\n|[\r\n])[^\r\n]+-se800(?:\r\n|[\r\n])') class SmartEdgeOSDriver(Driver): diff --git a/Exscript/protocols/drivers/sp_vxoa.py b/Exscript/protocols/drivers/sp_vxoa.py index 63b08c07..d2eb986c 100644 --- a/Exscript/protocols/drivers/sp_vxoa.py +++ b/Exscript/protocols/drivers/sp_vxoa.py @@ -24,11 +24,12 @@ A driver for devices running VXOA (by Silver Peak). """ import re -from Exscript.protocols.drivers.driver import Driver -_user_re = [re.compile(r'[\r\n]login ?:as :')] -_password_re = [re.compile(r'[\r\n][Pp]assword: ?$')] -_prompt_re = [re.compile(r'[\r\n].*?[>#] ?$')] +from .driver import Driver + +_user_re = [re.compile(r'(?:\r\n|[\r\n])login ?:as :')] +_password_re = [re.compile(r'(?:\r\n|[\r\n])[Pp]assword: ?$')] +_prompt_re = [re.compile(r'(?:\r\n|[\r\n]).*?[>#] ?$')] _error_re = [re.compile(r'^(unknown|invalid|error|syntax error)', re.I)] _banner_re = re.compile(r'Silver[\r\n\s]Peak[\r\n\s]Systems[\r\n\s]Inc\.') _auth_banner_re = re.compile(r'Last login:\s') diff --git a/Exscript/protocols/drivers/sros.py b/Exscript/protocols/drivers/sros.py index 2ee1c76d..cc56095b 100644 --- a/Exscript/protocols/drivers/sros.py +++ b/Exscript/protocols/drivers/sros.py @@ -24,9 +24,10 @@ A driver for Alcatel SROS. """ import re -from Exscript.protocols.drivers.driver import Driver -_prompt_re = [re.compile(r'[\r\n][*]?(?:A|B):[\-\w\.>]+[#\$] ?$')] +from .driver import Driver + +_prompt_re = [re.compile(r'(?:\r\n|[\r\n])[*]?(?:A|B):[\-\w\.>]+[#\$] ?$')] class SROSDriver(Driver): @@ -44,3 +45,6 @@ def init_terminal(self, conn): conn.execute('environment no more') conn.execute('environment reduced-prompt 2') conn.execute('environment no saved-ind-prompt') + + def logout(self, conn): + conn.send('logout\r') diff --git a/Exscript/protocols/drivers/vrp.py b/Exscript/protocols/drivers/vrp.py index 96a2ccc4..9365cb18 100644 --- a/Exscript/protocols/drivers/vrp.py +++ b/Exscript/protocols/drivers/vrp.py @@ -24,13 +24,16 @@ A driver for devices running VRP (by Huawei). """ import re -from Exscript.protocols.drivers.driver import Driver + +from .driver import Driver _user_re = [re.compile(r'user ?name: ', re.I)] -_password_re = [re.compile(r'[\r\n]Password: $')] -_prompt_re = [re.compile(r'[\r\n][\<\[]\S+[\>\]] ?$', re.I)] -_huawei_re = [re.compile(r'\bhuawei\b', re.I), - re.compile(r'The current login time is.*', re.I)] +_password_re = [re.compile(r'(?:\r\n|[\r\n])Password: $')] +_prompt_re = [re.compile(r'(?:\r\n|[\r\n])[\<\[]\S+[\>\]] ?$', re.I)] +_huawei_re = [ + re.compile(r'\bhuawei\b', re.I), + re.compile(r'The current login time is.*', re.I), +] class VRPDriver(Driver): @@ -52,3 +55,6 @@ def check_head_for_os(self, string): def init_terminal(self, conn): conn.execute('screen-length 0 temporary') + + def logout(self, conn): + conn.send('quit\r') diff --git a/Exscript/protocols/drivers/vxworks.py b/Exscript/protocols/drivers/vxworks.py index 2d3cf655..c8789704 100644 --- a/Exscript/protocols/drivers/vxworks.py +++ b/Exscript/protocols/drivers/vxworks.py @@ -24,14 +24,18 @@ A driver for devices running Vxworks, which can be found on huawei5600T """ import re -from Exscript.protocols.drivers.driver import Driver + +from .driver import Driver _user_re = [re.compile(r'user ?name:', re.I)] _password_re = [re.compile(r'(?:User )?Password:', re.I)] _prompt_re = [ - re.compile(r'[\r\n][\-\w+\.]+(?:\([^\)]+\))?[>#] ?$|(?:\(y/n\)\[n\])')] -_error_re = [re.compile(r'%Error'), - re.compile(r'(?:(?:Unknown|ambiguous|Incomplete) command)|Failure|Parameter error', re.I)] + re.compile(r'(?:\r\n|[\r\n])[\-\w+\.]+(?:\([^\)]+\))?[>#] ?$|(?:\(y/n\)\[n\])'), +] +_error_re = [ + re.compile(r'%Error'), + re.compile(r'(?:(?:Unknown|ambiguous|Incomplete) command)|Failure|Parameter error', re.I), +] _huawei_re = re.compile(r"Huawei Integrated Access Software", re.I) diff --git a/Exscript/protocols/drivers/zte.py b/Exscript/protocols/drivers/zte.py index a4583197..b651200d 100644 --- a/Exscript/protocols/drivers/zte.py +++ b/Exscript/protocols/drivers/zte.py @@ -24,16 +24,20 @@ A driver for devices running Zte operating system. """ import re -from Exscript.protocols.drivers.driver import Driver + +from .driver import Driver _user_re = [re.compile(r'user ?name:', re.I), re.compile(r'login', re.I)] _password_re = [re.compile(r'(?:User )?Password:', re.I)] -_prompt_re = [re.compile( - r'[\r\n][\-\w+\.\(\)]+(?:\([^\)]+\))?[>#$] ?$|(?:\(y/n\)\[n\])'), - re.compile(r"[\r\n]Password: ?", re.I)] # password prompt to be used in privilege mode when it has -# a password different from the login password. -_error_re = [re.compile(r'%Error'), - re.compile(r'(?:Unrecognized|Incomplete) command', re.I), re.compile(r'Invalid input', re.I)] +_prompt_re = [ + re.compile(r'(?:\r\n|[\r\n])[\-\w+\.\(\)]+(?:\([^\)]+\))?[>#$] ?$|(?:\(y/n\)\[n\])'), + re.compile(r"(?:\r\n|[\r\n])Password: ?", re.I), # password prompt to be used in privilege mode +] +_error_re = [ + re.compile(r'%Error'), + re.compile(r'(?:Unrecognized|Incomplete) command', re.I), + re.compile(r'Invalid input', re.I), +] _zte_re = re.compile(r"ZTE", re.I) diff --git a/Exscript/protocols/drivers/zyxel.py b/Exscript/protocols/drivers/zyxel.py index d0ef5f67..894e608e 100644 --- a/Exscript/protocols/drivers/zyxel.py +++ b/Exscript/protocols/drivers/zyxel.py @@ -24,11 +24,13 @@ A driver for devices running Zyxel operating system. """ import re -from Exscript.protocols.drivers.driver import Driver -_prompt_re = [re.compile( - r'[\r\n][\-\w+\.\(\)]+(?:\([^\)]+\))?[>#$]( \x1b7)?$|(?:\(y/n\)\[n\])'), - re.compile(r"[\r\n]Password: ?", re.I)] # password prompt to be used in privilege mode when it has +from .driver import Driver + +_prompt_re = [ + re.compile(r'(?:\r\n|[\r\n])[\-\w+\.\(\)]+(?:\([^\)]+\))?[>#$]( \x1b7)?$|(?:\(y/n\)\[n\])'), + re.compile(r"(?:\r\n|[\r\n])Password: ?", re.I), # password prompt to be used in privilege mode +] _zyxel_re = re.compile(r"Copyright .* ZyXEL Communications Corp.", re.I) diff --git a/Exscript/protocols/dummy.py b/Exscript/protocols/dummy.py index dfda0c28..6679b1ef 100644 --- a/Exscript/protocols/dummy.py +++ b/Exscript/protocols/dummy.py @@ -23,14 +23,14 @@ """ A client that talks to a :class:`Exscript.emulators.VirtualDevice`. """ -from __future__ import absolute_import, unicode_literals -from future import standard_library -standard_library.install_aliases() -from builtins import str + +import logging + from ..emulators import VirtualDevice from .protocol import Protocol -from .exception import TimeoutException, DriverReplacedException, \ - ExpectCancelledException + +_logger = logging.getLogger(__name__) +_logger.addHandler(logging.NullHandler()) class Dummy(Protocol): @@ -46,13 +46,17 @@ def __init__(self, device=None, **kwargs): Also supports all keyword arguments that :class:`Protocol` supports. :keyword device: The :class:`Exscript.emulators.VirtualDevice` with - which to communicate. + which to communicate. If not given, a default device is created. """ Protocol.__init__(self, **kwargs) + # Module-Level + self._logger = _logger + # Internals + self._default_port = 54321 # virtual port for testing + self._init_done = False + # Public + self.connect_rate_limit = 0 self.device = device - self.init_done = False - self.cancel = False - self.response = None if not self.device: self.device = VirtualDevice('dummy', strict=False) @@ -63,17 +67,16 @@ def _expect_any(self, prompt_list, flush=True): self._doinit() # Cancelled by a callback during self._say(). - if self.cancel: - self.cancel = False + if self._expect_canceled: return -2, None, self.response # Look for a match in the buffer. for i, prompt in enumerate(prompt_list): - matches = prompt.search(str(self.buffer)) + matches = prompt.search(str(self._buffer)) if matches is not None: - self.response = self.buffer.head(matches.start()) + self.response = self._buffer.head(matches.start()) if flush: - self.buffer.pop(matches.end()) + self._buffer.pop(matches.end()) return i, matches, self.response # "Timeout". @@ -81,20 +84,17 @@ def _expect_any(self, prompt_list, flush=True): def _say(self, string): self._receive_cb(string) - self.buffer.append(string) - - def cancel_expect(self): - self.cancel = True + self._buffer.append(string) - def _connect_hook(self, hostname, port): + def _connect(self): # To more correctly mimic the behavior of a network device, we # do not send the banner here, but in authenticate() instead. - self.buffer.clear() + self._buffer.clear() return True def _doinit(self): - if not self.init_done: - self.init_done = True + if not self._init_done: + self._init_done = True self._say(self.device.init()) def _protocol_authenticate(self, user, password): @@ -103,34 +103,30 @@ def _protocol_authenticate(self, user, password): def _protocol_authenticate_by_key(self, user, key): self._doinit() - def send(self, data): - self._dbg(4, 'Sending %s' % repr(data)) - self._say(self.device.do(data)) + def _send(self, data): + response = self.device.do(data) + self._send_cb(data) + self._say(response) def _domatch(self, prompt, flush): # Wait for a prompt. result, match, self.response = self._expect_any(prompt, flush) if match: - self._dbg(2, "Got a prompt, match was %s" % repr(match.group())) - else: - self._dbg(2, "No prompt match") - - self._dbg(5, "Response was %s" % repr(str(self.buffer))) + self._dbg(2, "Got a prompt, match was: %s" % repr(match.group())) + self._dbg(4, "Response (flush=%s): %s" % (repr(flush), repr(self.response))) if result == -1: - error = 'Error while waiting for response from device' - raise TimeoutException(error) + self._raise_on_response_timeout() if result == -2: - if self.driver_replaced: - self.driver_replaced = False - raise DriverReplacedException() - else: - raise ExpectCancelledException() - + self._raise_on_canceled_expect() + if self.response is None: + self._raise_on_response_eof() return result, match - def close(self, force=False): + def _logout(self): + pass + + def _close(self, force=False): self._say('\n') - self.buffer.clear() - super(Dummy, self).close() + self._init_done = False diff --git a/Exscript/protocols/exception.py b/Exscript/protocols/exception.py index 7f835a19..4ce79f9e 100644 --- a/Exscript/protocols/exception.py +++ b/Exscript/protocols/exception.py @@ -77,3 +77,63 @@ class InvalidCommandException(ProtocolException): a string that looked like an error. """ pass + + +class RPCError(InvalidCommandException): + + """ + An exception that is raised when a NETCONF rpc-reply contains one or + more rpc-error elements with error severity. + """ + + def __init__( + self, message, *, element, type, tag, severity, app_tag='', path='', info='' + ): + """ + :param message: The primary rpc-error message (error-message). + :param element: The XML element of the rpc-error node. + :param type: The NETCONF error-type (e.g. 'application', 'protocol'). + :param tag: The NETCONF error-tag (e.g. 'invalid-value', 'bad-element'). + :param severity: The NETCONF error-severity ('error' or 'warning'). + :param app_tag: The optional data-model-specific error sub-tag. + :param path: The optional NETCONF error-path. + :param info: The optional NETCONF error-info node as XML string. + """ + msg = '' + if severity: + msg += f'{severity}: ' + msg += message or 'no message available' + + super().__init__(msg) + self.element = element + self.type = type or '' + self.tag = tag or '' + self.severity = severity or '' + self.message = message or '' + self.app_tag = app_tag or '' + self.path = path or '' + self.info = info or '' + self.errlist = [self] + + @classmethod + def _from_ncclient(cls, rpc_errors): + """ + Translates a list of ncclient `RPCError` objects (from RPCReply.errors) + into Exscript :class:`RPCError` instances. + """ + all_errors = [ + cls( + error.message, + element=error.xml, + type=error.type, + tag=error.tag, + severity=error.severity, + app_tag=error.app_tag, + path=error.path, + info=error.info, + ) + for error in rpc_errors + ] + primary = all_errors[0] + primary.errlist = all_errors + return primary diff --git a/Exscript/protocols/netconf.py b/Exscript/protocols/netconf.py new file mode 100644 index 00000000..9eaeac6a --- /dev/null +++ b/Exscript/protocols/netconf.py @@ -0,0 +1,473 @@ +# +# Copyright (C) 2010-2017 Samuel Abels +# The MIT License (MIT) +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files +# (the "Software"), to deal in the Software without restriction, +# including without limitation the rights to use, copy, modify, merge, +# publish, distribute, sublicense, and/or sell copies of the Software, +# and to permit persons to whom the Software is furnished to do so, +# subject to the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" +SSH2 Netconf support, based on ncclient and paramiko. +""" +import functools +import inspect +import logging +import re +import time + +from lxml.etree import XMLSyntaxError +from ncclient import NCClientError +from ncclient.manager import Manager, make_device_handler +from ncclient.operations.errors import TimeoutExpiredError +from ncclient.operations.rpc import RaiseMode, RPCReply +from ncclient.transport.ssh import SSHSession +from ncclient.xml_ import NCElement, to_ele, to_xml +from paramiko.ssh_exception import SSHException + +from ..util.xml import XMLValidationError, validate_xml +from .exception import ( + LoginFailure, + ProtocolException, + RPCError, +) +from .ssh2 import SSH2 + +_logger = logging.getLogger(__name__) +_logger.addHandler(logging.NullHandler()) + +# Matches a NETCONF 1.1 chunk header: LF HASH chunk-size LF (RFC 6242 §4.2). +_chunk_header_re = re.compile(r'\n#(\d+)\n') +_end_of_chunks_re = re.compile(r'\n##\n') + + +class Netconf(SSH2): + + """ + The SSH Netconf protocol adapter, based on the SSH2 adapter and ncclient. + """ + + def __init__(self, **kwargs): + SSH2.__init__(self, **kwargs) + # Module-Level + self._logger = _logger + # Internals + self._ncc_device_handler = None + self._ncc_session = None + self._ncc_manager = None + self._ncc_response = None + self._huge_tree = False + self._receives_echo = False # Netconf does not echo the sent RPC. + self._replicate = False + + @property + def huge_tree(self): + """Whether huge_tree XML parsing is enabled for RPC replies (default: False). + + Enabling this disables lxml's security limits for very large XML documents. + See the lxml documentation for details and associated risks. + """ + return self._huge_tree + + @huge_tree.setter + def huge_tree(self, value): + self._huge_tree = value + if self._ncc_manager is not None: + self._ncc_manager.huge_tree = value + + def autoinit(self): + """Netconf does not support terminal/shell initialization.""" + pass + + def get_prompt(self): + if self._manual_prompt_re: + return self._manual_prompt_re + return self.get_driver().netconf_prompt_re + + def get_error_prompt(self): + if self._manual_error_re: + return self._manual_error_re + return self.get_driver().netconf_error_re + + def get_client_capabilities(self): + """Returns the client capabilities sent in the `` message.""" + if self._ncc_manager is None: + return None + return list(self._ncc_manager.client_capabilities) + + def get_server_capabilities(self): + """Returns the server capabilities received in the `` message.""" + if self._ncc_manager is None: + return None + return list(self._ncc_manager.server_capabilities) + + def get_session_id(self): + """Returns the session ID received in the `` message.""" + if self._ncc_manager is None: + return None + return str(self._ncc_manager.session_id) + + def get_response_element(self): + """ + Returns the parsed XML element tree of the latest RPC reply. This might + be also some transformed XML structure, depending on specific device + handler implementations (look for `transform_reply()` definitions). + + This is coupled to the existence of a valid plaintext response in + self.response to enforce correct usage of the send()/execute() API. + Therefore this method could also return `None`. + + .. HINT:: + The first call after the login will not return the received `` + element tree although it is available as a plaintext response! This + is by design. Please use the get_server_capabilities() method to + access a parsed list of hello capabilities instead. + + :rtype: lxml.etree.Element | None + """ + if self.response is None: + return None # send() was called without expect_prompt() + if isinstance(self._ncc_response, RPCReply): + return to_ele(self._ncc_response.xml, huge_tree=self.huge_tree) + if isinstance(self._ncc_response, NCElement): + return to_ele(self._ncc_response.data_xml, huge_tree=self.huge_tree) + return None + + @staticmethod + def cleanup_response( + response, *, strip_command=False, strip_chars='\x00\x07\x08', strip_ansi=True + ): + # Netconf does not echo any commands, so there is nothing to strip. + return SSH2.cleanup_response( + response, + strip_command=False, + strip_chars=strip_chars, + strip_ansi=strip_ansi, + ) + + def _paramiko_channel(self): + # This is a bit different from the SSH2 adapter. Here we need to init + # the Netconf subsystem. We integrate ncclient's device params in the + # drivers to support customization. + driver = self.get_driver() + device_handler = make_device_handler(driver.netconf_device_params) + subsystems = device_handler.get_ssh_subsystem_names() + for subname in subsystems: + try: + self._channel = self._transport.open_session() + self._channel.invoke_subsystem(subname) + self._channel.set_name( + "%s-subsystem-%s" % (subname, self._channel.get_id()) + ) + except SSHException as e: + msg = 'Failed to open a channel with %s subsystem. (%s)' + self._dbg(1, msg % (repr(subname), e)) + if self._channel: + self._channel.close() + self._channel = None + else: + self._ncc_device_handler = device_handler + return + msg = 'Failed to open a channel with any subsystem: ' + ', '.join(subsystems) + raise LoginFailure(msg) + + def _init_ncclient(self): + # Bypasses ncclient's default SSH connection flow by injecting the + # already-authenticated paramiko transport and the open NETCONF channel + # directly into a fresh SSHSession, then calling _post_connect() to + # complete the exchange and start ncclient's IO thread. + if self._ncc_manager: + return + self._dbg(1, 'Initialize ncclient and hand over channel ownership.') + + # Build a bare SSHSession — no connect() call, we inject everything. + device_handler = self._ncc_device_handler + session = SSHSession(device_handler) + session._host = self.host + session._transport = self._transport + session._channel = self._channel + session._channel_id = self._channel.get_id() + session._channel_name = self._channel.get_name() + session._connected = True + session._closing.clear() + + # Controls the behavior of the replication wrappers. The replication is + # needed as ncclient owns the channel within its IO thread. So we need + # to somehow fetch this raw data for display, logging and raw response + # buffering/matching. + self._replicate = True + + # Setup the channel receive wrapper. + recv = self._channel.recv + + @functools.wraps(recv) + def recv_wrapper(*args, **kwargs): + data = recv(*args, **kwargs) + if not self._replicate: + return data + if data: + string_data = data.decode(self.encoding) + self._receive_cb(string_data) + self._buffer.append(string_data) + else: + self._dbg(1, 'channel.recv() returned EOF') + self._replicate = False + return data + + self._channel.recv = recv_wrapper + + # Setup the channel send wrapper. This is needed as Netconf does not + # echo the sent data back to the client known from CLI shells. + send = self._channel.send + param = next(iter(inspect.signature(send).parameters)) # first param + + @functools.wraps(send) + def send_wrapper(*args, **kwargs): + count = send(*args, **kwargs) + if not self._replicate: + return count + data = kwargs[param] if param in kwargs else args[0] + if count is None: + string_data = data.decode(self.encoding) + self._send_cb(string_data) + elif count > 0: + string_data = data[:count].decode(self.encoding) + self._send_cb(string_data) + else: + self._dbg(1, 'channel.send() returned EOF') + self._replicate = False + return count + + self._channel.send = send_wrapper + + # Setup the RPC relevant to_xml wrapper so that ncclient's RPCs are + # always send (and logged by Exscript) in a pretty formatted style. + # This is quite hacky, but the only way to achieve it on the fly + # without additional XML parsing and reformatting. + import ncclient.operations.retrieve as retrieve_module + import ncclient.operations.rpc as rpc_module + import ncclient.transport.session as session_module + + @functools.wraps(to_xml) + def to_xml_wrapper(*args, **kwargs): + kwargs['pretty_print'] = True + return to_xml(*args, **kwargs) + + retrieve_module.to_xml = to_xml_wrapper # for retrieved RPC replies + rpc_module.to_xml = to_xml_wrapper # for all kinds of sent RPCs + session_module.to_xml = to_xml_wrapper # where client hello is build + + # Starts IO thread, sends client , consumes server . + self._buffer.clear() # keeps the first self.response clean + session._post_connect(self.get_timeout()) + + # Selects the driver/device-specific XML parser for all subsequent RPCs. + session.parser = device_handler.get_xml_parser(session) + + # Finally the manager is in charge of the channel. + self._ncc_session = session + self._ncc_manager = Manager( + session, + device_handler, + timeout=self.get_timeout(), + raise_mode=RaiseMode.NONE, # raising custom RPCError later... + ) + self._ncc_manager.huge_tree = self.huge_tree # ensure synced flags + + # Exscript needs to finalize the initialization. + self._os_guesser.check_netconf_capabilities(self.get_server_capabilities()) + self._update_auto_driver() + self.expect_prompt() # consumes the server from the buffer + + def login(self, account=None, app_account=None, flush=True): + # Simplified procedure for Netconf to suppress driver auto-authorize. + with self._get_account(account) as account: + if app_account is None: + app_account = account + self.authenticate(account, app_account, flush=False) + self.app_authorize() # just to set the flag + + def authenticate(self, account=None, app_account=None, flush=True): + super(Netconf, self).authenticate(account, app_account, flush=flush) + # Authentication is complete and the subsystem channel is open. The + # server waits unread in the channel buffer. Hand the channel to + # ncclient now — it will consume the and start its IO thread. + self._init_ncclient() + + def app_authenticate(self, account=None, flush=True, bailout=False): + """Netconf does not support app-authentication.""" + self._app_authenticated = True + + def app_authorize(self, account=None, flush=True, bailout=False): + """Netconf does not support app-authorization.""" + self._app_authorized = True + + def auto_app_authorize(self, account=None, flush=True, bailout=False): + """Netconf does not support app-authorization.""" + self.app_authorize() # just to set the flag + + @staticmethod + def reassemble_chunks(response): + """ + Reassembles a NETCONF 1.1 chunked response into plain content. + + Each chunk is introduced by a header `LF HASH chunk-size LF` where the + size is the exact byte count of the following chunk (RFC 6242 §4.2). + The end-of-chunks marker `LF HASH HASH LF` gets stripped finally. Useful + to also reassemble chunks in received data from events. + + Raises ValueError on unexpected trailing bytes after the last chunk. + + :param response: The chunked response to reassemble. + :rtype: str + :return: The reassembled response. + """ + match = _chunk_header_re.match(response) + if not match: + return response + assembled = '' + while match: + size = int(match.group(1)) + response = response[match.end():] # consume the chunk header + assembled += response[:size] # extract exactly size chars of content + response = response[size:] # consume the chunk content + match = _chunk_header_re.match(response) + if response: + # Internally this would be already consumed as the prompt. + match = _end_of_chunks_re.match(response) + if match: + response = response[match.end():] # consume the end-of-chunks marker + if response: + # This should only happen if we have internal issues with the prompt + # or buffer handling before this reassembly is called. ncclient + # should have caught buggy devices already at the transport layer. + msg = 'Unexpected trailing bytes after chunk reassembly: %s' + raise ValueError(msg % repr(response[:50] + '...')) + return assembled + + def _domatch(self, prompt, flush): + # This mimics the behavior of CLI-driven prompt matching to keep API + # parity with the Telnet and SSH2 adapter. + deadline = time.monotonic() + self.get_timeout() + while not self._expect_canceled: + n, match, search_window = self._domatch_buffer(prompt) + if not match: + # This is just a safety-net as the buffer is always filled by + # the replication wrapper and ncclient's blocking dispatch. + if time.monotonic() > deadline: + self._raise_on_response_timeout() + time.sleep(0.05) + continue + self.response = self._domatch_extract(match, len(search_window), flush) + self.response = self.reassemble_chunks(self.response) + return n, match + self._raise_on_canceled_expect() + + def _send(self, data): + self._ncc_response = None + self._validate_rpc_body(data) + try: + element = to_ele(data) + self._ncc_response = self._ncc_manager.dispatch(element) + # wrappers handle the self.data_sent_event + except TimeoutExpiredError: + self._raise_on_response_timeout() + except NCClientError as e: + msg = 'ncclient related error (%s)' % str(e) + raise ProtocolException(msg) + except Exception: + self._dbg(1, 'Error while writing to connection') + raise + self._dbg(4, 'Received response type: %s' % type(self._ncc_response).__name__) + + def _validate_rpc_body(self, data): + _wrapper_start_tag = '\n' + _wrapper_end_tag = '\n' + data = f'{_wrapper_start_tag}{data}{_wrapper_end_tag}' + try: + validate_xml(data, silent=False) + except UnicodeEncodeError as exc: + # Fix the absolute character offsets. + exc.start -= len(_wrapper_start_tag) + exc.end -= len(_wrapper_start_tag) + raise XMLValidationError( + 'XML validation of the RPC command has failed. ' + f'The error was caused by:\n{type(exc).__name__}: {exc}', + xmlstring=data, + ) + except XMLSyntaxError as exc: + err = XMLValidationError( + 'XML validation of the RPC command has failed. ' + f'The error was caused by:\n{type(exc).__name__}: {exc}', + xmlstring=data, + ) + err.add_note( + 'Note: Reported line numbers are shifted by +1 due to the ' + 'synthetic \\n...\\n wrapper used for validation.' + ) + raise err + + def _raise_on_response_with_errors(self): + # For device handlers that apply XSLT (e.g. Junos) or any transformation, + # dispatch() returns NCElement instead of RPCReply. + response = self._ncc_response + rpc_reply = getattr(response, '_NCElement__result', response) + for error in rpc_reply.errors: + if error.severity == 'error': + raise RPCError._from_ncclient(rpc_reply.errors) + + def _interact(self, key_handlers=None, handle_window_size=True): + # Netconf does not support interactive shells. User should use + # native `ssh HOST -p 830 -s netconf` for an interactive session. + raise NotImplementedError() + + def _logout(self): + try: + self._ncc_manager.close_session() + except Exception: + pass + + def _close(self, force=False): + # Clear the ncclient manager reference. + if self._ncc_manager: + self._ncc_manager = None + + # Signal ncclient's IO thread that this is an intentional close. + # Must happen before the channel is shut down so the thread treats the + # subsequent empty recv() as expected (breaks cleanly, no SessionCloseError). + ncc_session = self._ncc_session + if self._ncc_session: + try: + self._ncc_session._closing.set() + except Exception: + pass + self._ncc_session = None + + # Disable the replication wrappers. + self._replicate = False + + # SSH2 closes the paramiko channel, transport, and socket. + # Closing the channel causes channel.recv() to return b'', which — + # combined with _closing being set — causes the IO thread to exit. + super(Netconf, self)._close(force=force) + + # Wait for ncclient's IO thread to fully terminate. + # Must be after super() so the channel is already closed when we join. + if ncc_session: + try: + ncc_session.join(timeout=0.3) + except Exception: + pass diff --git a/Exscript/protocols/osguesser.py b/Exscript/protocols/osguesser.py index 6bf68446..f97c8348 100644 --- a/Exscript/protocols/osguesser.py +++ b/Exscript/protocols/osguesser.py @@ -20,36 +20,60 @@ # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -from __future__ import print_function -from builtins import object -from Exscript.protocols.drivers import drivers +""" +OS guesser for any protocol to auto-select a device driver. +""" +import logging -class OsGuesser(object): +from .drivers import drivers + +_logger = logging.getLogger(__name__) +_logger.addHandler(logging.NullHandler()) + + +class OsGuesser: """ - The OsGuesser monitors everything that happens on a Protocol, - and attempts to collect data out of the network activity. - It watches for specific patterns in the network traffic to decide - what operating system a connected host is running. - It is completely passive, and attempts no changes on the protocol - adapter. However, the protocol adapter may request information - from the OsGuesser, and perform changes based on the information - provided. + The OsGuesser monitors everything that happens on a Protocol, and attempts to + collect data out of the network activity. It watches for specific patterns in the + network traffic to decide what operating system a connected host is running. It is + completely passive, and attempts no changes on the protocol adapter. However, the + protocol adapter may request information from the OsGuesser, and perform changes + based on the information provided. """ def __init__(self): - self.info = {} - self.debug = False - self.protocol_os_map = [d._check_protocol for d in drivers] - self.auth_os_map = [d._check_head for d in drivers] - self.os_map = [d._check_response for d in drivers] - self.auth_buffer = '' - self.set('os', 'unknown', 0) - - def reset(self, auth_buffer=''): - self.__init__() - self.auth_buffer = auth_buffer + self._logger = _logger + self._buffer_limit = 512 + self._auth_os_map = [d._check_head for d in drivers] + self._netconf_os_map = [d._check_netconf for d in drivers] + self._protocol_os_map = [d._check_protocol for d in drivers] + self._response_os_map = [d._check_response for d in drivers] + self.debug = 0 + self._init() + + def _init(self): + self._auth_buffer = '' + self._protocol_buffer = '' + self._response_buffer = '' + self._info = {"os": ("unknown", 0)} + + def _dbg(self, level, msg, exc_info=None): + if level > self.debug: + return + self._logger.debug(msg, exc_info=exc_info) + + def reset(self, auth_buffer='', protocol_buffer='', response_buffer=''): + """ + Reset the guesser state and optionally pre-populate the monitoring buffers. + """ + msg = "Reset with new buffers: auth=%r, protocol=%r, response=%r" + self._dbg(2, msg % (auth_buffer, protocol_buffer, response_buffer)) + self._init() + self._auth_buffer = auth_buffer + self._protocol_buffer = protocol_buffer + self._response_buffer = response_buffer def set(self, key, value, confidence=100): """ @@ -58,11 +82,13 @@ def set(self, key, value, confidence=100): """ if value is None: return - if key in self.info: - old_confidence, old_value = self.info.get(key) - if old_confidence >= confidence: - return - self.info[key] = (confidence, value) + old_value, old_confidence = self._info.get(key, (None, None)) + if old_confidence is not None and old_confidence >= confidence: + return + self._dbg(2, "Higher confidence for %s: %s with %s -> %s with %s" % ( + key, old_value, old_confidence, value, confidence + )) + self._info[key] = (value, confidence) def set_from_match(self, key, regex_list, string): """ @@ -93,30 +119,55 @@ def get(self, key, confidence=0): Returns the info with the given key, if it has at least the given confidence. Returns None otherwise. """ - if key not in self.info: + if key not in self._info: return None - conf, value = self.info.get(key) + value, conf = self._info.get(key) if conf >= confidence: return value return None - def data_received(self, data, app_authentication_done): - # If the authentication procedure is complete, use the normal - # "runtime" matchers. - if app_authentication_done: + def check_received_data(self, data, authenticated): + """ + Callback to search the received data for matching patterns. + + The `authenticated` flag controls whether to also evaluate received data + from the authentication phase. + """ + if not data: + return + if authenticated: + self._response_buffer += data # Stop looking if we are already 80 percent certain. if self.get('os', 80) in ('unknown', None): - self.set_from_match('os', self.os_map, data) + self._dbg(3, "Matching received data: %r" % self._response_buffer) + self.set_from_match('os', self._response_os_map, self._response_buffer) + self._response_buffer = self._response_buffer[-self._buffer_limit:] return - # Else, check the head that we collected so far. - self.auth_buffer += data - if self.debug: - print("DEBUG: Matching buffer:", repr(self.auth_buffer)) - self.set_from_match('os', self.auth_os_map, self.auth_buffer) - self.set_from_match('os', self.os_map, self.auth_buffer) - - def protocol_info(self, data): - if self.debug: - print("DEBUG: Protocol info:", repr(data)) - self.set_from_match('os', self.protocol_os_map, data) + self._auth_buffer += data + self._dbg(3, "Matching authentication buffer: %r" % self._auth_buffer) + self.set_from_match('os', self._auth_os_map, self._auth_buffer) + self.set_from_match('os', self._response_os_map, self._auth_buffer) + self._auth_buffer = self._auth_buffer[-self._buffer_limit:] + + def check_protocol_info(self, data): + """ + Callback to search the received protocol information for matching patterns. + """ + if not data: + return + self._protocol_buffer += data + self._dbg(3, "Matching protocol info: %r" % self._protocol_buffer) + self.set_from_match('os', self._protocol_os_map, self._protocol_buffer) + self._protocol_buffer = self._protocol_buffer[-self._buffer_limit:] + + def check_netconf_capabilities(self, capabilities): + """ + Callback to search the received Netconf capabilities for matching patterns. + """ + if not capabilities: + return + if not isinstance(capabilities, str): + capabilities = "\n".join(capabilities) + self._dbg(3, "Matching Netconf capabilities: %r" % capabilities) + self.set_from_match('os', self._netconf_os_map, capabilities) diff --git a/Exscript/protocols/protocol.py b/Exscript/protocols/protocol.py index e4de3da1..de12cf0a 100644 --- a/Exscript/protocols/protocol.py +++ b/Exscript/protocols/protocol.py @@ -23,29 +23,37 @@ """ An abstract base class for all protocols. """ -from __future__ import absolute_import, unicode_literals -from future import standard_library -standard_library.install_aliases() -from builtins import object + +import errno +import logging +import os import re -import sys import select -import socket import signal -import errno -import os -from io import StringIO +import socket +import sys +import threading +import time from functools import partial -from ..util.impl import Context, _Context +from io import StringIO + from ..util.buffer import MonitoredBuffer +from ..util.cast import to_regexs from ..util.crypt import otp from ..util.event import Event -from ..util.cast import to_regexs +from ..util.impl import Context, _Context from ..util.tty import get_terminal_size -from .drivers import driver_map, Driver +from ..util.url import _WELL_KNOWN_PORTS +from .drivers import Driver, driver_map +from .exception import ( + DriverReplacedException, + ExpectCancelledException, + InvalidCommandException, + LoginFailure, + ProtocolException, + TimeoutException, +) from .osguesser import OsGuesser -from .exception import InvalidCommandException, LoginFailure, \ - TimeoutException, DriverReplacedException, ExpectCancelledException try: import termios @@ -54,10 +62,31 @@ except ImportError: _have_termios = False +_logger = logging.getLogger(__name__) +_logger.addHandler(logging.NullHandler()) + +_ansi_escape_re = re.compile( + r'\x1b\[' # CSI: ESC[ + + r'[\x30-\x3f]*' # PARAMETER: 0–9:;<=>? + + r'[\x20-\x2f]*' # INTERMEDIATE: !"#$%&'( )*+,-./ + + r'[\x40-\x7e]' # FINAL: @A–Z[\]^_`a–z{|}~ + + r'|\x1b' # alternate lone ESC +) _skey_re = re.compile(r'(?:s\/key|otp-md4) (\d+) (\S+)(?=\s|[\r\n])') +# Needed on module level as each host connection has its own thread. +_connection_monitor = {} +_connection_monitor_lock = threading.RLock() + +# NOTE: Encoding/Decoding non-ASCII byte-streams might not work properly. +# Right now the default encoding is latin-1 and most devices stick to 7 bit +# ASCII, even in Netconf XML/RPC (usually UTF-8). ASCII is a subset of UTF-8. +# This will start to break if a device sends or expects true UTF-8 encoded +# data, which is not ASCII. Then byte-stream decoding of stdin and the channel +# should be modernized to use an incremental decoder (see "codecs" package). -class Protocol(object): + +class Protocol: """ This is the base class for all protocols; it defines the common portions @@ -68,8 +97,8 @@ class Protocol(object): interchangeably without changing any other code. In order to achieve this, the main challenge are the differences - arising from the authentication methods that are used. - The reason is that many devices may support the following variety + arising from the authentication methods that are used. The reason + is that many devices may support the following variety of authentication/authorization methods: 1. Protocol level authentication, such as SSH's built-in @@ -96,9 +125,8 @@ class Protocol(object): 3. App level authorization: In order to implement the AAA protocol, some devices ask for two separate app-level logins, whereas the first serves to authenticate the user, and the second serves to - authorize him. - App-level authorization may support the same methods as app-level - authentication: + authorize him. App-level authorization may support the same + methods as app-level authentication: - A1: password only - A2: username @@ -118,6 +146,12 @@ class Protocol(object): - a1 - a3: optional - A1 - A3: optional + - Netconf: + + - p1 - p5: optional (derived from SSH) + - a1 - a3: never (not supported) + - A1 - A3: never (not supported) + To achieve authentication method compatibility across different protocols, we must hide all this complexity behind one single API call, and figure out which ones are supported. @@ -156,27 +190,36 @@ class Protocol(object): # Protocol level authentication. conn.protocol_authenticate(...) + # App-level authentication. conn.app_authenticate(...) + # App-level authorization. - conn.app_authorize(...) + conn.app_authorize(...) via conn.auto_app_authorize(...) The code produces the following result:: Telnet: - conn.protocol_authenticate -> NOP + conn.protocol_authenticate -> NOOP conn.app_authenticate -> waits for username or password prompt, authenticates, returns after a CLI prompt was seen. - conn.app_authorize - -> calls driver.enable(), waits for username or password - prompt, authorizes, returns after a CLI prompt was seen. + conn.app_authorize (via conn.auto_app_authorize) + -> delegates to driver.auto_authorize + -> calls specific implementation or conn.app_authorize + -> waits for username or password prompt, authorizes, + returns after a CLI prompt was seen. SSH: conn.protocol_authenticate -> authenticates using user/key/password conn.app_authenticate -> like Telnet conn.app_authorize -> like Telnet + Netconf: + conn.protocol_authenticate -> derived from SSH + conn.app_authenticate -> NOOP (not supported) + conn.app_authorize -> NOOP (not supported) + We can see the following: - protocol_authenticate() must not wait for a prompt, because else @@ -193,23 +236,22 @@ class Protocol(object): any authentication; i.e., when app_authenticate() is not called. - Since the prompt must still be contained in the buffer, - conn.driver.app_authorize() needs to eat it before it sends the - command for starting the authorization procedure. - - This has a drawback - if a user attempts to call app_authorize() - at a time where there is no prompt in the buffer, it would fail. - So we need to eat the prompt only in cases where we know that - auto_app_authorize() will attempt to execute a command. Hence - the driver requires the Driver.supports_auto_authorize() method. + conn.driver.auto_authorize() would somehow need to eat it before it + sends the command for starting the authorization procedure. This + will be handled internally by login(), but only if the call to + Driver.supports_auto_authorize() evaluates to `True` for a specific + driver. Otherwise the driver must fall back to app_authorize(), which + consumes the prompt internally. - However, app_authorize() must not eat the CLI prompt that follows. + If the user wants to handle app authorization manually, one may rely + on authenticate() instead of login(), which does not authorize + automatically. - Once all logins are processed, it makes sense to eat the prompt - depending on the wait parameter. Wait should default to True, + depending on the `flush` parameter. It should default to True, because it's better that the connection stalls waiting forever, than to risk that an error is not immediately discovered due to - timing issues (this is a race condition that I'm not going to - detail here). + timing issues (this is a race condition that is not detailed here). """ def __init__(self, @@ -217,35 +259,49 @@ def __init__(self, stdout=None, stderr=None, debug=0, + connect_rate_limit=60, connect_timeout=30, timeout=30, logfile=None, termtype='dumb', - verify_fingerprint=True, + disabled_algorithms=None, # SSH2 only + preferred_authentications=None, # SSH2 only + verify_fingerprint=True, # SSH2 only account_factory=None, - banner_timeout=20, + banner_timeout=20, # SSH2 only encoding='latin-1'): """ Constructor. + The following events are provided: - data_received_event: A packet was received from the connected host. + - data_sent_event: A packet was sent to the connected host. - otp_requested_event: The connected host requested a one-time-password to be entered. - :keyword driver: Driver()|str + :type driver: Driver()|str|None + :keyword driver: The manually picked device driver. :keyword stdout: Where to write the device response. Defaults to an in-memory buffer. :keyword stderr: Where to write debug info. Defaults to stderr. :keyword debug: An integer between 0 (no debugging) and 5 (very verbose debugging) that specifies the amount of debug info sent to the terminal. The default value is 0. + :keyword connect_rate_limit: Connection rate-limit per minute to avoid + hitting rate limits of the host in case of multiple parallel connections. :keyword connect_timeout: Timeout for the initial TCP connection attempt :keyword timeout: See set_timeout(). The default value is 30. :keyword logfile: A file into which a log of the conversation with the device is dumped. :keyword termtype: The terminal type to request from the remote host, e.g. 'vt100'. + :type disabled_algorithms: dict[str, list[str]]|None + :keyword disabled_algorithms: Dictionary to disable SSH algorithms via + 'paramiko.Transport(disabled_algorithms=...)' constructor. + :type preferred_authentications: list[str]|None + :keyword preferred_authentications: List of preferred SSH authentication + methods, referring to SSH config option 'PreferredAuthentications'. :keyword verify_fingerprint: Whether to verify the host's fingerprint. :keyword account_factory: A function that produces a new :class:`Account`. :type banner_timeout: bool @@ -253,59 +309,65 @@ def __init__(self, :type encoding: str :keyword encoding: The encoding of data received from the remote host. """ + # Module-Level + self._connection_monitor = _connection_monitor + self._connection_monitor_lock = _connection_monitor_lock + self._logger = _logger # must be set by subclass + # Internals + self._name = type(self).__name__.lower().strip('0123456789') + self._buffer = MonitoredBuffer() + self._os_guesser = OsGuesser() + self._os_guesser.debug = debug + self._default_port = _WELL_KNOWN_PORTS.get(self._name) + self._manual_user_re = None + self._manual_password_re = None + self._manual_prompt_re = None + self._manual_error_re = None + self._manual_login_error_re = None + self._manual_driver = None + self._receives_echo = True # does the remote shell echo sent commands? + self._write_lock = threading.Lock() + # Init remaining per-session attributes. + self._init() + # Public self.data_received_event = Event() + self.data_sent_event = Event() self.otp_requested_event = Event() - self.os_guesser = OsGuesser() - self.auto_driver = driver_map[self.guess_os()] - self.proto_authenticated = False - self.app_authenticated = False - self.app_authorized = False - self.manual_user_re = None - self.manual_password_re = None - self.manual_prompt_re = None - self.manual_error_re = None - self.manual_login_error_re = None - self.driver_replaced = False self.host = None self.port = None - self.last_account = None - self.termtype = termtype - self.verify_fingerprint = verify_fingerprint - self.manual_driver = None - self.debug = debug + self.account_factory = account_factory + self.connect_rate_limit = connect_rate_limit self.connect_timeout = connect_timeout - self.timeout = timeout + self.debug = debug + self.encoding = encoding self.logfile = logfile self.response = None - self.buffer = MonitoredBuffer() - self.account_factory = account_factory - self.banner_timeout = banner_timeout - self.encoding = encoding + # FIXME: Remove self.send_data in favor of self.data_sent_event! + # Right now this "buffer" is only used in interactive sessions but grows + # indefinitely, which is a memory leak. self.send_data = None - if stdout is None: - self.stdout = StringIO() - else: - self.stdout = stdout - if stderr is None: - self.stderr = sys.stderr - else: - self.stderr = stderr - if logfile is None: - self.log = None - else: - self.log = open(logfile, 'a') + self.stdout = stdout + self.stderr = stderr + self.termtype = termtype + self.timeout = timeout - # set manual_driver + # SSH2: + self.banner_timeout = banner_timeout + self.disabled_algorithms = disabled_algorithms + self.preferred_authentications = preferred_authentications + self.verify_fingerprint = verify_fingerprint + + # Set manual_driver: if driver is not None: if isinstance(driver, str): if driver in driver_map: - self.manual_driver = driver_map[driver] + self._manual_driver = driver_map[driver] else: - self._dbg(1, 'Invalid driver string given. Ignoring...') + self._dbg(1, 'Invalid driver string given. Ignoring.') elif isinstance(driver, Driver): - self.manual_driver = driver + self._manual_driver = driver else: - self._dbg(1, 'Invalid driver given. Ignoring...') + self._dbg(1, 'Invalid driver given. Ignoring.') def __copy__(self): """ @@ -329,36 +391,178 @@ def __deepcopy__(self, memo): """ return self - def _driver_replaced_notify(self, old, new): - self.driver_replaced = True - self.cancel_expect() - msg = 'Protocol: driver replaced: %s -> %s' % (old.name, new.name) - self._dbg(1, msg) + def __enter__(self): + """ + Enter the runtime context related to this object. + """ + return self - def _receive_cb(self, data, remove_cr=True): - # Clean the data up. - if remove_cr: - text = data.replace('\r', '') - else: - text = data + def __exit__(self, exc_type, exc_value, traceback): + """ + Ensures to close the connection when context is left. - # Write to a logfile. - self.stdout.write(text) - self.stdout.flush() - if self.log is not None: - self.log.write(text) + Closes the session gracefully with a logout, but if an exception was + raised, the connection will be closed forcefully. + """ + self.close(force=exc_type is not None) + + def __repr__(self): + clsname = type(self).__name__ + host = self.host or 'unknown' + port = self.port or self._default_port + state = 'connected' if self.connected else 'disconnected' + return f'<{clsname}({host}:{port} [{state}]) at {hex(id(self))}>' + + @property + def connected(self): + """ + Whether the connection to the remote host is currently open (read-only). + + :rtype: bool + :return: True if connected, False otherwise. + """ + return self._connected + + @property + def logfile(self): + """ + The path of the current logfile, or None if logging to file is disabled. + This also maintains any open logfile descriptor internally, especially + when a new filename is set. + + :rtype: str|None + :return: The logfile path. + """ + return self._logfile + + @logfile.setter + def logfile(self, value) -> None: + with self._write_lock: + if getattr(self, '_log', None): + try: + self._log.close() + except Exception: + pass + self._logfile = value + self._log = open(self._logfile, 'a') if value and self.connected else None + + @property + def os_guesser(self): + """ + The OS guesser used to auto-detect the remote host's OS/driver (read-only). + + Normally there should be no need to access the guesser. Any changes to + its state may influence the outcome of the auto-detection in a negative + way. If you need to pin a driver, use the set_driver() method. + + :rtype: OsGuesser + :return: The OS guesser instance. + """ + return self._os_guesser + + @property + def stdout(self): + """ + The stream to which device responses are written. Defaults to an internal + StringIO buffer which will be cleared when the connection is closed. + + :rtype: file-like object + :return: The current stdout stream. + """ + return self._stdout + + @stdout.setter + def stdout(self, value): + with self._write_lock: + if value is None: + self._stdout = StringIO() + self._stdout_owner = True + else: + self._stdout = value + self._stdout_owner = False + + @property + def stderr(self): + """ + The stream to which the debug/error output is written. Defaults to + `sys.stderr`. + + :rtype: file-like object + :return: The current stderr stream. + """ + return self._stderr + + @stderr.setter + def stderr(self, value): + with self._write_lock: + self._stderr = sys.stderr if value is None else value + + def _dbg(self, level, msg, exc_info=None): + if level > self.debug: + return + msg = '[%s] %s' % (self.get_driver().name, msg) + self._logger.debug(msg, exc_info=exc_info) + + def _init(self): + # Init all per-session state settings. + self._app_authenticated = False + self._app_authorized = False + self._proto_authenticated = False + self._connected = False + self._device_context_stack = [] + self._auto_driver = driver_map[self.guess_os()] + self._driver_replaced = False + self._expect_canceled = False + self._last_account = None + + def _reset(self): + # Reset all per-session state settings when close() is called. + self._buffer.clear() + self._os_guesser.reset() + if self._log: + try: + self._log.close() + except Exception: + pass + self._log = None + if self.stdout and self._stdout_owner: + self.stdout.truncate(0) + self.stdout.seek(0) + self._init() + + def _receive_cb(self, data): + with self._write_lock: + self._write_to_streams(data) + self._write_to_logs(data) # Check whether a better driver is found based on the incoming data. - old_driver = self.get_driver() - self.os_guesser.data_received(data, self.is_app_authenticated()) - self.auto_driver = driver_map[self.guess_os()] - new_driver = self.get_driver() - if old_driver != new_driver: - self._driver_replaced_notify(old_driver, new_driver) + self._os_guesser.check_received_data(data, self.is_app_authenticated()) + self._update_auto_driver() # Send signals to subscribers. self.data_received_event(data) + def _send_cb(self, data): + if not self._receives_echo: + # The sent commands are either logged via the _receive_cb() when + # they are echoed back, or they are logged here explicitly. + with self._write_lock: + self._write_to_streams(data) + self._write_to_logs(data) + + # Send signals to subscribers. + self.data_sent_event(data) + + def _write_to_logs(self, data): + if self._log is None: + return + self._log.write(data) + self._log.flush() + + def _write_to_streams(self, data): + self.stdout.write(data) + self.stdout.flush() + def is_dummy(self): """ Returns True if the adapter implements a virtual device, i.e. @@ -369,79 +573,97 @@ def is_dummy(self): """ return False - def _dbg(self, level, msg): - if self.debug < level: - return - self.stderr.write(self.get_driver().name + ': ' + msg + '\n') + # OPTIMIZE: OsGuesser should take this method as a callback on OS change. + def _update_auto_driver(self): + old_driver = self.get_driver() + self._auto_driver = driver_map[self.guess_os()] + new_driver = self.get_driver() + if old_driver != new_driver: + msg = 'Driver replaced: %s -> %s' % (old_driver.name, new_driver.name) + self._dbg(2, msg) + if self.connected: + self._driver_replaced = True + self.cancel_expect() def set_driver(self, driver=None): """ Defines the driver that is used to recognize prompts and implement - behavior depending on the remote system. - The driver argument may be an instance of a protocols.drivers.Driver - subclass, a known driver name (string), or None. - If the driver argument is None, the adapter automatically chooses - a driver using the guess_os() function. + behavior depending on the remote system. The driver argument may be an + instance of a protocols.drivers.Driver subclass, a known driver name + (string), or `None`. If the driver argument is `None`, the adapter + automatically chooses a driver using the guess_os() function. - :type driver: Driver()|str - :param driver: The pattern that, when matched, causes an error. + :type driver: Driver()|str|None + :param driver: The driver to be pinned (disables auto-detection). """ + old_driver = self.get_driver() + if driver is None: - self.manual_driver = None + self._manual_driver = None elif isinstance(driver, str): if driver not in driver_map: raise TypeError('no such driver:' + repr(driver)) - self.manual_driver = driver_map[driver] + self._manual_driver = driver_map[driver] elif isinstance(driver, Driver): - self.manual_driver = driver + self._manual_driver = driver else: - raise TypeError('unsupported argument type:' + type(driver)) + raise TypeError('unsupported argument type: ' + type(driver).__name__) + + new_driver = self.get_driver() + if old_driver != new_driver: + msg = 'Driver replaced: %s -> %s' % (old_driver.name, new_driver.name) + self._dbg(2, msg) + if self.connected: + self._driver_replaced = True + self.cancel_expect() def get_driver(self): """ Returns the currently used driver. - :rtype: Driver - :return: A regular expression. + :rtype: Driver() + :return: The currently used driver (auto-detected or pinned manually). """ - if self.manual_driver: - return self.manual_driver - return self.auto_driver + if self._manual_driver: + return self._manual_driver + return self._auto_driver def get_banner(self): """ Returns the banner that was received upon login. - Only supported on SSH2; returns None on all other protocols. - Also returns None if the client is not yet connected. + Only supported on SSH2/Netconf. Returns `None` on all other protocols. + Also returns `None` if the client is not yet connected. :rtype: str|None - :return: The banner as a string + :return: The banner as a string. """ return None def get_remote_version(self): """ - Returns the remote version idstring that was received upon login. - Only supported on SSH2; returns None on all other protocols. - Also returns None if the client is not yet connected. + Returns the remote version id-string that was received upon login. + Only supported on SSH2/Netconf. Returns `None` on all other protocols. + Also returns `None` if the client is not yet connected. :rtype: str|None - :return: The idstring. + :return: The version id-string. """ return None def autoinit(self): """ Make the remote host more script-friendly by automatically executing - one or more commands on it. + one or more commands on it to prepare the session/environment. The commands executed depend on the currently used driver. - For example, the driver for Cisco IOS would execute the - following commands:: + For example, the driver for Cisco IOS would execute the following + commands:: term len 0 term width 0 """ - self.get_driver().init_terminal(self) + driver = self.get_driver() + self._dbg(4, 'Calling %s.init_terminal().' % repr(driver)) + driver.init_terminal(self) def set_username_prompt(self, regex=None): """ @@ -452,9 +674,9 @@ def set_username_prompt(self, regex=None): :param regex: The pattern that, when matched, causes an error. """ if regex is None: - self.manual_user_re = regex + self._manual_user_re = regex else: - self.manual_user_re = to_regexs(regex) + self._manual_user_re = to_regexs(regex) def get_username_prompt(self): """ @@ -464,8 +686,8 @@ def get_username_prompt(self): :rtype: regex :return: A regular expression. """ - if self.manual_user_re: - return self.manual_user_re + if self._manual_user_re: + return self._manual_user_re return self.get_driver().user_re def set_password_prompt(self, regex=None): @@ -477,37 +699,45 @@ def set_password_prompt(self, regex=None): :param regex: The pattern that, when matched, causes an error. """ if regex is None: - self.manual_password_re = regex + self._manual_password_re = regex else: - self.manual_password_re = to_regexs(regex) + self._manual_password_re = to_regexs(regex) def get_password_prompt(self): """ Returns the regular expression that is used to monitor the response - of the connected host for a username prompt. + of the connected host for a password prompt. :rtype: regex :return: A regular expression. """ - if self.manual_password_re: - return self.manual_password_re + if self._manual_password_re: + return self._manual_password_re return self.get_driver().password_re - def set_prompt(self, prompt=None): + def set_prompt(self, prompt=None, *, extend=False): """ Defines a pattern that is waited for when calling the expect_prompt() - method. - If the set_prompt() method is not called, or if it is called with the - prompt argument set to None, a default prompt is used that should - work with many devices running Unix, IOS, IOS-XR, or Junos and others. + method. If the set_prompt() method is not called, or if it is called + with the prompt argument set to `None`, a default prompt is used that + should work with many devices running Unix, IOS, IOS-XR, or Junos and + others. Netconf uses the EOM or EOC markers to emulate prompt matching. + This should *NOT* be changed as its RFC is strict about this. :type prompt: RegEx :param prompt: The pattern that matches the prompt of the remote host. + :type extend: bool + :param extend: Whether to extend the default pattern instead of setting + it to the new value. """ if prompt is None: - self.manual_prompt_re = prompt + self._manual_prompt_re = prompt + elif extend: + if self._manual_prompt_re is None: + self._manual_prompt_re = self.get_prompt()[:] + self._manual_prompt_re.extend(to_regexs(prompt)) else: - self.manual_prompt_re = to_regexs(prompt) + self._manual_prompt_re = to_regexs(prompt) def get_prompt(self): """ @@ -517,23 +747,37 @@ def get_prompt(self): :rtype: list(re.RegexObject) :return: A list of regular expression objects. """ - if self.manual_prompt_re: - return self.manual_prompt_re + if self._manual_prompt_re: + return self._manual_prompt_re return self.get_driver().prompt_re - def set_error_prompt(self, error=None): + def set_error_prompt(self, error=None, *, extend=False): """ - Defines a pattern that is used to monitor the response of the - connected host. If the pattern matches (any time the expect() or - expect_prompt() methods are used), an error is raised. + Defines a pattern that is used to monitor the response of the connected + host. If the set_error_prompt() method is not called, or if it is called + with the error argument set to `None`, a default prompt is used that + should work with many devices running Unix, IOS, IOS-XR, or Junos and + others. Netconf looks for the `` tag with an error severity. + This should *NOT* be changed as its RFC is strict about this. + + If the pattern matches (any time the expect() or expect_prompt() methods + are used, but not waitfor()), an InvalidCommandException is raised. + Netconf raises the RPCError exception (subclass) with structured details. :type error: RegEx :param error: The pattern that, when matched, causes an error. + :type extend: bool + :param extend: Whether to extend the default pattern instead of setting + it to the new value. """ if error is None: - self.manual_error_re = error + self._manual_error_re = error + elif extend: + if self._manual_error_re is None: + self._manual_error_re = self.get_error_prompt()[:] + self._manual_error_re.extend(to_regexs(error)) else: - self.manual_error_re = to_regexs(error) + self._manual_error_re = to_regexs(error) def get_error_prompt(self): """ @@ -543,23 +787,23 @@ def get_error_prompt(self): :rtype: regex :return: A regular expression. """ - if self.manual_error_re: - return self.manual_error_re + if self._manual_error_re: + return self._manual_error_re return self.get_driver().error_re def set_login_error_prompt(self, error=None): """ Defines a pattern that is used to monitor the response of the connected host during the authentication procedure. - If the pattern matches an error is raised. + If the pattern matches a LoginFailure exception is raised. :type error: RegEx :param error: The pattern that, when matched, causes an error. """ if error is None: - self.manual_login_error_re = error + self._manual_login_error_re = error else: - self.manual_login_error_re = to_regexs(error) + self._manual_login_error_re = to_regexs(error) def get_login_error_prompt(self): """ @@ -570,8 +814,8 @@ def get_login_error_prompt(self): :rtype: regex :return: A regular expression. """ - if self.manual_login_error_re: - return self.manual_login_error_re + if self._manual_login_error_re: + return self._manual_login_error_re return self.get_driver().login_error_re def set_connect_timeout(self, timeout): @@ -585,68 +829,217 @@ def set_connect_timeout(self, timeout): def get_connect_timeout(self): """ - Returns the current connect_timeout in seconds. + Returns the current connect timeout in seconds. :rtype: int - :return: The connect_timeout in seconds. + :return: The connect timeout in seconds. """ return self.connect_timeout def set_timeout(self, timeout): """ Defines the maximum time that the adapter waits before a call to - :class:`expect()` or :class:`expect_prompt()` fails. + :meth:`expect()`, :meth:`waitfor()` or :meth:`expect_prompt()` fails + with a raised TimeoutException. :type timeout: int - :param timeout: The maximum time in seconds. + :param timeout: The maximum response time in seconds. """ self.timeout = int(timeout) def get_timeout(self): """ - Returns the current timeout in seconds. + Returns the current response timeout in seconds. :rtype: int - :return: The timeout in seconds. + :return: The response timeout in seconds. """ return self.timeout - def _connect_hook(self, host, port): + def get_host(self): + """ + Returns the name or address of the currently connected host. + + :rtype: string + :return: A name or an address (whatever was used initially). """ - Should be overwritten. + return self.host + + def get_protocol(self): + """ + Returns the name of this protocol. + + This is the class name of the protocol adapter in lower-case without + any trailing digits. Therefore class names of protocol adapters should + mimic their later URL scheme. + + :rtype: str + :return: The name of the protocol. + """ + return self._name + + def get_response( + self, *, strip_command=True, strip_chars='\x00\x07\x08', strip_ansi=True + ): + """ + Returns a clean copy of the latest plaintext response. + + See `cleanup_response()` for details on the parameters and the behavior. + + :rtype: str + :return: The cleaned plaintext response. + """ + return self.cleanup_response( + self.response, + strip_command=strip_command, + strip_chars=strip_chars, + strip_ansi=strip_ansi, + ) + + @staticmethod + def cleanup_response( + response, *, strip_command=True, strip_chars='\x00\x07\x08', strip_ansi=True + ): + """ + Returns a clean copy of the given plaintext response. + + This cleans up the given response in terms of undesired characters. Use + the parameters to customize the behavior. Transformation of lone CR and + CRLF to LF is always done. Useful to also clean received data from events. + + :param response: The plaintext response to clean up. + :param strip_command: Whether to remove the first line, which normally + contains the echo of the sent command. This is always `False` for + specific protocols that do not echo commands, like Netconf. + :param strip_chars: Characters to be removed from the response. Must be + a string, but can be empty to disable stripping. By default some + control characters (NUL, BEL, BS) are removed. + :param strip_ansi: Whether to strip ANSI escape sequences (CSI) such as + color codes and cursor movement. This is fully CSI compliant. + :rtype: str + :return: The cleaned plaintext response. + """ + if not response: + return '' + response = response.replace('\r\n', '\n').replace('\r', '\n') + if strip_command: + # When this is not enough some day, in case of multi-line commands, + # we should start to track the last sent commands/data and its length + # and strip by that length instead, then also lstrip CR/LF. + _, _, response = response.partition('\n') + if strip_chars: + response = response.translate(str.maketrans('', '', strip_chars)) + if strip_ansi: + response = _ansi_escape_re.sub('', response) + return response + + def guess_os(self): + """ + Returns an identifier that specifies the operating system that is + running on the remote host. This OS is obtained by watching the + response of the remote host, such as any messages retrieved during + the login procedure. The OS identifier matches the supported driver + names, so it can also be used with set_driver(). + + The OS is also a wild guess that often depends on volatile information, + so there is no guarantee that this will always work reliably. + + :rtype: string + :return: A string to help identify the remote operating system. + """ + return self._os_guesser.get('os') + + def _enforce_connect_rate_limit(self, rate_limit=0): + """ + Enforce a connection rate-limit to prevent remote connection resets. + The limit represents the maximum number of connects per minute. However, + an offset of 0.1s is added per login to avoid exactly hitting the limit. + Uses `self.connect_rate_limit`, if the given rate-limit is zero. + + :type rate_limit: int + :param rate_limit: The connection rate-limit (connects per minute). + """ + key = self.host.lower() + wait = 0.0 + offset = 0.1 + precision = 3 + + if not rate_limit: + rate_limit = self.connect_rate_limit + + if rate_limit > 0: + self._dbg(1, "Connection rate limit: %s per minute" % rate_limit) + wait = round((60 / rate_limit) + offset, precision) + self._dbg(2, "Maximum connection delay: %s seconds" % wait) + + with self._connection_monitor_lock: + now = time.time() + if wait: + # Calculate the remaining wait time since last connection attempt. + last = self._connection_monitor.get(key, 0.0) + wait = round(wait - min((now - last), wait), precision) + if wait: + self._dbg(2, "Remaining connection delay: %s seconds" % wait) + time.sleep(wait) + self._connection_monitor[key] = now + + def _connect(self): + """ + Should be overwritten. That's the place where the connection should be + established. """ raise NotImplementedError() def connect(self, hostname=None, port=None): """ - Opens the connection to the remote host or IP address. + Opens the connection to the remote host or IP address. Raises a + ProtocolException if the adapter is already connected or hostname/port + is missing or invalid (e.g. first connect). The hostname and port may + be omitted for reconnects to the same device. :type hostname: string :param hostname: The remote host or IP address. :type port: int :param port: The remote TCP port number. """ - if hostname is not None: - self.host = hostname - conn = self._connect_hook(self.host, port) - self.os_guesser.protocol_info(self.get_remote_version()) - self.auto_driver = driver_map[self.guess_os()] - if self.get_banner(): - self.os_guesser.data_received(self.get_banner(), False) - return conn + if self.connected: + msg = 'already connected to %s:%s' % (self.host, self.port) + raise ProtocolException(msg) + + self.host = hostname or self.host + self.port = port or self.port or self._default_port + if not self.host or not self.port: + msg = 'invalid hostname or port: %s:%s' % (self.host, self.port) + raise ProtocolException(msg) + + # Connecting... + self._dbg(1, "Attempting to connect: '%s:%s'" % (self.host, self.port)) + if self.logfile and not self._log: + self._log = open(self.logfile, 'a') + self._enforce_connect_rate_limit() + self._connected = bool(self._connect()) + self._dbg(2, "Finished to connect: '%s:%s'" % (self.host, self.port)) + + # Check whether a better driver is found based on the incoming data. + self._os_guesser.check_protocol_info(self.get_remote_version() or '') + self._os_guesser.check_received_data( + self.get_banner() or '', self.is_app_authenticated() + ) + self._update_auto_driver() + return self._connected def _get_account(self, account): if isinstance(account, Context) or isinstance(account, _Context): return account.context() if account is None: - account = self.last_account + account = self._last_account if self.account_factory: account = self.account_factory(account) else: if account is None: raise TypeError('An account is required') account.__enter__() - self.last_account = account + self._last_account = account return account.context() def login(self, account=None, app_account=None, flush=True): @@ -654,26 +1047,31 @@ def login(self, account=None, app_account=None, flush=True): Log into the connected host using the best method available. If an account is not given, default to the account that was used during the last call to login(). If a previous call was not - made, use the account that was passed to the constructor. If that - also fails, raise a TypeError. + made, use the account-factory that was passed to the constructor. + If that also fails, raise a TypeError. - The app_account is passed to :class:`app_authenticate()` and - :class:`app_authorize()`. - If app_account is not given, default to the value of the account - argument. + The app_account is passed to :meth:`app_authenticate` and + :meth:`app_authorize`. If app_account is not given, default + to the value of the account argument. Netconf does not support + app-level authentication or authorization, so the app_account + is obsolete for that protocol. However, Netconf in general flags + the app authentication/authorization successfully done. :type account: Account :param account: The account for protocol level authentication. :type app_account: Account - :param app_account: The account for app level authentication. + :param app_account: The account for app level authentication and + authorization (see password vs. authorization-password). :type flush: bool :param flush: Whether to flush the last prompt from the buffer. """ with self._get_account(account) as account: if app_account is None: app_account = account - self.authenticate(account, flush=False) - if self.get_driver().supports_auto_authorize(): + self.authenticate(account, app_account, flush=False) + driver = self.get_driver() + if driver.supports_auto_authorize(): + self._dbg(4, '%s supports auto-authorization.' % repr(driver)) self.expect_prompt() self.auto_app_authorize(app_account, flush=flush) @@ -682,8 +1080,8 @@ def authenticate(self, account=None, app_account=None, flush=True): Like login(), but skips the authorization procedure. .. HINT:: - If you are unsure whether to use :class:`authenticate()` or - :class:`login()`, stick with :class:`login`. + If you are unsure whether to use :meth:`authenticate` or + :meth:`login`, stick with :meth:`login`. :type account: Account :param account: The account for protocol level authentication. @@ -695,15 +1093,20 @@ def authenticate(self, account=None, app_account=None, flush=True): with self._get_account(account) as account: if app_account is None: app_account = account - - if not self.proto_authenticated: + if not self._proto_authenticated: self.protocol_authenticate(account) self.app_authenticate(app_account, flush=flush) def _protocol_authenticate(self, user, password): + """ + Should be overwritten if protocol authentication is supported. + """ pass def _protocol_authenticate_by_key(self, user, key): + """ + Should be overwritten if protocol authentication is supported. + """ pass def protocol_authenticate(self, account=None): @@ -723,12 +1126,13 @@ def protocol_authenticate(self, account=None): password = account.get_password() key = account.get_key() if key is None: - self._dbg(1, "Attempting to authenticate %s." % user) + self._dbg(1, "Attempting to protocol-authenticate %s with password." % user) self._protocol_authenticate(user, password) else: - self._dbg(1, "Authenticate %s with key." % user) + self._dbg(1, "Attempting to protocol-authenticate %s with key." % user) self._protocol_authenticate_by_key(user, key) - self.proto_authenticated = True + self._proto_authenticated = True + self._dbg(2, "Finished to protocol-authenticate %s." % user) def is_protocol_authenticated(self): """ @@ -738,13 +1142,16 @@ def is_protocol_authenticated(self): :rtype: bool :return: Whether the authentication was completed. """ - return self.proto_authenticated + return self._proto_authenticated def _app_authenticate(self, account, password, flush=True, bailout=False): + """ + Generic implementation for app-level authentications or authorizations. + """ user = account.get_name() while True: @@ -759,8 +1166,8 @@ def _app_authenticate(self, ('cli', self.get_prompt())) prompt_map = [] prompt_list = [] - for section, sectionprompts in prompts: - for prompt in sectionprompts: + for section, section_prompts in prompts: + for prompt in section_prompts: prompt_map.append((section, prompt)) prompt_list.append(prompt) @@ -768,19 +1175,14 @@ def _app_authenticate(self, try: index, match = self._waitfor(prompt_list) except TimeoutException: - if self.response is None: - self.response = '' - msg = "Buffer: %s" % repr(self.response) - raise TimeoutException(msg) + self._raise_on_response_timeout() except DriverReplacedException: - # Driver replaced, retry. - self._dbg(1, 'Protocol.app_authenticate(): driver replaced') + msg = "Handling DriverReplacedException, retrying to wait for a prompt." + self._dbg(4, msg) continue except ExpectCancelledException: - self._dbg(1, 'Protocol.app_authenticate(): expect cancelled') raise except EOFError: - self._dbg(1, 'Protocol.app_authenticate(): EOF') raise # Login error detected. @@ -790,32 +1192,31 @@ def _app_authenticate(self, # User name prompt. elif section == 'username': - self._dbg(1, "Username prompt %s received." % index) + self._dbg(2, "Username prompt received.") self.expect(prompt) # consume the prompt from the buffer self.send(user + '\r') continue # s/key prompt. elif section == 'skey': - self._dbg(1, "S/Key prompt received.") + self._dbg(2, "S/Key prompt received.") self.expect(prompt) # consume the prompt from the buffer seq = int(match.group(1)) seed = match.group(2) self.otp_requested_event(account, seq, seed) - self._dbg(2, "Seq: %s, Seed: %s" % (seq, seed)) + self._dbg(3, "Seq: %s, Seed: %s" % (seq, seed)) phrase = otp(password, seed, seq) # A password prompt is now required. self.expect(self.get_password_prompt()) self.send(phrase + '\r') - self._dbg(1, "Password sent.") if bailout: break continue # Cleartext password prompt. elif section == 'password': - self._dbg(1, "Cleartext password prompt received.") + self._dbg(2, "Password prompt received.") self.expect(prompt) # consume the prompt from the buffer self.send(password + '\r') if bailout: @@ -824,7 +1225,7 @@ def _app_authenticate(self, # Shell prompt. elif section == 'cli': - self._dbg(1, 'Shell prompt received.') + self._dbg(2, 'Shell/CLI prompt received.') if flush: self.expect_prompt() break @@ -856,8 +1257,12 @@ def app_authenticate(self, account=None, flush=True, bailout=False): on whether the flush argument is True, it also removes the prompt from the incoming buffer. + .. HINT:: + In most cases, you want to use the login() method instead, as + it automatically chooses the best login method for each protocol. + :type account: Account - :param account: An account object, like login(). + :param account: An account object, like for login() or authenticate(). :type flush: bool :param flush: Whether to flush the last prompt from the buffer. :type bailout: bool @@ -868,7 +1273,8 @@ def app_authenticate(self, account=None, flush=True, bailout=False): password = account.get_password() self._dbg(1, "Attempting to app-authenticate %s." % user) self._app_authenticate(account, password, flush, bailout) - self.app_authenticated = True + self._app_authenticated = True + self._dbg(2, "Finished to app-authenticate %s." % user) def is_app_authenticated(self): """ @@ -878,7 +1284,7 @@ def is_app_authenticated(self): :rtype: bool :return: Whether the authentication was completed. """ - return self.app_authenticated + return self._app_authenticated def app_authorize(self, account=None, flush=True, bailout=False): """ @@ -886,10 +1292,16 @@ def app_authorize(self, account=None, flush=True, bailout=False): of the account. For the difference between authentication and authorization - please google for AAA. + please search the Internet for AAA. + + .. HINT:: + In most cases, you want to use the login() method instead, as + it automatically chooses the best login method for each protocol. + If you want to handle authorization manually, choose authenticate() + instead. :type account: Account - :param account: An account object, like login(). + :param account: An account object, like for login() or authenticate(). :type flush: bool :param flush: Whether to flush the last prompt from the buffer. :type bailout: bool @@ -902,32 +1314,35 @@ def app_authorize(self, account=None, flush=True, bailout=False): password = account.get_password() self._dbg(1, "Attempting to app-authorize %s." % user) self._app_authenticate(account, password, flush, bailout) - self.app_authorized = True + self._app_authorized = True + self._dbg(2, "Finished to app-authorize %s." % user) def auto_app_authorize(self, account=None, flush=True, bailout=False): """ - Like authorize(), but instead of just waiting for a user or + Like app_authorize(), but instead of just waiting for a user or password prompt, it automatically initiates the authorization - procedure by sending a driver-specific command. + procedure. - In the case of devices that understand AAA, that means sending - a command to the device. For example, on routers running Cisco - IOS, this command executes the 'enable' command before expecting - the password. + In the case of devices that understand AAA, this means sending a + driver-specific command to the device. For example, on Cisco routers + running IOS, this command executes the `enable` command to initialize + the authorization before expecting the password prompt. - In the case of a device that is not recognized to support AAA, this - method does nothing. + In the case of a device that is not recognized to support AAA, the + driver will fall back to the app_authorize() method. Without any + authorization prompt it does nothing. :type account: Account - :param account: An account object, like login(). + :param account: An account object, like for login() or authenticate(). :type flush: bool :param flush: Whether to flush the last prompt from the buffer. :type bailout: bool :param bailout: Whether to wait for a prompt after sending the password. """ with self._get_account(account) as account: - self._dbg(1, 'Calling driver.auto_authorize().') - self.get_driver().auto_authorize(self, account, flush, bailout) + driver = self.get_driver() + self._dbg(4, 'Calling %s.auto_authorize().' % repr(driver)) + driver.auto_authorize(self, account, flush, bailout) def is_app_authorized(self): """ @@ -937,33 +1352,64 @@ def is_app_authorized(self): :rtype: bool :return: Whether the authorization was completed. """ - return self.app_authorized + return self._app_authorized + + def _send(self, data): + """ + Should be overwritten. That's the place where the data is sent to the + remote host without waiting for response. + """ + raise NotImplementedError() def send(self, data): """ - Sends the given data to the remote host. - Returns without waiting for a response. + Sends the given data to the remote host. Returns without waiting for a + response while the response (self.response) attribute is cleared. Please + be aware that responses of consecutive send() calls will accumulate in + the socket buffer until they are received at once with the next call of + expect(), waitfor() or expect_prompt(). + + However, Netconf's synchronous implementation in Exscript always blocks + until the structured XML response was received. There is no accumulation + in the socket buffer but in the internal buffer for the plaintext XML + response. Therefore each send() call should be followed by a consuming + expect() or expect_prompt() call. In other words it is recommended + to only use execute() with Netconf, which covers this constraint. + + In case of Netconf the RPC XML encoding/structure is additionally + validated and if that fails, XMLValidationError is raised. :type data: string :param data: The data that is sent to the remote host. - :rtype: Boolean - :return: True on success, False otherwise. """ - raise NotImplementedError() + self.response = None + self._dbg(4, 'Sending: %s' % repr(data)) + self._send(data) def execute(self, command, consume=True): """ - Sends the given data to the remote host (with a newline appended) - and waits for a prompt in the response. The prompt attempts to use - a sane default that works with many devices running Unix, IOS, - IOS-XR, or Junos and others. If that fails, a custom prompt may - also be defined using the set_prompt() method. + Sends the given command to the remote host (with a CR appended) and + waits for a prompt in the response. Netconf expects the RPC body as + the command and the outer `` tag is handled internally. In that + case the RPC XML encoding/structure is validated and if that fails, + XMLValidationError is raised. + + The prompt attempts to use a sane default for CLI-driven protocols that + works with many devices running Unix, IOS, IOS-XR, or Junos and others. + Netconf does not have prompts, so this attempts to use the Netconf 1.0 + EOM marker `]]>]]>` or the Netconf 1.1 EOC marker `LF HASH HASH LF` as + a prompt-like delimiter. If that fails, a custom prompt may also be + defined using the set_prompt() method, while for Netconf as RFC 6142 is + quite strict on the framing behavior, the need might never arise. + This method also modifies the value of the response (self.response) - attribute, for details please see the documentation of the - expect() method. + attribute. For Netconf this refers to the raw RPC reply. Please use the + get_response() method to retrieve a clean copy of the plaintext response. + For details please see the documentation of the expect(), get_response() + and in case of Netconf get_response_element() methods. :type command: string - :param command: The data that is sent to the remote host. + :param command: The command that is sent to the remote host. :type consume: boolean (Default: True) :param consume: Whether to consume the prompt from the buffer or not. :rtype: int, re.MatchObject @@ -975,34 +1421,34 @@ def execute(self, command, consume=True): def _domatch(self, prompt, flush): """ - Should be overwritten. + Should be overwritten. That's the place where self.response should be set. """ raise NotImplementedError() def _waitfor(self, prompt): - re_list = to_regexs(prompt) - patterns = [p.pattern for p in re_list] - self._dbg(2, 'waiting for: ' + repr(patterns)) - result = self._domatch(re_list, False) + prompt = to_regexs(prompt) + self._dbg(2, "Waiting for a prompt (non-consuming).") + self._dbg(3, "Matching prompt pattern: " + repr([p.pattern for p in prompt])) + result = self._domatch(prompt, flush=False) return result def waitfor(self, prompt): """ Monitors the data received from the remote host and waits until - the response matches the given prompt. - Once a match has been found, the buffer containing incoming data - is NOT changed. In other words, consecutive calls to this function - will always work, e.g.:: + the response matches the given prompt. Once a match has been found, + the buffer containing incoming data is *NOT* changed. In other words, + consecutive calls to this function will always work, e.g.:: conn.waitfor('myprompt>') conn.waitfor('myprompt>') conn.waitfor('myprompt>') - will always work. Hence in most cases, you probably want to use - expect() instead. + will always work. Hence in most cases, you probably want to use expect() + or expect_prompt() instead as both consume the response and prompt and + also analyze the response for errors. - This method also stores the received data in the response - attribute (self.response). + This method also stores the received data in the response (self.response) + attribute. For more details see the documentation of expect_prompt(). Returns the index of the regular expression that matched. @@ -1012,40 +1458,44 @@ def waitfor(self, prompt): :return: The index of the regular expression that matched, and the match object. - @raise TimeoutException: raised if the timeout was reached. - - @raise ExpectCancelledException: raised when cancel_expect() was - called in a callback. - - @raise ProtocolException: on other internal errors. - - @raise Exception: May raise other exceptions that are caused + :raise TimeoutException: When the timeout was reached. + :raise ExpectCancelledException: When cancel_expect() was + called in a callback. + :raise ProtocolException: On other internal errors. + :raise Exception: May raise other exceptions that are caused within the underlying protocol implementations. """ while True: try: result = self._waitfor(prompt) except DriverReplacedException: + msg = "Handling DriverReplacedException, retrying to wait for a prompt." + self._dbg(4, msg) continue # retry return result def _expect(self, prompt): - result = self._domatch(to_regexs(prompt), True) + prompt = to_regexs(prompt) + self._dbg(2, "Expecting a prompt (consuming).") + self._dbg(3, "Matching prompt pattern: " + repr([p.pattern for p in prompt])) + result = self._domatch(prompt, flush=True) return result def expect(self, prompt): """ - Like waitfor(), but also removes the matched string from the buffer - containing the incoming data. In other words, the following may not - alway complete:: + Like waitfor(), but also removes the response and the matched string + from the buffer containing the incoming data. In other words, the + following may not always complete:: conn.expect('myprompt>') conn.expect('myprompt>') # timeout - Returns the index of the regular expression that matched. + Checks the response for errors and raises an InvalidCommandException or + RPCError (Netconf). Returns the index of the regular expression that + matched. .. HINT:: - May raise the same exceptions as :class:`waitfor`. + May raise the same exceptions as :meth:`waitfor`. :type prompt: str|re.RegexObject|list(str|re.RegexObject) :param prompt: One or more regular expressions. @@ -1057,21 +1507,34 @@ def expect(self, prompt): try: result = self._expect(prompt) except DriverReplacedException: + msg = "Handling DriverReplacedException, retrying to expect a prompt." + self._dbg(4, msg) continue # retry + self._check_response_for_errors() return result def expect_prompt(self, consume=True): """ - Monitors the data received from the remote host and waits for a - prompt in the response. The prompt attempts to use - a sane default that works with many devices running Unix, IOS, - IOS-XR, or Junos and others. If that fails, a custom prompt may - also be defined using the set_prompt() method. - This method also stores the received data in the response - attribute (self.response). - - :type consume: boolean (Default: True) - :param consume: Whether to consume the prompt from the buffer or not. + Monitors the data received from the remote host and waits for a prompt + in the response. + + The prompt attempts to use a sane default for CLI-driven protocols that + works with many devices running Unix, IOS, IOS-XR, or Junos and others. + Netconf does not have prompts, so this attempts to use the Netconf 1.0 + EOM marker `]]>]]>` or the Netconf 1.1 EOC marker `LF HASH HASH LF` as + a prompt-like delimiter. If that fails, a custom prompt may also be + defined using the set_prompt() method, while for Netconf as RFC 6142 is + quite strict on the framing behavior, the need might never arise. + + This method also stores the received data in the response (self.response) + attribute. For Netconf this refers to the raw RPC reply. Please use the + get_response() method to retrieve a clean copy of the plaintext response. + For details please see the documentation of the expect(), get_response() + and in case of Netconf get_response_element() methods. + + :type consume: boolean + :param consume: Whether to consume the response and prompt from the + buffer or not. The later allows consecutive calls to succeed. :rtype: int, re.MatchObject :return: The index of the prompt regular expression that matched, and the match object. @@ -1079,21 +1542,21 @@ def expect_prompt(self, consume=True): if consume: result = self.expect(self.get_prompt()) else: - self._dbg(1, "DO NOT CONSUME PROMPT!") result = self.waitfor(self.get_prompt()) + return result - # We skip the first line because it contains the echo of the command - # sent. - self._dbg(5, "Checking %s for errors" % repr(self.response)) - for line in self.response.split('\n')[1:]: - for prompt in self.get_error_prompt(): + def _check_response_for_errors(self): + error_prompts = self.get_error_prompt() + if not error_prompts: + return + self._dbg(3, "Checking response for errors: %s" % repr(self.response)) + for lineno, line in enumerate(self.get_response().split('\n'), start=1): + for prompt in error_prompts: if not prompt.search(line): continue - args = repr(prompt.pattern), repr(line) - self._dbg(5, "error prompt (%s) matches %s" % args) - raise InvalidCommandException('Device said:\n' + self.response) - - return result + args = repr(prompt.pattern), lineno, repr(line) + self._dbg(3, "Error prompt %s matches line %s: %s" % args) + self._raise_on_response_with_errors() def add_monitor(self, pattern, callback, limit=80): """ @@ -1115,15 +1578,42 @@ def add_monitor(self, pattern, callback, limit=80): :param limit: The maximum size of the tail of the buffer that is searched, in number of bytes. """ - self.buffer.add_monitor(pattern, partial(callback, self), limit) + self._buffer.add_monitor(pattern, partial(callback, self), limit) def cancel_expect(self): """ - Cancel the current call to :class:`expect()` as soon as control returns - to the protocol adapter. This method may be used in callbacks to - the events emitted by this class, e.g. Protocol.data_received_event. - """ - raise NotImplementedError() + Cancel the current call to :meth:`expect()` or :meth:`waitfor()` as + soon as control returns to the protocol adapter. This method may be + used in callbacks to the events emitted by this class, e.g. + Protocol.data_received_event. + """ + self._dbg(4, "Cancelling any expect() or waitfor() calls.") + self._expect_canceled = True + + def _raise_on_canceled_expect(self): + self._expect_canceled = False + if self._driver_replaced: + self._driver_replaced = False + raise DriverReplacedException() + raise ExpectCancelledException() + + def _raise_on_response_with_errors(self): + msg = 'Response from device:\n' + self.response + raise InvalidCommandException(msg) + + def _raise_on_response_timeout(self): + buffer = str(self._buffer)[-200:] + if buffer: + buffer = '...' + buffer + msg = 'Timeout while waiting for a response/prompt. Buffer: %s' + raise TimeoutException(msg % repr(buffer)) + + def _raise_on_response_eof(self): + buffer = str(self._buffer)[-200:] + if buffer: + buffer = '...' + buffer + msg = 'EOF while waiting for a response/prompt. Buffer: %s' + raise ProtocolException(msg % repr(buffer)) def _call_key_handlers(self, key_handlers, data): if key_handlers is not None: @@ -1134,12 +1624,13 @@ def _call_key_handlers(self, key_handlers, data): return False def _set_terminal_size(self, rows, cols): + """ + Should be overwritten. That's the place where the terminal size is set + in the interactive shell. + """ raise NotImplementedError() - def _open_posix_shell(self, - channel, - key_handlers, - handle_window_size): + def _open_posix_shell(self, channel, key_handlers, handle_window_size): # We need to make sure to use an unbuffered stdin, else multi-byte # chars (such as arrow keys) won't work properly. with os.fdopen(sys.stdin.fileno(), 'rb', 0) as stdin: @@ -1149,6 +1640,7 @@ def _open_posix_shell(self, if handle_window_size: def handle_sigwinch(signum, frame): rows, cols = get_terminal_size() + self._dbg(2, "Trying to update terminal size: %dx%d" % (rows, cols)) self._set_terminal_size(rows, cols) signal.signal(signal.SIGWINCH, handle_sigwinch) handle_sigwinch(None, None) @@ -1176,13 +1668,13 @@ def handle_sigwinch(signum, frame): except socket.timeout: pass if not data: - self._dbg(1, 'EOF from remote') + self._dbg(1, 'EOF from remote host') break - self._receive_cb(data, False) - self.buffer.append(data) + self._receive_cb(data) + self._buffer.append(data) if stdin in r: - data = stdin.read(1).decode(self.encoding) - self.buffer.clear() + data = stdin.read(1).decode(sys.stdin.encoding) + self._buffer.clear() if len(data) == 0: break @@ -1194,9 +1686,10 @@ def handle_sigwinch(signum, frame): termios.tcsetattr(stdin, termios.TCSADRAIN, curtty) if not is_handled: - if not self.send_data is None: - self.send_data.write(data) channel.send(data.encode(self.encoding)) + self._send_cb(data) + if self.send_data is not None: + self.send_data.write(data) finally: termios.tcsetattr(stdin, termios.TCSADRAIN, oldtty) @@ -1205,24 +1698,39 @@ def _open_windows_shell(self, channel, key_handlers, handle_window_size): def writeall(sock): while True: - data = sock.recv(256) + # NOTE: Windows stdin is always cooked, remote may send double-echos. + # With remote PTY/Shell connections we might see our echos from + # our cooked stdin and also from the remote channel. Needs to be + # analyzed on next possible testing with a windows box. + data = sock.recv(1024).decode(self.encoding) if not data: - self._dbg(1, 'EOF from remote') + self._dbg(1, 'EOF from remote host') break self._receive_cb(data) + self._buffer.append(data) - writer = threading.Thread(target=writeall, args=(channel,)) + # NOTE: Thread in windows might need a daemon=True to avoid hanging on exit. + # Needs to be analyzed on next possible testing with a windows box. + writer = threading.Thread( + name="channel.recv", target=writeall, args=(channel,) + ) writer.start() try: while True: - data = sys.stdin.read(1) + # NOTE: Windows stdin is always cooked, special control chars may fail. + # With a cooked stdin we might not be able to use cursors/arrows + # and other special escape sequences. Depends on the local shell. + # Needs to be analyzed on next possible testing with a windows box. + data = sys.stdin.read(1) # string, not bytes + self._buffer.clear() if not data: break if not self._call_key_handlers(key_handlers, data): - if not self.send_data is None: + channel.send(data.encode(self.encoding)) + self._send_cb(data) + if self.send_data is not None: self.send_data.write(data) - channel.send(data) except EOFError: self._dbg(1, 'User hit ^Z or F6') @@ -1232,15 +1740,25 @@ def _open_shell(self, channel, key_handlers, handle_window_size): else: return self._open_windows_shell(channel, key_handlers, handle_window_size) + def _interact(self, key_handlers=None, handle_window_size=True): + """ + Should be overwritten. That's the place where the interactive shell is + opened. + """ + raise NotImplementedError() + def interact(self, key_handlers=None, handle_window_size=True): """ - Opens a simple interactive shell. Returns when the remote host - sends EOF. + Opens a simple interactive shell. Returns when the remote host sends EOF. + The optional key handlers are functions that are called whenever the user presses a specific key. For example, to catch CTRL+y:: conn.interact({'\031': mycallback}) + May raise NotImplementedError if the underlying protocol does not + support interactive shells. + .. WARNING:: handle_window_size is not supported on Windows platforms. @@ -1248,40 +1766,168 @@ def interact(self, key_handlers=None, handle_window_size=True): :param key_handlers: A dictionary mapping chars to a functions. :type handle_window_size: bool :param handle_window_size: Whether the connected host is notified - when the terminal size changes. - """ - raise NotImplementedError() - - def close(self, force=False): - """ - Closes the connection with the remote host. - """ - if self.log: - try: - self.log.close() - except: - pass + when the terminal size changes. + """ + self._dbg(1, "Switching to interactive session.") + self._interact(key_handlers=key_handlers, handle_window_size=handle_window_size) + + def _capture_device_context(self): + """Captures all context-specific state into a plain dict snapshot.""" + return { + # Authentication + 'app_authenticated': self._app_authenticated, + 'app_authorized': self._app_authorized, + 'last_account': self._last_account, + # OS guesser + 'guesser_info': dict(self._os_guesser._info), + 'guesser_auth_buffer': self._os_guesser._auth_buffer, + 'guesser_protocol_buffer': self._os_guesser._protocol_buffer, + 'guesser_response_buffer': self._os_guesser._response_buffer, + # Driver + 'auto_driver': self._auto_driver, + 'manual_driver': self._manual_driver, + # Prompts + 'manual_prompt_re': self._manual_prompt_re, + 'manual_error_re': self._manual_error_re, + 'manual_user_re': self._manual_user_re, + 'manual_password_re': self._manual_password_re, + 'manual_login_error_re': self._manual_login_error_re, + } + + def switch_device_context(self, *, reset_auth=True): + """ + Saves the current OS guesser, driver, and prompt state onto an internal + stack and resets them for a new device context. Calling the method + restore_device_context() will restore the context to the previous state. + This context-switching might be useful while hopping to another remote + host or entering a local shell. The caller is responsible to implement + the additional logic when and how to switch the context in the specific + domain. + + The OS guesser is always reset so that OS/driver auto-detection restarts + from scratch for the new context. Any previous manually pinned driver is + unconditionally cleared and needs to be set again manually, if required. + + All manual prompt, error, username, password and login-error patterns + are unconditionally cleared and need to be set again manually, if required. + + Use `reset_auth=False` when switching into a local environment on the + same device (e.g. local shell or container) where the app authentication + or app authorization flags are still valid. Use the default `reset_auth=True` + when hopping to a remote host that requires its own login sequence. Please + note that this only resets the mentioned flags. It does not automatically + trigger any login/authentication sequence. + + Calls may be nested arbitrarily:: + + conn.switch_device_context() # hop to remote router + + conn.switch_device_context(reset_auth=False) # enter local bash shell + conn.set_driver('shell') + + conn.restore_device_context() # back to remote router + conn.restore_device_context() # back to original + + :type reset_auth: bool + :param reset_auth: Whether to reset app-level authentication and authorization + flags. Default is `True` for remote hops. Pass `False` when the new + context is on the same authenticated device (e.g. local shell). + """ + self._device_context_stack.append(self._capture_device_context()) + + # Reset auth state if a new login is expected. + if reset_auth: + self._app_authenticated = False + self._app_authorized = False + self._last_account = None + + # Reset os-guesser and clear any manual driver so that auto-detection + # restarts from scratch. + self._os_guesser.reset() + self._manual_driver = None + self._update_auto_driver() + + # Clear all manual patterns to prevent cross-context inconsistencies. + self._manual_prompt_re = None + self._manual_error_re = None + self._manual_user_re = None + self._manual_password_re = None + self._manual_login_error_re = None + + def _restore_device_context(self, snapshot): + """Restores all context-specific state from a snapshot dict.""" + # Authentication + self._app_authenticated = snapshot['app_authenticated'] + self._app_authorized = snapshot['app_authorized'] + self._last_account = snapshot['last_account'] + # OS guesser + self._os_guesser._info = dict(snapshot['guesser_info']) + self._os_guesser._auth_buffer = snapshot['guesser_auth_buffer'] + self._os_guesser._protocol_buffer = snapshot['guesser_protocol_buffer'] + self._os_guesser._response_buffer = snapshot['guesser_response_buffer'] + # Driver + self._auto_driver = snapshot['auto_driver'] + self._manual_driver = snapshot['manual_driver'] + self._update_auto_driver() # should only trigger notifications + # Prompts + self._manual_prompt_re = snapshot['manual_prompt_re'] + self._manual_error_re = snapshot['manual_error_re'] + self._manual_user_re = snapshot['manual_user_re'] + self._manual_password_re = snapshot['manual_password_re'] + self._manual_login_error_re = snapshot['manual_login_error_re'] + + def restore_device_context(self): + """ + Pops and fully restores the most recently saved device context from the + internal stack, including the OS guesser state, the driver, all prompt + patterns and app authentication/authorization flags captured by the last + switch_device_context() call. + + :raises ProtocolException: Raised when no saved context exists on the stack. + """ + if not self._device_context_stack: + raise ProtocolException('No saved device context to restore.') + self._restore_device_context(self._device_context_stack.pop()) + + def _logout(self): + """ + Logout from the remote host using the driver-specific command. + """ + driver = self.get_driver() + self._dbg(4, 'Calling %s.logout().' % repr(driver)) + try: + driver.logout(self) + except Exception: + pass - def get_host(self): + def _close(self, force=False): """ - Returns the name or address of the currently connected host. - - :rtype: string - :return: A name or an address. + Should be overwritten. That's the place where the connection should be + closed and protocol-specific references/resources cleared. """ - return self.host + raise NotImplementedError() - def guess_os(self): + def close(self, force=False): """ - Returns an identifier that specifies the operating system that is - running on the remote host. This OS is obtained by watching the - response of the remote host, such as any messages retrieved during - the login procedure. + Closes the connection with the remote host. The instance may be + reconnected afterwards by calling connect() again. - The OS is also a wild guess that often depends on volatile - information, so there is no guarantee that this will always work. + With `force=False` the driver-specific logout() method is called before + the connection is closed. Dummy and Netconf adapters have a special + static logout sequence. - :rtype: string - :return: A string to help identify the remote operating system. - """ - return self.os_guesser.get('os') + .. WARNING:: + Ensure the remote host has already left any previously used submode + state (e.g. config mode, local shell etc.) before closing the + connection gracefully. Otherwise the logout command might fail. + + :type force: bool + :param force: Whether to close the connection without logout. + """ + msg = 'Closing connection to %s:%s (force=%s).' + self._dbg(1, msg % (self.host, self.port, repr(force))) + if self.connected: + if not force: + self._logout() + self._close(force=force) + self._reset() diff --git a/Exscript/protocols/ssh2.py b/Exscript/protocols/ssh2.py index b96c47c9..d3b94e77 100644 --- a/Exscript/protocols/ssh2.py +++ b/Exscript/protocols/ssh2.py @@ -23,31 +23,36 @@ """ SSH version 2 support, based on paramiko. """ -from __future__ import absolute_import, unicode_literals -from builtins import str -import sys + +import logging import os -import time import select import socket +import time +from binascii import hexlify + import paramiko + try: import Cryptodome as Crypto except ImportError: import Crypto -from binascii import hexlify -from paramiko import util -from paramiko.ssh_exception import SSHException, AuthenticationException, \ - BadHostKeyException, BadAuthenticationType -from ..util.tty import get_terminal_size -from ..util.crypt import otp +from paramiko.ssh_exception import ( + AuthenticationException, + BadAuthenticationType, + BadHostKeyException, + SSHException, +) + +from ..account import Account from ..key import PrivateKey +from ..util.crypt import otp +from ..util.tty import get_terminal_size +from .exception import LoginFailure from .protocol import Protocol, _skey_re -from .exception import ProtocolException, LoginFailure, TimeoutException, \ - DriverReplacedException, ExpectCancelledException -# Workaround for paramiko error; avoids a warning message. -util.log_to_file(os.devnull) +_logger = logging.getLogger(__name__) +_logger.addHandler(logging.NullHandler()) # Register supported key types. keymap = {'rsa': paramiko.RSAKey, 'ecdsa': paramiko.ECDSAKey, 'ed25519': paramiko.Ed25519Key} @@ -62,16 +67,26 @@ class SSH2(Protocol): """ - The secure shell protocol version 2 adapter, based on Paramiko. + The SSH2 protocol adapter, based on Paramiko. """ KEEPALIVE_INTERVAL = 2.5 * 60 # Two and a half minutes def __init__(self, **kwargs): Protocol.__init__(self, **kwargs) - self.sock = None - self.client = None - self.shell = None - self.cancel = False + # Module-Level + self._logger = _logger + # Internals + self._sock = None + self._transport = None + self._channel = None + + # Set default preferred auth types, that are supported. + if not self.preferred_authentications: + self.preferred_authentications = [ + "publickey", + "keyboard-interactive", + "password", + ] # Since each protocol may be created in it's own thread, we must # re-initialize the random number generator to make sure that @@ -99,13 +114,13 @@ def _reject_host_key(self, key): name = key.get_name() fp = hexlify(key.get_fingerprint()) msg = 'Rejecting %s host key for %s: %s' % (name, self.host, fp) - self._dbg(1, msg) + self._dbg(2, msg) def _add_host_key(self, key): name = key.get_name() fp = hexlify(key.get_fingerprint()) msg = 'Adding %s host key for %s: %s' % (name, self.host, fp) - self._dbg(1, msg) + self._dbg(2, msg) self._host_keys.add(self.host, name, key) if self._host_keys_filename is not None: self._save_host_keys() @@ -158,26 +173,29 @@ def _paramiko_connect(self): break # Open a socket. - self.sock = socket.socket(af, socket.SOCK_STREAM) + self._sock = socket.socket(af, socket.SOCK_STREAM) try: - self.sock.settimeout(self.connect_timeout or None) + self._sock.settimeout(self.connect_timeout or None) except: pass - self.sock.connect(addr) + self._sock.connect(addr) # Init the paramiko protocol. - t = paramiko.Transport(self.sock) + t = paramiko.Transport(self._sock, disabled_algorithms=self.disabled_algorithms) t.banner_timeout = self.banner_timeout + t.name = self.host + if self.debug < 5: + t.logger.disabled = True + if self.debug > 5: + t.set_hexdump(True) t.start_client() # Check system host keys. server_key = t.get_remote_server_key() keytype = server_key.get_name() - our_server_key = self._system_host_keys.get( - self.host, {}).get(keytype, None) + our_server_key = self._system_host_keys.get(self.host, {}).get(keytype, None) if our_server_key is None: - our_server_key = self._host_keys.get( - self.host, {}).get(keytype, None) + our_server_key = self._host_keys.get(self.host, {}).get(keytype, None) if our_server_key is None: self._missing_host_key(server_key) # if the callback returns, assume the key is ok @@ -189,7 +207,7 @@ def _paramiko_connect(self): return t def _paramiko_auth_none(self, username, password=None): - self.client.auth_none(username) + self._transport.auth_none(username) def _paramiko_auth_interactive(self, username, password=None): if password is None: @@ -201,22 +219,29 @@ def handler(title, instructions, prompt_list): for prompt, visible in prompt_list: match = _skey_re.search(prompt) if match is not None: - skey_password = otp(password, - match.group(2), - int(match.group(1))) + self._receive_cb(prompt + '\n') # Expose the prompt (hide the PW). + self._dbg(2, "S/Key prompt received.") + seq = int(match.group(1)) + seed = match.group(2) + self.otp_requested_event(Account(username, password), seq, seed) + self._dbg(3, "Seq: %s, Seed: %s" % (seq, seed)) + skey_password = otp(password, seed, seq) response.append(skey_password) continue for regex in self.get_password_prompt(): match = regex.search(prompt) if match is not None: + self._receive_cb(prompt + '\n') # Expose the prompt (hide the PW). + self._dbg(2, "Password prompt received.") response.append(password) break return response - self.client.auth_interactive(username, handler) + self._transport.auth_interactive(username, handler) def _paramiko_auth_password(self, username, password): - self.client.auth_password(username, password or '') + self._receive_cb('Password: \n') # Expose a prompt (hide the PW). + self._transport.auth_password(username, password or '') def _paramiko_auth_agent(self, username, password=None): keys = paramiko.Agent().get_keys() @@ -229,8 +254,8 @@ def _paramiko_auth_agent(self, username, password=None): for key in keys: try: fp = hexlify(key.get_fingerprint()) - self._dbg(1, 'Trying SSH agent key %s' % fp) - self.client.auth_publickey(username, key) + self._dbg(2, 'Trying SSH agent key %s' % fp) + self._transport.auth_publickey(username, key) return except SSHException as e: saved_exception = e @@ -247,8 +272,8 @@ def _paramiko_auth_key(self, username, keys, password): try: key = pkey_class.from_private_key_file(filename, password) fp = hexlify(key.get_fingerprint()) - self._dbg(1, 'Trying key %s in %s' % (fp, filename)) - self.client.auth_publickey(username, key) + self._dbg(2, 'Trying key %s in %s' % (fp, filename)) + self._transport.auth_publickey(username, key) return except SSHException as e: saved_exception = e @@ -270,10 +295,26 @@ def _paramiko_auth_autokey(self, username, password): self._paramiko_auth_key(username, keyfiles, password) def _get_auth_methods(self, allowed_types): + if self.preferred_authentications: + self._dbg(3, 'Preferred auth. types: %s' % self.preferred_authentications) + requested_types = [] + for method in self.preferred_authentications: + if method not in allowed_types: + self._dbg(3, 'Preferred auth. type %s is not allowed.' % repr(method)) + continue + requested_types.append(method) + if not requested_types: + raise BadAuthenticationType( + 'invalid preferred auth. types: %s' % repr(self.preferred_authentications), + allowed_types, + ) + else: + requested_types = allowed_types + auth_methods = [] - for method in allowed_types: + for method in requested_types: if method not in auth_types: - self._dbg(1, 'Unsupported auth method %s' % repr(method)) + self._dbg(1, 'Unsupported auth. type: %s' % repr(method)) continue for type_name in auth_types[method]: auth_methods.append(getattr(self, type_name)) @@ -283,12 +324,19 @@ def _paramiko_auth(self, username, password): # Try authentication using auth_none. This should (almost) always fail, # but provides us with info about allowed authentication types. try: - self.client.auth_none(username) + self._dbg(2, 'Authenticating with client.auth_none().') + self._transport.auth_none(username) except BadAuthenticationType as err: - self._dbg(1, 'auth_none failed, supported: %s' % err.allowed_types) + self._dbg(2, 'Authentication with client.auth_none() failed, allowed: %s' % err.allowed_types) auth_methods = self._get_auth_methods(err.allowed_types) else: return + finally: + # Expose the authentication banner to the user. + banner = self.get_banner() + if banner: + self._receive_cb(banner) + self._buffer.append(banner) # Finally try all supported login methods. errors = [] @@ -296,93 +344,103 @@ def _paramiko_auth(self, username, password): # Some OSes (e.g. JunOS ERX OS, Huawei) do not accept further login # attempts after failing one. So in this hack, we # re-connect after each attempt... - if self.get_driver().reconnect_between_auth_methods or not self.client.active: + if ( + self.get_driver().reconnect_between_auth_methods + or not self._transport.active + ): + if self.get_driver().reconnect_between_auth_methods: + self._dbg(1, 'Forced reconnect by driver.') + else: + self._dbg(1, 'Forced reconnect by inactive client.') self.close(force=True) - self.client = self._paramiko_connect() + # Some devices (e.g. Cisco ASR9010) seem to have tight rate limits + # for reconnects, so connect() waits some time before the reconnect. + self.connect(self.host, self.port) - self._dbg(1, 'Authenticating with %s' % method.__name__) + self._dbg(2, 'Authenticating with %s().' % method.__name__) try: method(username, password) return except BadHostKeyException as e: - msg = '%s: Bad host key: %s' % (method.__name__, str(e)) + msg = 'Bad host key with %s(): %s' % (method.__name__, str(e)) self._dbg(1, msg) errors.append(msg) except AuthenticationException as e: - msg = 'Authentication with %s failed' % method.__name__ - msg += ': ' + str(e) + msg = 'Authentication with %s() failed: %s' % (method.__name__, str(e)) self._dbg(1, msg) errors.append(msg) except SSHException as e: - msg = '%s: SSHException: %s' % (method.__name__, str(e)) + msg = 'SSHException with %s(): %s' % (method.__name__, str(e)) self._dbg(1, msg) errors.append(msg) raise LoginFailure('Login failed: ' + '; '.join(errors)) - def _paramiko_shell(self): + def _paramiko_channel(self): rows, cols = get_terminal_size() - try: - self.shell = self.client.open_session() - self.shell.get_pty(self.termtype, cols, rows) - self.shell.invoke_shell() + self._channel = self._transport.open_session() + self._channel.get_pty(self.termtype, cols, rows) + self._channel.invoke_shell() + self._channel.set_name("pty-shell-%s" % self._channel.get_id()) except SSHException as e: - self._dbg(1, 'Failed to open shell.') - raise LoginFailure('Failed to open shell: ' + str(e)) + msg = 'Failed to open a channel with pty/shell. (%s)' % e + self._dbg(1, msg) + raise LoginFailure(msg) - def _connect_hook(self, hostname, port): - self.host = hostname - self.port = port or 22 - self.client = self._paramiko_connect() + def _connect(self): + self._transport = self._paramiko_connect() self._load_system_host_keys() return True def _protocol_authenticate(self, user, password): self._paramiko_auth(user, password) - self._paramiko_shell() + self._paramiko_channel() def _protocol_authenticate_by_key(self, user, key): # Allow multiple key files. key_file = key.get_filename() - - # Python3 does not support unicode and base string - if sys.version_info >= (3,0): - _str_types = (str) - else: - _str_types = (str, unicode, basestring) - if key_file is None: key_file = [] - elif isinstance(key_file, (str, _str_types)): + elif isinstance(key_file, str): key_file = [key_file] # Try each key. keys = [] for file in key_file: keys.append((keymap[key.get_type()], file)) - self._dbg(1, 'authenticating using _paramiko_auth_key().') self._paramiko_auth_key(user, keys, key.get_password()) - self._paramiko_shell() + # Open the channel. + self._paramiko_channel() def get_banner(self): - if not self.client: + if not self._transport: return None - return self.client.get_banner() + banner = self._transport.get_banner() + if isinstance(banner, bytes): + return banner.decode(self.encoding) + return banner def get_remote_version(self): - if not self.client: + if not self._transport: return None - return self.client.remote_version + version = self._transport.remote_version + if isinstance(version, bytes): + return version.decode(self.encoding) + return version - def send(self, data): - self._dbg(4, 'Sending %s' % repr(data)) - self.shell.sendall(data) + def _send(self, data): + try: + self._channel.sendall(data.encode(self.encoding)) + self._send_cb(data) + except Exception: + self._dbg(1, 'Error while writing to connection') + raise def _wait_for_data(self): end = time.time() + self.timeout while True: - readable, writeable, excp = select.select([self.shell], [], [], 1) + readable, writeable, excp = select.select([self._channel], [], [], 1) if readable: return True if time.time() > end: @@ -391,76 +449,77 @@ def _wait_for_data(self): def _fill_buffer(self): # Wait for a response of the device. if not self._wait_for_data(): - error = 'Timeout while waiting for response from device' - raise TimeoutException(error) + self._raise_on_response_timeout() # Read the response. - data = self.shell.recv(200) + data = self._channel.recv(1024).decode(self.encoding) if not data: return False - data = data.decode(self.encoding) - self._receive_cb(data, False) - self.buffer.append(data) + self._receive_cb(data) + self._buffer.append(data) return True - def _domatch(self, prompt, flush): - self._dbg(1, "Expecting a prompt") - self._dbg(2, "Expected pattern: " + - repr([repr(p.pattern) for p in prompt])) + def _domatch_buffer(self, prompt): + # Check whether the buffered data we received so far matches the prompt. + driver = self.get_driver() search_window_size = 150 - while not self.cancel: - # Check whether what's buffered matches the prompt. - driver = self.get_driver() - search_window = self.buffer.tail(search_window_size) - search_window, incomplete_tail = driver.clean_response_for_re_match( - search_window) - match = None - for n, regex in enumerate(prompt): - match = regex.search(search_window) - if match is not None: - break + search_window = self._buffer.tail(search_window_size) + search_window, incomplete_tail = driver.clean_response_for_re_match( + search_window + ) + n, match = -1, None + for n, regex in enumerate(prompt): + match = regex.search(search_window) + if match is not None: + break + if match: + self._dbg(2, "Got a prompt, match was: %s" % repr(match.group())) + return n, match, search_window + + def _domatch_extract(self, match, search_window_length, flush): + # This extracts the response from the buffer. The end position is + # where the matched prompt starts. So the response is everything + # in the buffer up to this end position. + end = self._buffer.size() - search_window_length + match.start() + response = self._buffer.head(end) + if flush: + prompt_length = match.end() - match.start() + self._buffer.pop(end + prompt_length) + self._dbg(4, "Response (flush=%s): %s" % (repr(flush), repr(response))) + return response + def _domatch(self, prompt, flush): + while not self._expect_canceled: + n, match, search_window = self._domatch_buffer(prompt) if not match: - if not self._fill_buffer(): - error = 'EOF while waiting for response from device' - raise ProtocolException(error) + if not self._fill_buffer(): # reads the channel + self._raise_on_response_eof() continue - - end = self.buffer.size() - len(search_window) + match.start() - if flush: - self.response = self.buffer.pop(end) - self.buffer.pop(match.end() - match.start()) - else: - self.response = self.buffer.head(end) + self.response = self._domatch_extract(match, len(search_window), flush) return n, match - - # Ending up here, self.cancel_expect() was called. - self.cancel = False - if self.driver_replaced: - self.driver_replaced = False - raise DriverReplacedException() - raise ExpectCancelledException() - - def cancel_expect(self): - self.cancel = True + self._raise_on_canceled_expect() def _set_terminal_size(self, rows, cols): - self.shell.resize_pty(cols, rows) - - def interact(self, key_handlers=None, handle_window_size=True): - return self._open_shell(self.shell, key_handlers, handle_window_size) - - def close(self, force=False): - if self.shell is None: - super(SSH2, self).close() - return - if not force: - self._fill_buffer() - self.shell.close() - self.shell = None - self.client.close() - self.client = None - self.sock.close() - self.sock = None - self.buffer.clear() - super(SSH2, self).close() + self._channel.resize_pty(cols, rows) + + def _interact(self, key_handlers=None, handle_window_size=True): + return self._open_shell(self._channel, key_handlers, handle_window_size) + + def _close(self, force=False): + if self._channel: + if not force: + try: + if self._channel.recv_ready(): + self._fill_buffer() + except Exception: + pass + self._channel.close() + self._channel = None + + if self._transport: + self._transport.close() + self._transport = None + + if self._sock: + self._sock.close() + self._sock = None diff --git a/Exscript/protocols/telnet.py b/Exscript/protocols/telnet.py index 298adaea..3fdb8f1e 100644 --- a/Exscript/protocols/telnet.py +++ b/Exscript/protocols/telnet.py @@ -23,14 +23,14 @@ """ The Telnet protocol. """ -from __future__ import absolute_import, unicode_literals -from future import standard_library -standard_library.install_aliases() +import logging + from ..util.tty import get_terminal_size from . import telnetlib from .protocol import Protocol -from .exception import ProtocolException, TimeoutException, \ - DriverReplacedException, ExpectCancelledException + +_logger = logging.getLogger(__name__) +_logger.addHandler(logging.NullHandler()) class Telnet(Protocol): @@ -41,91 +41,92 @@ class Telnet(Protocol): def __init__(self, **kwargs): Protocol.__init__(self, **kwargs) - self.tn = None + # Module-Level + self._logger = _logger + # Internals + self._tn = None def _telnetlib_received(self, data): - self._receive_cb(data, False) - self.buffer.append(data) + self._receive_cb(data) + self._buffer.append(data) - def _connect_hook(self, hostname, port): - assert self.tn is None + def _connect(self): + assert self._tn is None rows, cols = get_terminal_size() - self.tn = telnetlib.Telnet(hostname, - port or 23, - encoding=self.encoding, - connect_timeout=self.connect_timeout, - termsize=(rows, cols), - termtype=self.termtype, - stderr=self.stderr, - receive_callback=self._telnetlib_received) + self._tn = telnetlib.Telnet( + self.host, + self.port, + encoding=self.encoding, + connect_timeout=self.connect_timeout, + termsize=(rows, cols), + termtype=self.termtype, + stderr=self.stderr, + receive_callback=self._telnetlib_received, + ) + self._tn.cancel_expect = self._expect_canceled # sync states if self.debug >= 5: - self.tn.set_debuglevel(1) - if self.tn is None: - return False + self._tn.set_debuglevel(1) return True - def send(self, data): - self._dbg(4, 'Sending %s' % repr(data)) + def _send(self, data): try: - self.tn.write(data) + self._tn.write(data) # telnetlib handles encoding + self._send_cb(data) except Exception: self._dbg(1, 'Error while writing to connection') raise def _domatch(self, prompt, flush): if flush: - func = self.tn.expect + func = self._tn.expect else: - func = self.tn.waitfor + func = self._tn.waitfor # Wait for a prompt. - clean = self.get_driver().clean_response_for_re_match - self.response = None + cleanup_callback = self.get_driver().clean_response_for_re_match try: result, match, self.response = func( - prompt, self.timeout, cleanup=clean) + prompt, self.timeout, cleanup=cleanup_callback) except Exception: self._dbg(1, 'Error while waiting for ' + repr(prompt)) raise if match: - self._dbg(2, "Got a prompt, match was %s" % repr(match.group())) - self.buffer.pop(len(self.response)) - - self._dbg(5, "Response was %s" % repr(self.response)) + self._dbg(2, "Got a prompt, match was: %s" % repr(match.group())) + self._dbg(4, "Response (flush=%s): %s" % (repr(flush), repr(self.response))) + if flush: + # Only to mimic the telnetlib cooked-queue behavior to keep the + # buffer clean for the buffer monitoring feature. + prompt_length = match.end() - match.start() + self._buffer.pop(len(self.response) + prompt_length) if result == -1: - error = 'Error while waiting for response from device' - raise TimeoutException(error) + self._raise_on_response_timeout() if result == -2: - if self.driver_replaced: - self.driver_replaced = False - raise DriverReplacedException() - else: - raise ExpectCancelledException() + self._raise_on_canceled_expect() if self.response is None: - raise ProtocolException('whoops - response is None') + self._raise_on_response_eof() return result, match def cancel_expect(self): - self.tn.cancel_expect = True + super(Telnet, self).cancel_expect() + if self._tn: + self._tn.cancel_expect = True def _set_terminal_size(self, rows, cols): - self.tn.set_window_size(rows, cols) - - def interact(self, key_handlers=None, handle_window_size=True): - return self._open_shell(self.tn.sock, key_handlers, handle_window_size) - - def close(self, force=False): - if self.tn is None: - return - if not force: - try: - self.response = self.tn.read_all() - except Exception: - pass - self.tn.close() - self.tn = None - self.buffer.clear() - super(Telnet, self).close() + self._tn.set_window_size(rows, cols) + + def _interact(self, key_handlers=None, handle_window_size=True): + return self._open_shell(self._tn.sock, key_handlers, handle_window_size) + + def _close(self, force=False): + if self._tn: + if not force: + try: + self._tn.sock.settimeout(2) + self._tn.read_all() + except Exception: + pass + self._tn.close() + self._tn = None diff --git a/Exscript/protocols/telnetlib.py b/Exscript/protocols/telnetlib.py index 63301a11..cfbe9d25 100644 --- a/Exscript/protocols/telnetlib.py +++ b/Exscript/protocols/telnetlib.py @@ -56,27 +56,21 @@ option on one of the read calls only """ -from __future__ import print_function -from future import standard_library -standard_library.install_aliases() -from builtins import chr -from builtins import bytes -from builtins import range -from builtins import object - -# Imported modules -import sys -import time -import socket +import logging import select import selectors -from time import monotonic as _time +import socket import struct +import sys +import time from io import StringIO +from time import monotonic as _time __all__ = ["Telnet"] -py2 = sys.version_info[0] == 2 + +_logger = logging.getLogger(__name__) +_logger.addHandler(logging.NullHandler()) # Tunable parameters DEBUGLEVEL = 0 @@ -164,7 +158,7 @@ else: _TelnetSelector = selectors.SelectSelector -class Telnet(object): +class Telnet: """Telnet interface class. @@ -180,7 +174,7 @@ class Telnet(object): raise EOFError when the end of the connection is read, because they can return an empty string for other reasons. See the individual doc strings. - + read_until(expected, [timeout]) Read until the expected string has been seen, or a timeout is hit (default is no timeout); may block. @@ -208,7 +202,7 @@ class Telnet(object): I/O. """ - def __init__(self, host=None, port=0, encoding='latin1', **kwargs): + def __init__(self, host=None, port=0, encoding='latin-1', **kwargs): """Constructor. When called without arguments, create an unconnected instance. @@ -216,6 +210,7 @@ def __init__(self, host=None, port=0, encoding='latin1', **kwargs): number is optional. """ + self._logger = _logger self.debuglevel = DEBUGLEVEL self.can_naws = False self.host = host @@ -276,12 +271,10 @@ def msg(self, msg, *args): """ if self.debuglevel > 0: - self.stderr.write('Telnet(%s,%d): ' % (self.host, self.port)) + msg = '' + msg if args: - self.stderr.write(msg % args) - else: - self.stderr.write(msg) - self.stderr.write('\n') + msg = msg % args + self._logger.debug(msg) def set_debuglevel(self, debuglevel): """Set the debug level. @@ -319,9 +312,9 @@ def write(self, buffer): buffer = buffer.encode(self.encoding) if IAC in buffer: buffer = buffer.replace(IAC, IAC+IAC) - self.msg("send %s", repr(buffer)) + self.msg("Send: %s", repr(buffer)) self.sock.send(buffer) - + def read_until(self, match, timeout=None): """Read until a given string is encountered or until timeout. When no match is found, return whatever is available instead, @@ -534,12 +527,13 @@ def process_rawq(self): if (iac, end) != (IAC, SE): # whoops, that's an unexpected response... self.msg( - 'expected IAC SE, but got %d %d', ord(iac), ord(end)) + 'Expected IAC SE, but got %d %d.', ord(iac), ord(end) + ) self.msg('IAC SUBCOMMAND_END') # Send the next supported terminal. - ttype = self.termtype.encode('latin1') - self.msg('indicating support for terminal type %s', ttype) + ttype = self.termtype.encode('latin-1') + self.msg('Indicating support for terminal type %s.', ttype) self.sock.send(IAC+SB+TTYPE+theNULL+ttype+IAC+SE) elif command in (WILL, WONT): opt = self.rawq_getchar() @@ -550,7 +544,7 @@ def process_rawq(self): else: self.sock.send(IAC+DONT+opt) else: - self.msg('IAC %d not recognized' % ord(command)) + self.msg('IAC %d not recognized.' % ord(command)) except EOFError: # raised by self.rawq_getchar() pass buf = buf.decode(self.encoding) @@ -561,8 +555,8 @@ def process_rawq(self): def rawq_getchar(self): """Get next char from raw queue. - Block if no data is immediately available. Raise EOFError - when connection is closed. + Block if no data is immediately available. Raise EOFError when + connection is closed. """ if not self.rawq: @@ -570,10 +564,8 @@ def rawq_getchar(self): if self.eof: raise EOFError - # headaches for Py2/Py3 compatibility... c = self.rawq[self.irawq] - if not py2: - c = c.to_bytes((c.bit_length()+7)//8, 'big') + c = c.to_bytes((c.bit_length()+7)//8, 'big') self.irawq += 1 if self.irawq >= len(self.rawq): @@ -594,7 +586,7 @@ def fill_rawq(self): # The buffer size should be fairly small so as to avoid quadratic # behavior in process_rawq() above. buf = self.sock.recv(64) - self.msg("recv %s", repr(buf)) + self.msg("Recv: %s", repr(buf)) self.eof = (not buf) self.rawq = self.rawq + buf @@ -667,14 +659,14 @@ def _waitfor(self, relist, timeout=None, flush=False, cleanup=None): if not re: import re relist[i] = re.compile(relist[i]) - self.msg("Expecting %s" % [l.pattern for l in relist]) + self.msg("Expecting: %s" % [l.pattern for l in relist]) incomplete_tail = u'' clean_sw_size = search_window_size while True: self.process_rawq() if self.cancel_expect: self.cancel_expect = False - self.msg('cancelling expect()') + self.msg('Cancelling expect()') return -2, None, '' qlen = self.cookedq.tell() if cleanup: @@ -697,8 +689,9 @@ def _waitfor(self, relist, timeout=None, flush=False, cleanup=None): for i in indices: m = relist[i].search(search_window) if m is not None: + self.msg("Match found: %s" % repr(m.group())) e = m.end() - m.start() - e = qlen - e + 1 + e = qlen - e self.cookedq.seek(0) text = self.cookedq.read(e) if flush: @@ -746,14 +739,14 @@ def waitfor(self, relist, timeout=None, cleanup=None): or if more than one expression can match the same input, the results are undeterministic, and may depend on the I/O timing. """ - return self._waitfor(relist, timeout, False, cleanup) + return self._waitfor(relist, timeout, flush=False, cleanup=cleanup) def expect(self, relist, timeout=None, cleanup=None): """ Like waitfor(), but removes the matched data from the incoming buffer. """ - return self._waitfor(relist, timeout, True, cleanup=cleanup) + return self._waitfor(relist, timeout, flush=True, cleanup=cleanup) def test(): diff --git a/Exscript/queue.py b/Exscript/queue.py index 19541014..c3eaddcd 100644 --- a/Exscript/queue.py +++ b/Exscript/queue.py @@ -23,25 +23,26 @@ """ The heart of Exscript. """ -from __future__ import division, absolute_import -from builtins import str -from builtins import object -import sys -import os import gc -import select +import logging +import os +import sys import threading import weakref from functools import partial from multiprocessing import Pipe -from .util.cast import to_hosts -from .util.tty import get_terminal_size -from .util.impl import format_exception, serializeable_sys_exc_info -from .util.decorator import get_label + from .account import AccountManager, AccountProxy -from .logger import logger_registry, LoggerProxy -from .workqueue import WorkQueue, Task +from .logger import LoggerProxy, logger_registry from .protocols import prepare +from .util.cast import to_hosts +from .util.decorator import get_label +from .util.impl import format_exception, serializeable_sys_exc_info +from .util.tty import get_terminal_size +from .workqueue import Task, WorkQueue + +_logger = logging.getLogger(__name__) +_logger.addHandler(logging.NullHandler()) def _account_factory(accm, host, account): @@ -86,6 +87,10 @@ def _wrapped(job, *args, **kwargs): log_cb = partial(proxy.log, job_id) proxy.add_log(job_id, job.name, job.failures + 1) conn.data_received_event.listen(log_cb) + if not conn._receives_echo: + # The sent commands are either logged via the data_received_event + # when they are echoed back, or they are logged here explicitly. + conn.data_sent_event.listen(log_cb) try: conn.connect(host.get_address(), host.get_tcp_port()) result = func(job, host, conn, *args, **kwargs) @@ -97,6 +102,8 @@ def _wrapped(job, *args, **kwargs): proxy.log_succeeded(job_id) finally: conn.data_received_event.disconnect(log_cb) + if not conn._receives_echo: + conn.data_sent_event.disconnect(log_cb) else: conn.connect(host.get_address(), host.get_tcp_port()) result = func(job, host, conn, *args, **kwargs) @@ -187,7 +194,7 @@ def run(self): self._handle_request(request) -class Queue(object): +class Queue: """ Manages hosts/tasks, accounts, connections, and threads. @@ -241,7 +248,9 @@ def __init__(self, :type stderr: file :param stderr: The error channel, defaults to sys.stderr. """ + self._logger = _logger self.workqueue = WorkQueue(mode=mode) + self.workqueue.set_debug(verbose) self.account_manager = AccountManager() self.pipe_handlers = weakref.WeakValueDictionary() self.domain = domain @@ -250,9 +259,9 @@ def __init__(self, self.stderr = stderr self.host_driver = host_driver self.exc_cb = exc_cb + self.lock = threading.RLock() self.devnull = open(os.devnull, 'w') - self.channel_map = {'fatal_errors': self.stderr, - 'debug': self.stdout} + self.channel_map = {'fatal_errors': self.stderr} self.completed = 0 self.total = 0 self.failed = 0 @@ -299,8 +308,9 @@ def _update_verbosity(self): self.channel_map['tracebacks'] = self.stderr def _write(self, channel, msg): - self.channel_map[channel].write(msg) - self.channel_map[channel].flush() + with self.lock: + self.channel_map[channel].write(msg) + self.channel_map[channel].flush() def _create_pipe(self): """ @@ -375,10 +385,10 @@ def _print(self, channel, msg): self._write(channel, msg + '\n') self._print_status_bar() - def _dbg(self, level, msg): + def _dbg(self, level, msg, exc_info=None): if level > self.verbose: return - self._print('debug', msg) + self._logger.info(msg, exc_info=exc_info) def _on_job_init(self, job): if job.data is None: @@ -407,17 +417,16 @@ def _on_job_error(self, job, exc_info): def _on_job_succeeded(self, job): self._on_job_destroy(job) self.completed += 1 - self._print('status_bar', job.name + ' succeeded.') - self._dbg(2, job.name + ' job is done.') self._del_status_bar() + self._write('status_bar', job.name + ' succeeded.' + '\n') self._print_status_bar(exclude=job.name) def _on_job_aborted(self, job): self._on_job_destroy(job) self.completed += 1 self.failed += 1 - self._print('errors', job.name + ' finally failed.') self._del_status_bar() + self._write('errors', job.name + ' finally failed.' + '\n') self._print_status_bar(exclude=job.name) def set_max_threads(self, n_connections): @@ -527,6 +536,7 @@ def join(self): self.workqueue.wait_until_done() for child in list(self.pipe_handlers.values()): child.join() + self._dbg(3, 'Queue has finished.') self._del_status_bar() self._print_status_bar() gc.collect() @@ -546,9 +556,9 @@ def shutdown(self, force=False): if not force: self.join() - self._dbg(2, 'Shutting down queue...') + self._dbg(2, 'Shutting down the queue.') self.workqueue.shutdown(True) - self._dbg(2, 'Queue shut down.') + self._dbg(3, 'Queue has shut down.') self._del_status_bar() def destroy(self, force=False): @@ -564,28 +574,28 @@ def destroy(self, force=False): if not force: self.join() finally: - self._dbg(2, 'Destroying queue...') + self._dbg(2, 'Destroying the queue.') self.workqueue.destroy() self.account_manager.reset() self.completed = 0 self.total = 0 self.failed = 0 self.status_bar_length = 0 - self._dbg(2, 'Queue destroyed.') + self._dbg(3, 'Queue was destroyed.') self._del_status_bar() def reset(self): """ Remove all accounts, hosts, etc. """ - self._dbg(2, 'Resetting queue...') + self._dbg(2, 'Resetting the queue.') self.account_manager.reset() self.workqueue.shutdown(True) self.completed = 0 self.total = 0 self.failed = 0 self.status_bar_length = 0 - self._dbg(2, 'Queue reset.') + self._dbg(3, 'Queue was reset.') self._del_status_bar() def _run(self, hosts, callback, queue_function, *args): diff --git a/Exscript/servers/__init__.py b/Exscript/servers/__init__.py index 01f90617..a5244dfa 100644 --- a/Exscript/servers/__init__.py +++ b/Exscript/servers/__init__.py @@ -23,11 +23,11 @@ """ Very simple servers, useful for emulating a device for testing. """ -from __future__ import absolute_import -from .telnetd import Telnetd -from .sshd import SSHd +import inspect + from .httpd import HTTPd +from .sshd import SSHd +from .telnetd import Telnetd -import inspect __all__ = [name for name, obj in list(locals().items()) if not (name.startswith('_') or inspect.ismodule(obj))] diff --git a/Exscript/servers/httpd.py b/Exscript/servers/httpd.py index b7f10e0c..e704417a 100644 --- a/Exscript/servers/httpd.py +++ b/Exscript/servers/httpd.py @@ -23,19 +23,19 @@ """ A threaded HTTP server with support for HTTP/Digest authentication. """ -from __future__ import print_function -from future import standard_library -standard_library.install_aliases() -import sys -import time -import urllib.request, urllib.parse, urllib.error import hashlib -from urllib.parse import urlparse, parse_qs -from urllib.request import parse_http_list, parse_keqv_list -from traceback import format_exc -from http.server import BaseHTTPRequestHandler -from http.server import HTTPServer +import logging +import time +import urllib.parse +from http.server import BaseHTTPRequestHandler, HTTPServer from socketserver import ThreadingMixIn +from traceback import format_exc +from urllib.parse import parse_qs, urlparse +from urllib.request import parse_http_list, parse_keqv_list + +_logger = logging.getLogger(__name__) +_logger.addHandler(logging.NullHandler()) + def md5hex(x): return hashlib.md5(x).hexdigest() @@ -51,12 +51,7 @@ def _parse_url(path): """Given a urlencoded path, returns the path and the dictionary of query arguments, all in Unicode.""" - # path changes from bytes to Unicode in going from Python 2 to - # Python 3. - if sys.version_info[0] < 3: - o = urlparse(urllib.parse.unquote_plus(path).decode('utf8')) - else: - o = urlparse(urllib.parse.unquote_plus(path)) + o = urlparse(urllib.parse.unquote_plus(path)) path = o.path args = {} @@ -179,6 +174,7 @@ def __init__(self, addr, handler_cls, user_data=None): :type user_data: object :param user_data: Optional data that, stored in self.user_data. """ + self._logger = _logger self.debug = False self.realm = default_realm self.accounts = {} @@ -205,9 +201,10 @@ def get_password(self, username): """ return self.accounts.get(username) - def _dbg(self, msg): - if self.debug: - print(msg) + def _dbg(self, msg, exc_info=None): + if not self.debug: + return + self._logger.debug(msg, exc_info=exc_info) class RequestHandler(BaseHTTPRequestHandler): diff --git a/Exscript/servers/server.py b/Exscript/servers/server.py index c61efc2e..e4a18244 100644 --- a/Exscript/servers/server.py +++ b/Exscript/servers/server.py @@ -23,14 +23,21 @@ """ Base class for all servers. """ -from __future__ import print_function -from builtins import str +import logging import select import socket -from multiprocessing import Process, Pipe +import threading +from multiprocessing import Pipe +_logger = logging.getLogger(__name__) +_logger.addHandler(logging.NullHandler()) -class Server(Process): + +# This formerly was derived from multiprocessing.Process but as POSIX Python +# does not support child processes in threaded environments/apps, we had to +# switch to threaded servers too. This will hopefully add more stability to the +# test suite. +class Server(threading.Thread): """ Base class of the Telnet and SSH servers. Servers are intended to be @@ -52,36 +59,38 @@ def __init__(self, host, port, device, encoding='utf8'): :type host: str :param host: The address against which the daemon binds. - :type port: str + :type port: int :param port: The TCP port on which to listen. :type device: VirtualDevice :param device: A virtual device instance. :type encoding: str :param encoding: The encoding of data between client and server. """ - Process.__init__(self, target=self._run) + threading.Thread.__init__(self, target=self._run, daemon=True) + self._logger = _logger + self.name = '%s-%s:%s' % (self._get_server_type(), host, port) self.host = host self.port = int(port) self.timeout = .5 self.dbg = 0 self.running = False - self.buf = b'' self.socket = None - self.device = device + self.device = device # Shared device does not support concurrent connections! self.encoding = encoding self.to_child, self.to_parent = Pipe() self.processes = [] - def _dbg(self, level, msg): - if self.dbg >= level: - print(self.host + ':' + str(self.port), '-', end=' ') - print(msg) + def _dbg(self, level, msg, exc_info=None): + if level > self.dbg: + return + self._logger.debug(msg, exc_info=exc_info) def _poll_child_process(self): if not self.to_parent.poll(): return False try: msg = self.to_parent.recv() + self._dbg(1, 'Received command via pipe from parent: %s' % repr(msg)) except socket.error: self.running = False return False @@ -95,34 +104,51 @@ def _poll_child_process(self): def _handle_connection(self, conn): raise NotImplementedError() + def _get_server_type(self): + return type(self).__name__.lower() + def _run(self): + self._dbg(1, 'Starting the %s server.' % self._get_server_type()) self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) self.socket.bind((self.host, self.port)) self.socket.listen(1) self.running = True + self._dbg(1, 'Server bound to: %s:%s' % (self.host, self.port)) - while self.running: - self._poll_child_process() - - r, w, x = select.select([self.socket], [], [], self.timeout) - if not r: - continue - - conn, addr = self.socket.accept() - proc = Process(target=self._handle_connection, args=(conn,)) - self.processes.append(proc) - proc.start() - - for proc in self.processes: - proc.join() - self.processes = [] - self.socket.close() + try: + while self.running: + self._poll_child_process() + # Purge finished handlers to void zombie accumulation. + self.processes = [p for p in self.processes if p.is_alive()] + + r, w, x = select.select([self.socket], [], [], self.timeout) + if not r: + continue + + conn, addr = self.socket.accept() + self._dbg(1, 'Accepted connection from: %s:%s' % (addr[0], addr[1])) + proc = threading.Thread( + target=self._handle_connection, + args=(conn,), + name='%s-handler-%s:%s' % ( + self._get_server_type(), addr[0], addr[1] + ), + daemon=True, + ) + self.processes.append(proc) + proc.start() + finally: + for proc in self.processes: + proc.join() + self.processes = [] + self.socket.close() def exit(self): """ Stop the daemon without waiting for the thread to terminate. """ + self._dbg(1, 'Shutdown the %s server.' % self._get_server_type()) self.to_child.send('shutdown') def exit_command(self, cmd): diff --git a/Exscript/servers/sshd.py b/Exscript/servers/sshd.py index 1b1342b5..5462a6d8 100644 --- a/Exscript/servers/sshd.py +++ b/Exscript/servers/sshd.py @@ -23,33 +23,44 @@ """ An SSH2 server. """ -from __future__ import absolute_import -import os import base64 +import logging +import os import socket import threading + try: import Cryptodome as Crypto except ImportError: import Crypto + import paramiko -from copy import deepcopy from paramiko import ServerInterface -from Exscript.version import __version__ + +from ..version import __version__ from .server import Server +_logger = logging.getLogger(__name__) +_logger.addHandler(logging.NullHandler()) + local_version = 'SSH-2.0-Exscript-' + __version__ + class _ParamikoServer(ServerInterface): - # 'data' is the output of base64.encodestring(str(key)) - data = b'AAAAB3NzaC1yc2EAAAABIwAAAIEAyO4it3fHlmGZWJaGrfeHOVY7RWO3P9M7hp'\ - b'fAu7jJ2d7eothvfeuoRFtJwhUmZDluRdFyhFY/hFAh76PJKGAusIqIQKlkJxMC'\ - b'KDqIexkgHAfID/6mqvmnSJf0b5W8v5h2pI/stOSwTQ+pxVhwJ9ctYDhRSlF0iT'\ - b'UWT10hcuO4Ks8=' - good_pub_key = paramiko.RSAKey(data=base64.decodestring(data)) + # Must match the public key of the test suite! + data = ( + b'AAAAB3NzaC1yc2EAAAADAQABAAABAQChbnHaPrmxa5NpvtKXSMfMJEklUlinrnaNc' + b'VsLZeQv+3DfZBrdR4pTwJVIeda60JS6IwqtAl+pkSgZxdgW1TTxteMYCNz/zCpJSC' + b'KQ6oVP3/DVyZYODkmZaLz44INFDQp6cOfFzrhjvyi1SsUAKHvJIhS74u1swGm7TbD' + b'Nw56WAhjeQqfk+mOcLdzwwSBYiw4q7TdJ/iVT3DFFcoikBajegUpbaG8GVypBkluN' + b'sW3FmKG3WH0+V6Zi8HrV2t+kh1jaSSOsZfN4QVNYKlDs+pn9ZJc/nhQ40wFPgWDSW' + b'rasB8FNmAy7rpRlfQJ6Mnv3ez2lbrhieMSsiwLoyPpqxn87' + ) + good_pub_key = paramiko.RSAKey(data=base64.decodebytes(data)) def __init__(self): self.event = threading.Event() + self._logger = _logger # Since each server is created in it's own thread, we must # re-initialize the random number generator to make sure that @@ -110,31 +121,31 @@ class SSHd(Server): def __init__(self, host, port, device, key=None): Server.__init__(self, host, port, device) + self._logger = _logger if key: keyfile = key.get_filename() else: keyfile = os.path.expanduser('~/.ssh/id_rsa') self.host_key = paramiko.RSAKey(filename=keyfile) - def _recvline(self, channel): - while not b'\n' in self.buf: + def _recvline(self, channel, buf): + while b'\n' not in buf: if not self.running: - return None + return None, buf try: data = channel.recv(1024) except socket.timeout: continue if not data: - self.running = False - return None - self.buf += data.replace(b'\r\n', b'\n').replace(b'\r', b'\n') - lines = self.buf.split(b'\n') - self.buf = b'\n'.join(lines[1:]) - return lines[0].decode(self.encoding) + '\n' + return None, buf # client disconnected — keep server running + buf += data.replace(b'\r\n', b'\n').replace(b'\r', b'\n') + lines = buf.split(b'\n') + return lines[0].decode(self.encoding) + '\n', b'\n'.join(lines[1:]) def _handle_connection(self, conn): t = paramiko.Transport(conn) t.local_version = local_version + t.name = '%s-server-%s:%s' % (self._get_server_type(), self.host, self.port) try: t.load_server_moduli() except: @@ -161,9 +172,6 @@ def _handle_connection(self, conn): conn.close() return - # Prepare virtual device. - device = deepcopy(self.device) - # wait for auth channel = t.accept(2) if channel is None: @@ -175,25 +183,29 @@ def _handle_connection(self, conn): try: # wait for shell request server.event.wait(10) - if not server.event.isSet(): + if not server.event.is_set(): self._dbg(1, 'Client never asked for a shell.') t.close() return # send the banner - res = self.device.init() - channel.send(res) + channel.send(self.device.init().encode('utf8')) # accept commands + buf = b'' while self.running: - line = self._recvline(channel) - if not line: - continue + line, buf = self._recvline(channel, buf) + if line is None: + break response = self.device.do(line) if response: - channel.send(response) - except socket.error as err: - self._dbg(1, 'Client disappeared: ' + str(err)) + channel.send(response.encode('utf8')) + except (socket.error, EOFError) as err: + msg = 'Client disappeared: ' + str(err) + self._dbg(1, msg) + except Exception as err: + msg = "%s in connection handler: %s" + self._dbg(1, msg % (type(err).__name__, str(err))) finally: # closing transport closes channel t.close() diff --git a/Exscript/servers/telnetd.py b/Exscript/servers/telnetd.py index cb743272..e88ba8e1 100644 --- a/Exscript/servers/telnetd.py +++ b/Exscript/servers/telnetd.py @@ -23,11 +23,15 @@ """ A Telnet server. """ -from __future__ import absolute_import, print_function +import logging import select -from copy import deepcopy +import socket + from .server import Server +_logger = logging.getLogger(__name__) +_logger.addHandler(logging.NullHandler()) + class Telnetd(Server): @@ -43,35 +47,43 @@ class Telnetd(Server): daemon.join() # Wait until it terminates. """ - def _recvline(self, conn): - while not b'\n' in self.buf: + def __init__(self, host, port, device): + Server.__init__(self, host, port, device) + self._logger = _logger + + def _recvline(self, conn, buf): + while b'\n' not in buf: r, w, x = select.select([conn], [], [], self.timeout) if not self.running: - return None + return None, buf if not r: continue - buf = conn.recv(1024) - if not buf: - self.running = False - return None - self.buf += buf.replace(b'\r\n', b'\n').replace(b'\r', b'\n') - lines = self.buf.split(b'\n') - self.buf = b'\n'.join(lines[1:]) - return lines[0].decode(self.encoding) + '\n' + data = conn.recv(1024) + if not data: + return None, buf # client disconnected — keep server running + buf += data.replace(b'\r\n', b'\n').replace(b'\r', b'\n') + lines = buf.split(b'\n') + return lines[0].decode(self.encoding) + '\n', b'\n'.join(lines[1:]) def _handle_connection(self, conn): try: - device = deepcopy(self.device) - conn.send(device.init().encode('utf8')) + # send the banner + conn.send(self.device.init().encode('utf8')) + # accept commands + buf = b'' while self.running: - line = self._recvline(conn) - if not line: - continue + line, buf = self._recvline(conn, buf) + if line is None: + break response = self.device.do(line) if response: conn.send(response.encode('utf8')) + except (socket.error, EOFError) as err: + msg = 'Client disappeared: ' + str(err) + self._dbg(1, msg) except Exception as err: - self._dbg(1, "_handle_connection(): %s" % err) + msg = "%s in connection handler: %s" + self._dbg(1, msg % (type(err).__name__, str(err))) finally: conn.close() diff --git a/Exscript/stdlib/__init__.py b/Exscript/stdlib/__init__.py index d15c46af..61ed369d 100644 --- a/Exscript/stdlib/__init__.py +++ b/Exscript/stdlib/__init__.py @@ -20,55 +20,57 @@ # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -from Exscript.stdlib import connection -from Exscript.stdlib import crypt -from Exscript.stdlib import file -from Exscript.stdlib import ipv4 -from Exscript.stdlib import list -from Exscript.stdlib import string -from Exscript.stdlib import mysys +from . import connection, crypt, file, ipv4, list, mysys, string functions = { - 'connection.authenticate': connection.authenticate, - 'connection.authorize': connection.authorize, - 'connection.auto_authorize': connection.auto_authorize, - 'connection.autoinit': connection.autoinit, - 'connection.close': connection.close, - 'connection.exec': connection.exec_, - 'connection.execline': connection.execline, - 'connection.guess_os': connection.guess_os, - 'connection.send': connection.send, - 'connection.sendline': connection.sendline, - 'connection.set_error': connection.set_error, - 'connection.set_prompt': connection.set_prompt, - 'connection.set_timeout': connection.set_timeout, - 'connection.wait_for': connection.wait_for, - 'crypt.otp': crypt.otp, - 'file.chmod': file.chmod, - 'file.clear': file.clear, - 'file.exists': file.exists, - 'file.mkdir': file.mkdir, - 'file.read': file.read, - 'file.rm': file.rm, - 'file.write': file.write, - 'ipv4.in_network': ipv4.in_network, - 'ipv4.mask': ipv4.mask, - 'ipv4.mask2pfxlen': ipv4.mask2pfxlen, - 'ipv4.pfxlen2mask': ipv4.pfxlen2mask, - 'ipv4.pfxmask': ipv4.pfxmask, - 'ipv4.network': ipv4.network, - 'ipv4.broadcast': ipv4.broadcast, - 'ipv4.remote_ip': ipv4.remote_ip, - 'list.new': list.new, - 'list.get': list.get, - 'list.length': list.length, - 'list.unique': list.unique, - 'string.replace': string.replace, - 'string.split': string.split, - 'string.tolower': string.tolower, - 'string.toupper': string.toupper, - 'sys.env': mysys.env, - 'sys.exec': mysys.execute, - 'sys.message': mysys.message, - 'sys.wait': mysys.wait, + 'connection.add_error': connection.add_error, + 'connection.add_prompt': connection.add_prompt, + 'connection.authenticate': connection.authenticate, + 'connection.authenticate_user': connection.authenticate_user, + 'connection.authorize': connection.authorize, + 'connection.auto_authorize': connection.auto_authorize, + 'connection.autoinit': connection.autoinit, + 'connection.close': connection.close, + 'connection.exec': connection.exec_, + 'connection.execline': connection.execline, + 'connection.expect': connection.expect, + 'connection.get_protocol': connection.get_protocol, + 'connection.guess_os': connection.guess_os, + 'connection.send': connection.send, + 'connection.sendline': connection.sendline, + 'connection.set_error': connection.set_error, + 'connection.set_prompt': connection.set_prompt, + 'connection.set_timeout': connection.set_timeout, + 'connection.wait_for': connection.wait_for, + 'crypt.otp': crypt.otp, + 'file.chmod': file.chmod, + 'file.clear': file.clear, + 'file.exists': file.exists, + 'file.mkdir': file.mkdir, + 'file.read': file.read, + 'file.rm': file.rm, + 'file.write': file.write, + 'ipv4.in_network': ipv4.in_network, + 'ipv4.mask': ipv4.mask, + 'ipv4.mask2pfxlen': ipv4.mask2pfxlen, + 'ipv4.pfxlen2mask': ipv4.pfxlen2mask, + 'ipv4.pfxmask': ipv4.pfxmask, + 'ipv4.network': ipv4.network, + 'ipv4.broadcast': ipv4.broadcast, + 'ipv4.remote_ip': ipv4.remote_ip, + 'list.new': list.new, + 'list.get': list.get, + 'list.length': list.length, + 'list.sort': list.sort, + 'list.reverse': list.reverse, + 'list.unique': list.unique, + 'string.replace': string.replace, + 'string.split': string.split, + 'string.strip': string.strip, + 'string.tolower': string.tolower, + 'string.toupper': string.toupper, + 'sys.env': mysys.env, + 'sys.exec': mysys.execute, + 'sys.message': mysys.message, + 'sys.wait': mysys.wait, } diff --git a/Exscript/stdlib/connection.py b/Exscript/stdlib/connection.py index 7775291c..63652cb1 100644 --- a/Exscript/stdlib/connection.py +++ b/Exscript/stdlib/connection.py @@ -20,8 +20,8 @@ # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -from Exscript import Account -from Exscript.stdlib.util import secure_function +from ..account import Account +from .util import secure_function @secure_function @@ -121,15 +121,19 @@ def autoinit(scope): @secure_function -def close(scope): +def close(scope, force=[False]): """ - Closes the existing connection with the remote host. This function is - rarely used, as normally Exscript closes the connection automatically + Closes the existing connection with the remote host gracefully, sending + the driver-specific logout command first. This function is rarely used + from templates, as normally Exscript closes the connection automatically when the script has completed. + + :type force: boolean + :param force: Whether the connection is closed without logout. """ conn = scope.get('__connection__') - conn.close(1) - scope.define(__response__=conn.response) + conn.close(force=force[0]) + scope.define(__response__=conn.get_response().split('\n')) return True @@ -152,7 +156,7 @@ def exec_(scope, data): for line in data: conn.send(line) conn.expect_prompt() - response += conn.response.split('\n')[1:] + response += conn.get_response().split('\n') scope.define(__response__=response) return True @@ -160,8 +164,7 @@ def exec_(scope, data): @secure_function def execline(scope, data): """ - Like exec(), but appends a newline to the command in data before sending - it. + Like exec(), but appends a newline to the data before sending it. :type data: string :param data: The data that is sent. @@ -169,12 +172,28 @@ def execline(scope, data): conn = scope.get('__connection__') response = [] for line in data: - conn.execute(line) - response += conn.response.split('\n')[1:] + conn.send(line + '\r') + conn.expect_prompt() + response += conn.get_response().split('\n') scope.define(__response__=response) return True +@secure_function +def get_protocol(scope): + """ + Returns the name of the connection protocol. + + This is the class name of the protocol adapter in lower-case without + any trailing digits. + + :rtype: string + :return: The connection protocol. + """ + conn = scope.get('__connection__') + return [conn.get_protocol()] + + @secure_function def guess_os(scope): """ @@ -195,7 +214,7 @@ def guess_os(scope): def send(scope, data): """ Like exec(), but does not wait for a response of the remote host after - sending the command. + sending the data. :type data: string :param data: The data that is sent. @@ -203,6 +222,7 @@ def send(scope, data): conn = scope.get('__connection__') for line in data: conn.send(line) + scope.define(__response__=conn.get_response().split('\n')) return True @@ -210,7 +230,7 @@ def send(scope, data): def sendline(scope, data): """ Like execline(), but does not wait for a response of the remote host after - sending the command. + sending the data. :type data: string :param data: The data that is sent. @@ -218,20 +238,39 @@ def sendline(scope, data): conn = scope.get('__connection__') for line in data: conn.send(line + '\r') + scope.define(__response__=conn.get_response().split('\n')) return True @secure_function -def wait_for(scope, prompt): +def expect(scope, prompt): """ Waits until the response of the remote host contains the given pattern. + Does consume the prompt, can be run only once, then it fails with a timeouts. + :type prompt: regex :param prompt: The prompt pattern. """ conn = scope.get('__connection__') conn.expect(prompt) - scope.define(__response__=conn.response) + scope.define(__response__=conn.get_response().split('\n')) + return True + + +@secure_function +def wait_for(scope, prompt): + """ + Waits until the response of the remote host contains the given pattern. + + Does not consume the prompt, can be rerun multiple times. + + :type prompt: regex + :param prompt: The prompt pattern. + """ + conn = scope.get('__connection__') + conn.waitfor(prompt) + scope.define(__response__=conn.get_response().split('\n')) return True @@ -252,7 +291,22 @@ def set_prompt(scope, prompt=None): :param prompt: The prompt pattern. """ conn = scope.get('__connection__') - conn.set_prompt(prompt) + conn.set_prompt(prompt, extend=False) + return True + + +@secure_function +def add_prompt(scope, prompt): + """ + Extends the current list of patterns, that are recognized at any future time + when Exscript needs to wait for a prompt, each time when called. Use + set_prompt() instead to replace the current/default pattern. + + :type prompt: regex + :param prompt: The additional prompt pattern. + """ + conn = scope.get('__connection__') + conn.set_prompt(prompt, extend=True) return True @@ -270,7 +324,22 @@ def set_error(scope, error_re=None): :param error_re: The error pattern. """ conn = scope.get('__connection__') - conn.set_error_prompt(error_re) + conn.set_error_prompt(error_re, extend=False) + return True + + +@secure_function +def add_error(scope, error_re): + """ + Extends the list of patterns, that, whenever detected in the response of the + remote host, cause an error to be raised. Use set_error() instead to replace + the current/default pattern. + + :type error_re: regex + :param error_re: The additional error pattern. + """ + conn = scope.get('__connection__') + conn.set_error_prompt(error_re, extend=True) return True diff --git a/Exscript/stdlib/crypt.py b/Exscript/stdlib/crypt.py index 970f3ed3..3b2f6095 100644 --- a/Exscript/stdlib/crypt.py +++ b/Exscript/stdlib/crypt.py @@ -20,8 +20,8 @@ # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -from Exscript.util import crypt -from Exscript.stdlib.util import secure_function +from ..util import crypt +from .util import secure_function @secure_function @@ -34,7 +34,7 @@ def otp(scope, password, seed, seqs): :type password: string :param password: A password. :type seed: string - :param seed: A username. + :param seed: A cryptographic seed. :type seqs: int :param seqs: A sequence number, or a list of sequence numbers. :rtype: string diff --git a/Exscript/stdlib/list.py b/Exscript/stdlib/list.py index 53ffb57b..ec01675b 100644 --- a/Exscript/stdlib/list.py +++ b/Exscript/stdlib/list.py @@ -20,18 +20,18 @@ # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -from Exscript.stdlib.util import secure_function +from .util import secure_function @secure_function -def new(scope): +def new(scope, *args): """ - Returns a new, empty list. + Returns a new list with optional initial values. - :rtype: string - :return: The model of the remote device. + :rtype: list + :return: A new list with optional initial values. """ - return [] + return [] + list(arg[0] for arg in args) @secure_function @@ -39,8 +39,10 @@ def length(scope, mylist): """ Returns the number of items in the list. - :rtype: string - :return: The model of the remote device. + :type mylist: list + :param mylist: A list of strings. + :rtype: int + :return: The length """ return [len(mylist)] @@ -49,14 +51,14 @@ def length(scope, mylist): def get(scope, source, index): """ Returns a copy of the list item with the given index. - It is an error if an item with teh given index does not exist. + It is an error if an item with the given index does not exist. - :type source: string + :type source: list :param source: A list of strings. - :type index: string - :param index: A list of strings. + :type index: int + :param index: The index of the item to retrieve. :rtype: string - :return: The cleaned up list of strings. + :return: The string at the given index. """ try: index = int(index[0]) @@ -69,6 +71,29 @@ def get(scope, source, index): except IndexError: raise ValueError('no such item in the list') +@secure_function +def sort(scope, source): + """ + Returns a copy of the given list sorted in ascending order. + + :type source: list + :param source: A list of strings. + :rtype: list + :return: The sorted list of strings. + """ + return sorted(source) + +@secure_function +def reverse(scope, source): + """ + Returns a copy of the given list with the items in reverse order. + + :type source: list + :param source: A list of strings. + :rtype: list + :return: The reversed list of strings. + """ + return list(reversed(source)) @secure_function def unique(scope, source): @@ -76,9 +101,9 @@ def unique(scope, source): Returns a copy of the given list in which all duplicates are removed such that one of each item remains in the list. - :type source: string + :type source: list :param source: A list of strings. - :rtype: string + :rtype: list :return: The cleaned up list of strings. """ return list(dict([(a, 1) for a in source]).keys()) diff --git a/Exscript/stdlib/mysys.py b/Exscript/stdlib/mysys.py index 44802b97..cde6d512 100644 --- a/Exscript/stdlib/mysys.py +++ b/Exscript/stdlib/mysys.py @@ -21,10 +21,11 @@ # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. import os -import time import sys -from subprocess import Popen, PIPE, STDOUT -from Exscript.stdlib.util import secure_function +import time +from subprocess import PIPE, STDOUT, Popen + +from .util import secure_function def env(scope, varname): @@ -50,8 +51,12 @@ def execute(scope, command): stdin=PIPE, stdout=PIPE, stderr=STDOUT, + text=True, close_fds=True) - scope.define(__response__=process.stdout.read()) + if process.stdout: + scope.define(__response__=process.stdout.readlines()) + else: + scope.define(__response__=['']) return True diff --git a/Exscript/stdlib/string.py b/Exscript/stdlib/string.py index 416f0872..85d6606b 100644 --- a/Exscript/stdlib/string.py +++ b/Exscript/stdlib/string.py @@ -20,7 +20,7 @@ # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -from Exscript.stdlib.util import secure_function +from .util import secure_function @secure_function @@ -44,14 +44,12 @@ def replace(scope, strings, source, dest): def split(scope, strings, separator): """ Returns a list with the split values of the given string (or list of strings). - The values are split at the seperator + The values are split at the separator. :type strings: string :param strings: A string, or a list of strings. - :type source: string - :param source: What to replace. - :type dest: string - :param dest: What to replace it with. + :type separator: string + :param separator: The separator to split the string at. :rtype: string :return: The resulting string, or list of strings. """ @@ -61,6 +59,22 @@ def split(scope, strings, separator): return result +@secure_function +def strip(scope, strings, chars=[None]): + """ + Returns a copy of the given string (or list of strings) with leading and + trailing whitespace or custom characters removed. + + :type strings: string + :param strings: A string, or a list of strings. + :type chars: string + :param chars: The characters to remove. + :rtype: string + :return: The resulting string, or list of strings. + """ + return [s.strip(chars[0]) for s in strings] + + @secure_function def tolower(scope, strings): """ diff --git a/Exscript/stdlib/util.py b/Exscript/stdlib/util.py index 08322727..eec04c4b 100644 --- a/Exscript/stdlib/util.py +++ b/Exscript/stdlib/util.py @@ -24,8 +24,13 @@ def secure_function(function): + """Decorator to indicate a secure function in scripts. + + The function is secure, if it does not have any side effects on and does not + read data from the system where the interpreter is running. + """ @wraps(function) def wrapper(*args, **kwargs): return function(*args, **kwargs) - wrapper._is_secure = True + setattr(wrapper, '_is_secure', True) return wrapper diff --git a/Exscript/util/buffer.py b/Exscript/util/buffer.py index 9a5a9877..04114462 100644 --- a/Exscript/util/buffer.py +++ b/Exscript/util/buffer.py @@ -23,14 +23,12 @@ """ A buffer object. """ -from __future__ import absolute_import -from builtins import object -from builtins import str from io import StringIO + from .cast import to_regexs -class MonitoredBuffer(object): +class MonitoredBuffer: """ A specialized string buffer that allows for monitoring diff --git a/Exscript/util/cast.py b/Exscript/util/cast.py index b0946e78..bcf0d0c0 100644 --- a/Exscript/util/cast.py +++ b/Exscript/util/cast.py @@ -23,11 +23,11 @@ """ Handy shortcuts for converting types. """ -from __future__ import absolute_import -from builtins import str import re + import Exscript + def to_list(item): """ If the given item is iterable, this function returns the given item. @@ -44,7 +44,7 @@ def to_list(item): return [item] -def to_host(host, default_protocol='telnet', default_domain=''): +def to_host(host, default_protocol='ssh', default_domain=''): """ Given a string or a Host object, this function returns a Host object. @@ -66,7 +66,7 @@ def to_host(host, default_protocol='telnet', default_domain=''): return Exscript.Host(host, default_protocol=default_protocol) -def to_hosts(hosts, default_protocol='telnet', default_domain=''): +def to_hosts(hosts, default_protocol='ssh', default_domain=''): """ Given a string or a Host object, or a list of strings or Host objects, this function returns a list of Host objects. diff --git a/Exscript/util/collections.py b/Exscript/util/collections.py index fbf37982..44c52d99 100644 --- a/Exscript/util/collections.py +++ b/Exscript/util/collections.py @@ -1,17 +1,6 @@ -from __future__ import unicode_literals, absolute_import -from future import standard_library -standard_library.install_aliases() -from builtins import str -from builtins import chr -from builtins import range -from builtins import object import copy -import sys -if sys.version_info[1] >= 10: - from collections.abc import Callable - from collections import OrderedDict -else: - from collections import OrderedDict, Callable +from collections import OrderedDict +from collections.abc import Callable class OrderedDefaultDict(OrderedDict): diff --git a/Exscript/util/crypt.py b/Exscript/util/crypt.py index d371ec14..3ab7c8e9 100644 --- a/Exscript/util/crypt.py +++ b/Exscript/util/crypt.py @@ -23,14 +23,9 @@ """ Encryption related utilities. """ -from __future__ import print_function, absolute_import -from builtins import chr -from builtins import ord -from builtins import range -from builtins import bytes -import sys -import string import random +import string + try: from Cryptodome.Hash import MD4 except ImportError: @@ -323,27 +318,16 @@ def _long_from_raw(thehash): def _sixword_from_raw(key): return _sixword_from_long(_long_from_raw(key)) -if sys.version_info[0] < 3: - def _fold_md4_or_md5(digest): - if len(digest) < 16: - raise ValueError('digest is too short') - result = b'' - for i in range(0, 8): - one = ord(bytes(digest[i])) - two = ord(bytes(digest[i+8])) - result = result + bytes([one^two]) - return result -else: - def _fold_md4_or_md5(digest): - if len(digest) < 16: - raise ValueError('digest is too short') - result = b'' - for i in range(0, 8): - #print(len(digest[i])) - one = ord(bytes([digest[i]])) - two = ord(bytes([digest[i+8]])) - result = result + bytes([one^two]) - return result +def _fold_md4_or_md5(digest): + if len(digest) < 16: + raise ValueError('digest is too short') + result = b'' + for i in range(0, 8): + #print(len(digest[i])) + one = ord(bytes([digest[i]])) + two = ord(bytes([digest[i+8]])) + result = result + bytes([one^two]) + return result def otp(password, seed, sequence): """ diff --git a/Exscript/util/decorator.py b/Exscript/util/decorator.py index 3ffbb624..80d812a9 100644 --- a/Exscript/util/decorator.py +++ b/Exscript/util/decorator.py @@ -23,9 +23,8 @@ """ Decorators for callbacks passed to Queue.run(). """ -from __future__ import absolute_import -from .impl import add_label, get_label, copy_labels from ..protocols.exception import LoginFailure +from .impl import add_label, copy_labels, get_label # noqa: F401 def bind(function, *args, **kwargs): diff --git a/Exscript/util/event.py b/Exscript/util/event.py index 0b4bf45a..e9d1a1a4 100644 --- a/Exscript/util/event.py +++ b/Exscript/util/event.py @@ -23,13 +23,12 @@ """ A simple signal/event mechanism. """ -from __future__ import print_function, absolute_import -from builtins import object from threading import Lock + from . import weakmethod -class Event(object): +class Event: """ A simple signal/event mechanism, to be used like this:: diff --git a/Exscript/util/file.py b/Exscript/util/file.py index f7db3a39..c3f51493 100644 --- a/Exscript/util/file.py +++ b/Exscript/util/file.py @@ -23,22 +23,18 @@ """ Utilities for reading data from files. """ -from __future__ import print_function, absolute_import -from builtins import str -from future import standard_library -standard_library.install_aliases() -import sys -import re -import os import base64 -import codecs -import imp # Py2 -import importlib # Py3 +import binascii +import configparser +import importlib +import os +import re + from .. import Account from .cast import to_host -def get_accounts_from_file(filename): +def get_accounts_from_file(filename, encoding='utf-8', password_encoding='utf-8'): """ Reads a list of user/password combinations from the given file and returns a list of Account instances. The file content @@ -54,26 +50,33 @@ def get_accounts_from_file(filename): Each password needs to be base64 encrypted. To encrypt a password, you may use the following command:: - python -c 'import base64; print(base64.b64encode("thepassword"))' + python3 -c 'import base64; print(base64.b64encode("thepassword".encode()).decode())' :type filename: string :param filename: The name of the file containing the list of accounts. + :type encoding: str + :param encoding: The encoding of the file. + :type password_encoding: str + :param password_encoding: The encoding of the original passwords. :rtype: list[Account] :return: The newly created account instances. """ accounts = [] - cfgparser = __import__('configparser', {}, {}, ['']) - parser = cfgparser.RawConfigParser() + parser = configparser.RawConfigParser() parser.optionxform = str - parser.read(filename) + parser.read(filename, encoding) for user, password in parser.items('account-pool'): - password = base64.decodebytes(password.encode('latin1')) - accounts.append(Account(user, password.decode('latin1'))) + try: + password = base64.decodebytes(password.encode('ascii')) + accounts.append(Account(user, password.decode(password_encoding))) + except (UnicodeDecodeError, binascii.Error) as exc: + msg = 'Could not decode password for user: %s (%s)' % (user, exc) + raise Exception(msg) return accounts def get_hosts_from_file(filename, - default_protocol='telnet', + default_protocol='ssh', default_domain='', remove_duplicates=False, encoding='utf-8'): @@ -100,7 +103,7 @@ def get_hosts_from_file(filename, # Read the hostnames. have = set() hosts = [] - with codecs.open(filename, 'r', encoding) as file_handle: + with open(filename, 'r', encoding=encoding) as file_handle: for line in file_handle: hostname = line.split('#')[0].strip() if hostname == '': @@ -114,7 +117,7 @@ def get_hosts_from_file(filename, def get_hosts_from_csv(filename, - default_protocol='telnet', + default_protocol='ssh', default_domain='', encoding='utf-8'): """ @@ -155,7 +158,7 @@ def get_hosts_from_csv(filename, if not os.path.exists(filename): raise IOError('No such file: %s' % filename) - with codecs.open(filename, 'r', encoding) as file_handle: + with open(filename, 'r', encoding=encoding) as file_handle: # Read and check the header. header = file_handle.readline().rstrip() if re.search(r'^(?:hostname|address)\b', header) is None: @@ -213,11 +216,6 @@ def load_lib(filename): callables. The namespace is the basename of the file, without file extension. - The result of this function can later be passed to run_template:: - - functions = load_lib('my_library.py') - run_template(conn, 'foo.exscript', **functions) - :type filename: string :param filename: A full filename. :rtype: dict[string->object] @@ -228,9 +226,6 @@ def load_lib(filename): raise IOError('No such file: %s' % filename) name = os.path.splitext(os.path.basename(filename))[0] - if sys.version_info[0] < 3: - module = imp.load_source(name, filename) - else: - module = importlib.machinery.SourceFileLoader(name, filename).load_module() + module = importlib.machinery.SourceFileLoader(name, filename).load_module() return dict((name + '.' + k, v) for (k, v) in list(module.__lib__.items())) diff --git a/Exscript/util/impl.py b/Exscript/util/impl.py index 836bfb23..b655f3bc 100644 --- a/Exscript/util/impl.py +++ b/Exscript/util/impl.py @@ -23,10 +23,9 @@ """ Development tools. """ -from builtins import object import sys -import warnings import traceback +import warnings from functools import wraps @@ -121,9 +120,9 @@ def deprecation(msg): """ Prints a deprecation warning. """ - warnings.warn('deprecated', + warnings.warn(msg, category=DeprecationWarning, - stacklevel=2) + stacklevel=3) def deprecated(func): @@ -132,9 +131,7 @@ def deprecated(func): a printed warning message when the function is used. """ def decorated(*args, **kwargs): - warnings.warn('Call to deprecated function %s.' % func.__name__, - category=DeprecationWarning, - stacklevel=2) + deprecation('Call to deprecated function %s.' % func.__name__) return func(*args, **kwargs) decorated.__name__ = func.__name__ decorated.__doc__ = func.__doc__ @@ -178,7 +175,7 @@ def wrapped(*args, **kwargs): return wrapped -class Decorator(object): +class Decorator: def __init__(self, obj): self.__dict__['obj'] = obj diff --git a/Exscript/util/interact.py b/Exscript/util/interact.py index f13fba49..00d20bb4 100644 --- a/Exscript/util/interact.py +++ b/Exscript/util/interact.py @@ -23,24 +23,16 @@ """ Tools for interacting with the user on the command line. """ -from __future__ import print_function, absolute_import -from future import standard_library -standard_library.install_aliases() -from builtins import input -from builtins import str -from builtins import object +import configparser +import getpass import os import sys -import getpass -import configparser -import codecs -import shutil -from tempfile import NamedTemporaryFile + from .. import Account from .cast import to_list -class InputHistory(object): +class InputHistory: """ When prompting a user for input it is often useful to record his @@ -72,16 +64,9 @@ def __init__(self, """ self.section = section self.parser = configparser.ConfigParser() - filename = os.path.expanduser(filename) + self.filename = os.path.expanduser(filename) - try: - self.file = open(filename, 'a+') - except IOError: - import warnings - warnings.warn('could not open %s, using tempfile' % filename) - self.file = NamedTemporaryFile() - - self.parser.readfp(self.file) + self.parser.read(self.filename) if not self.parser.has_section(self.section): self.parser.add_section(self.section) @@ -124,18 +109,8 @@ def set(self, key, value): return None self.parser.set(self.section, key, value) - - # Unfortunately ConfigParser attempts to write a string to the file - # object, and NamedTemporaryFile uses binary mode. So we nee to create - # the tempfile, and then re-open it. - with NamedTemporaryFile(delete=False) as tmpfile: - pass - with codecs.open(tmpfile.name, 'w', encoding='utf8') as fp: + with open(self.filename, mode="w") as fp: self.parser.write(fp) - - self.file.close() - shutil.move(tmpfile.name, self.file.name) - self.file = open(self.file.name) return value diff --git a/Exscript/util/ipv4.py b/Exscript/util/ipv4.py index c2c4fe4a..0a1f8452 100644 --- a/Exscript/util/ipv4.py +++ b/Exscript/util/ipv4.py @@ -23,12 +23,10 @@ """ IPv4 address calculation and conversion. """ -from builtins import str -from builtins import range -import socket -import struct import math import re +import socket +import struct def _least_bit(number): diff --git a/Exscript/util/ipv6.py b/Exscript/util/ipv6.py index bf3cc08c..4e3b81db 100644 --- a/Exscript/util/ipv6.py +++ b/Exscript/util/ipv6.py @@ -23,7 +23,6 @@ """ IPv6 address calculation and conversion. """ -from builtins import range def is_ip(string): diff --git a/Exscript/util/log.py b/Exscript/util/log.py index 30e4ff97..4088e44a 100644 --- a/Exscript/util/log.py +++ b/Exscript/util/log.py @@ -47,10 +47,10 @@ def decorator(function): def log_to_file(logdir, mode='a', delete=False, clearmem=True): """ - Like :class:`log_to()`, but automatically creates a new FileLogger + Like :func:`log_to`, but automatically creates a new FileLogger instead of having one passed. Note that the logger stays alive (in memory) forever. If you need - to control the lifetime of a logger, use :class:`log_to()` instead. + to control the lifetime of a logger, use :func:`log_to` instead. """ logger = FileLogger(logdir, mode, delete, clearmem) _loggers.append(logger) diff --git a/Exscript/util/mail.py b/Exscript/util/mail.py index 694dbbf6..37d1ee1c 100644 --- a/Exscript/util/mail.py +++ b/Exscript/util/mail.py @@ -23,21 +23,20 @@ """ Sending and formatting emails. """ -from builtins import str -from builtins import object +import mimetypes import os -import time import re -import socket import smtplib -import mimetypes -from getpass import getuser +import socket +import time from email import encoders -from email.mime.multipart import MIMEMultipart from email.mime.audio import MIMEAudio from email.mime.base import MIMEBase from email.mime.image import MIMEImage +from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText +from getpass import getuser + from Exscript.util.event import Event # @@ -47,7 +46,7 @@ _string_re = re.compile(r'(\\?){([\w_]+)}', re.I) -class _TemplateParser(object): +class _TemplateParser: """ This exists for backward compatibility; Python 2.3 does not come @@ -112,7 +111,7 @@ def _cleanup_mail_addresses(recipients): # -class Mail(object): +class Mail: """ Represents an email. diff --git a/Exscript/util/match.py b/Exscript/util/match.py index c6e3fa89..e5b93e25 100644 --- a/Exscript/util/match.py +++ b/Exscript/util/match.py @@ -23,8 +23,8 @@ """ Shorthands for regular expression matching. """ -from __future__ import print_function, absolute_import import re + from ..protocols import Protocol @@ -43,7 +43,7 @@ def _first_match(string, compiled): def first_match(string, regex, flags=re.M): - """ + r""" Matches the given string against the given regex. - If no match is found and the regular expression has zero or one @@ -82,12 +82,12 @@ def first_match(string, regex, flags=re.M): :return: A match, or a tuple of matches. """ if isinstance(string, Protocol): - string = string.response + string = string.get_response() return _first_match(string, re.compile(regex, flags)) def any_match(string, regex, flags=re.M): - """ + r""" Matches the given string against the given regex. - If no match is found, this function returns an empty list. @@ -103,7 +103,7 @@ def any_match(string, regex, flags=re.M): This behavior ensures that the following can never fail:: - foo = '1 uno\\n2 due' + foo = '1 uno\n2 due' for m in any_match(foo, r'aaa'): # Returns [] print(m) @@ -132,7 +132,7 @@ def any_match(string, regex, flags=re.M): :return: A list of strings, or a list of tuples. """ if isinstance(string, Protocol): - string = string.response + string = string.get_response() compiled = re.compile(regex, flags) results = [] if compiled.groups <= 1: diff --git a/Exscript/util/pidutil.py b/Exscript/util/pidutil.py index 94705ff4..2ebcaa5e 100644 --- a/Exscript/util/pidutil.py +++ b/Exscript/util/pidutil.py @@ -23,11 +23,13 @@ """ Handling PID (process id) files. """ -from builtins import str -import os -import logging -import fcntl import errno +import fcntl +import logging +import os + +_logger = logging.getLogger(__name__) +_logger.addHandler(logging.NullHandler()) def read(path): @@ -42,7 +44,7 @@ def read(path): :return: The PID, or none if the file was not found. """ # Try to read the pid from the pidfile. - logging.info("Checking pidfile '%s'", path) + _logger.info("Checking pidfile '%s'", path) try: return int(open(path).read()) except IOError as xxx_todo_changeme: @@ -91,7 +93,7 @@ def kill(path): return # Try to kill the process. - logging.info("Killing PID %s", pid) + _logger.info("Killing PID %s", pid) try: os.kill(pid, 9) except OSError as xxx_todo_changeme2: @@ -110,7 +112,7 @@ def write(path): :param path: The name of the pidfile. """ pid = os.getpid() - logging.info("Writing PID %s to '%s'", pid, path) + _logger.info("Writing PID %s to '%s'", pid, path) try: pidfile = open(path, 'wb') # get a non-blocking exclusive lock @@ -134,7 +136,7 @@ def remove(path): :type path: str :param path: The name of the pidfile. """ - logging.info("Removing pidfile '%s'", path) + _logger.info("Removing pidfile '%s'", path) try: os.unlink(path) except IOError: diff --git a/Exscript/util/sigint.py b/Exscript/util/sigint.py index b2eb16c0..fcaf4201 100644 --- a/Exscript/util/sigint.py +++ b/Exscript/util/sigint.py @@ -23,14 +23,13 @@ """ A class for catching SIGINT, such that CTRL+c works. """ -from __future__ import print_function -from builtins import object import os -import sys import signal +import sys +import time -class SigIntWatcher(object): +class SigIntWatcher: """ This class solves two problems with multithreaded programs in Python: @@ -60,19 +59,38 @@ def __init__(self): except RuntimeError: # prevent "not holding the import lock" on some systems self.child = 0 if self.child: - self.watch() + self._watch() - def watch(self): + def _watch(self): try: pid, status = os.wait() except KeyboardInterrupt: print('********** SIGINT RECEIVED - SHUTTING DOWN! **********') - self.kill() + # Child may also receive SIGINT, so we wait a bit for its cleanup. + deadline = time.monotonic() + 2.0 + while time.monotonic() < deadline: + try: + pid, status = os.waitpid(self.child, os.WNOHANG) + if pid: # child exited on its own + self._exit_from_status(status) + time.sleep(0.05) + except ChildProcessError: + sys.exit(1) # child already gone + except KeyboardInterrupt: + pass # second CTRL+C -> simply continue waiting + self._kill() sys.exit(1) - sys.exit(status >> 8) + self._exit_from_status(status) - def kill(self): + def _kill(self): try: os.kill(self.child, signal.SIGKILL) except OSError: pass + + def _exit_from_status(self, status): + if os.WIFEXITED(status): + sys.exit(os.WEXITSTATUS(status)) + else: + # Follows shell convention for signal-killed child processes. + sys.exit(128 + os.WTERMSIG(status)) diff --git a/Exscript/util/start.py b/Exscript/util/start.py index b9433850..ef62336a 100644 --- a/Exscript/util/start.py +++ b/Exscript/util/start.py @@ -23,10 +23,9 @@ """ Quickstart methods for the Exscript queue. """ -from __future__ import print_function, absolute_import from .. import Queue +from .decorator import autoauthenticate, autologin from .interact import read_login -from .decorator import autologin, autoauthenticate def run(users, hosts, func, **kwargs): diff --git a/Exscript/util/syslog.py b/Exscript/util/syslog.py index 0228cec5..0a177e4d 100644 --- a/Exscript/util/syslog.py +++ b/Exscript/util/syslog.py @@ -24,22 +24,17 @@ Send messages to a syslog server. """ import os -import sys -import imp import socket - -# This way of loading a module prevents Python from looking in the -# current directory. (We need to avoid it due to the syslog module -# name collision.) -syslog = imp.load_module('syslog', *imp.find_module('syslog')) +import sys +import syslog as _syslog def netlog(message, source=None, host='localhost', port=514, - priority=syslog.LOG_DEBUG, - facility=syslog.LOG_USER): + priority=_syslog.LOG_DEBUG, + facility=_syslog.LOG_USER): """ Python's built in syslog module does not support networking, so this is the alternative. diff --git a/Exscript/util/template.py b/Exscript/util/template.py index 3e252457..37c78c6e 100644 --- a/Exscript/util/template.py +++ b/Exscript/util/template.py @@ -26,24 +26,32 @@ from Exscript import stdlib from Exscript.interpreter import Parser +debug = 0 + def _compile(conn, filename, template, parser_kwargs, **kwargs): + undefined = 'undefined' if conn: hostname = conn.get_host() - account = conn.last_account - username = account is not None and account.get_name() or None + account = conn._last_account + username = account is not None and account.get_name() or undefined else: - hostname = 'undefined' - username = None + hostname = undefined + username = undefined # Init the parser. + if "debug" not in parser_kwargs: + parser_kwargs["debug"] = debug parser = Parser(**parser_kwargs) # Define the built-in variables and functions. - builtin = dict(__filename__=[filename or 'undefined'], - __username__=[username], - __hostname__=[hostname], - __connection__=conn) + builtin = dict( + __filename__=[filename or undefined], + __username__=[username], + __hostname__=[hostname], + __response__=[''], + __connection__=conn, + ) parser.define_object(**builtin) parser.define_object(**stdlib.functions) @@ -59,35 +67,28 @@ def _run(conn, filename, template, parser_kwargs, **kwargs): return compiled.execute() -def test(string, **kwargs): +def test(string, secure=False, **kwargs): """ Compiles the given template, and raises an exception if that failed. Does nothing otherwise. - :type string: string - :param string: The template to compile. - :type kwargs: dict - :param kwargs: Variables to define in the template. - """ - _compile(None, None, string, {}, **kwargs) - - -def test_secure(string, **kwargs): - """ - Like test(), but makes sure that each function that is used in + With secure mode enabled it makes sure that each function that is used in the template has the Exscript.stdlib.util.safe_function decorator. - Raises Exscript.interpreter.Exception.PermissionError if any - function lacks the decorator. + Raises Exscript.interpreter.Exception.PermissionError if any function + lacks the decorator. :type string: string :param string: The template to compile. + :type secure: bool + :param secure: Whether to enable secure mode. :type kwargs: dict :param kwargs: Variables to define in the template. """ - _compile(None, None, string, {'secure': True}, **kwargs) + parser_args = {'secure': secure, 'no_escaping': False} + _compile(None, None, string, parser_args, **kwargs) -def test_file(filename, **kwargs): +def test_file(filename, secure=False, **kwargs): """ Convenience wrapper around test() that reads the template from a file instead. @@ -97,17 +98,17 @@ def test_file(filename, **kwargs): :type kwargs: dict :param kwargs: Variables to define in the template. """ + parser_args = {'secure': secure, 'no_escaping': False} with open(filename) as fp: - _compile(None, filename, fp.read(), {}, **kwargs) + _compile(None, filename, fp.read(), parser_args, **kwargs) -def eval(conn, string, strip_command=True, **kwargs): - """ - Compiles the given template and executes it on the given - connection. +def eval(conn, string, strip_command=True, no_escaping=False, **kwargs): + r""" + Compiles the given template and executes it on the given connection. Raises an exception if the compilation fails. - if strip_command is True, the first line of each response that is + If strip_command is True, the first line of each response that is received after any command sent by the template is stripped. For example, consider the following template:: @@ -124,7 +125,7 @@ def eval(conn, string, strip_command=True, **kwargs): myfile2 [...] - By setting strip_command to True, the first line is ommitted. + By setting strip_command to True, the first line is omitted. :type conn: Exscript.protocols.Protocol :param conn: The connection on which to run the template. @@ -132,16 +133,20 @@ def eval(conn, string, strip_command=True, **kwargs): :param string: The template to compile. :type strip_command: bool :param strip_command: Whether to strip the command echo from the response. + :type no_escaping: bool + :param no_escaping: Whether to disable advanced escape evaluation for + control characters, octal, hexadecimal and unicode character sequences. + Evaluation for CR, LF and TAB in strings is always enabled. :type kwargs: dict :param kwargs: Variables to define in the template. :rtype: dict :return: The variables that are defined after execution of the script. """ - parser_args = {'strip_command': strip_command} + parser_args = {'strip_command': strip_command, 'no_escaping': no_escaping} return _run(conn, None, string, parser_args, **kwargs) -def eval_file(conn, filename, strip_command=True, **kwargs): +def eval_file(conn, filename, strip_command=True, no_escaping=False, **kwargs): """ Convenience wrapper around eval() that reads the template from a file instead. @@ -152,15 +157,19 @@ def eval_file(conn, filename, strip_command=True, **kwargs): :param filename: The name of the template file. :type strip_command: bool :param strip_command: Whether to strip the command echo from the response. + :type no_escaping: bool + :param no_escaping: Whether to disable advanced escape evaluation for + control characters, octal, hexadecimal and unicode character sequences. + Evaluation for CR, LF and TAB in strings is always enabled. :type kwargs: dict :param kwargs: Variables to define in the template. """ - parser_args = {'strip_command': strip_command} + parser_args = {'strip_command': strip_command, 'no_escaping': no_escaping} with open(filename, 'r') as fp: return _run(conn, filename, fp.read(), parser_args, **kwargs) -def paste(conn, string, **kwargs): +def paste(conn, string, no_escaping=False, **kwargs): """ Compiles the given template and executes it on the given connection. This function differs from eval() such that it does not @@ -176,15 +185,20 @@ def paste(conn, string, **kwargs): :param conn: The connection on which to run the template. :type string: string :param string: The template to compile. + :type no_escaping: bool + :param no_escaping: Whether to disable advanced escape evaluation for + control characters, octal, hexadecimal and unicode character sequences. + Evaluation for CR, LF and TAB in strings is always enabled. :type kwargs: dict :param kwargs: Variables to define in the template. :rtype: dict :return: The variables that are defined after execution of the script. """ - return _run(conn, None, string, {'no_prompt': True}, **kwargs) + parser_args = {'no_prompt': True, 'no_escaping': no_escaping} + return _run(conn, None, string, parser_args, **kwargs) -def paste_file(conn, filename, **kwargs): +def paste_file(conn, filename, no_escaping=False, **kwargs): """ Convenience wrapper around paste() that reads the template from a file instead. @@ -193,8 +207,13 @@ def paste_file(conn, filename, **kwargs): :param conn: The connection on which to run the template. :type filename: string :param filename: The name of the template file. + :type no_escaping: bool + :param no_escaping: Whether to disable advanced escape evaluation for + control characters, octal, hexadecimal and unicode character sequences. + Evaluation for CR, LF and TAB in strings is always enabled. :type kwargs: dict :param kwargs: Variables to define in the template. """ + parser_args = {'no_prompt': True, 'no_escaping': no_escaping} with open(filename, 'r') as fp: - return _run(conn, None, fp.read(), {'no_prompt': True}, **kwargs) + return _run(conn, filename, fp.read(), parser_args, **kwargs) diff --git a/Exscript/util/tty.py b/Exscript/util/tty.py index 3b338996..c26eb174 100644 --- a/Exscript/util/tty.py +++ b/Exscript/util/tty.py @@ -24,10 +24,10 @@ TTY utilities. """ import os -import sys import struct -from subprocess import Popen, PIPE -from builtins import int +import sys +from subprocess import PIPE, Popen + def _get_terminal_size(fd): try: diff --git a/Exscript/util/url.py b/Exscript/util/url.py index 5074ea04..746a8068 100644 --- a/Exscript/util/url.py +++ b/Exscript/util/url.py @@ -23,18 +23,12 @@ """ Working with URLs (as used in URL formatted hostnames). """ -from __future__ import unicode_literals, absolute_import -from future import standard_library -standard_library.install_aliases() -from builtins import str -from builtins import chr -from builtins import range -from builtins import object import re -from urllib.parse import urlencode, quote -from urllib.parse import urlparse, urlsplit +from urllib.parse import quote, urlencode, urlparse, urlsplit + from .collections import OrderedDefaultDict + def _make_hexmap(): hexmap = dict() for i in range(256): @@ -45,15 +39,16 @@ def _make_hexmap(): _HEXTOCHR = _make_hexmap() _WELL_KNOWN_PORTS = { - 'ftp': 21, - 'ssh': 22, - 'ssh1': 22, - 'ssh2': 22, + 'ftp': 21, + 'ssh': 22, + 'ssh2': 22, 'telnet': 23, - 'smtp': 25, - 'http': 80, - 'pop3': 110, - 'imap': 143 + 'smtp': 25, + 'http': 80, + 'pop3': 110, + 'imap': 143, + 'https': 443, + 'netconf': 830, } def _unquote(string): @@ -102,7 +97,7 @@ def _urlparse_qs(url): return result -class Url(object): +class Url: """ Represents a URL. @@ -164,7 +159,7 @@ def to_string(self): return str(self) @staticmethod - def from_string(url, default_protocol='telnet'): + def from_string(url, default_protocol='ssh'): """ Parses the given URL and returns an URL object. There are some differences to Python's built-in URL parser: diff --git a/Exscript/util/weakmethod.py b/Exscript/util/weakmethod.py index 569deb81..d5bbc665 100644 --- a/Exscript/util/weakmethod.py +++ b/Exscript/util/weakmethod.py @@ -23,7 +23,6 @@ """ Weak references to bound and unbound methods. """ -from builtins import object import weakref @@ -36,7 +35,7 @@ class DeadMethodCalled(Exception): pass -class WeakMethod(object): +class WeakMethod: """ Do not create this class directly; use :class:`ref()` instead. diff --git a/Exscript/util/xml.py b/Exscript/util/xml.py new file mode 100644 index 00000000..3930c712 --- /dev/null +++ b/Exscript/util/xml.py @@ -0,0 +1,63 @@ +# +# Copyright (C) 2010-2017 Samuel Abels +# The MIT License (MIT) +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files +# (the "Software"), to deal in the Software without restriction, +# including without limitation the rights to use, copy, modify, merge, +# publish, distribute, sublicense, and/or sell copies of the Software, +# and to permit persons to whom the Software is furnished to do so, +# subject to the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" +Working with XML. +""" + +from lxml import etree + + +class XMLValidationError(Exception): + """ + Raised when XML validation fails. + + Provides the XML string that failed in its `xmlstring` attribute. + """ + def __init__(self, *args: object, xmlstring=None) -> None: + super().__init__(*args) + self.xmlstring = xmlstring + + +def validate_xml(string, *, silent=False): + """ + Try to parse the given string as XML to verify its syntax. + + Raises UnicodeEncodeError when the string cannot be encoded to UTF-8. + Raises lxml.etree.XMLSyntaxError when the string is not valid XML. + + :param string: The string to validate. + :param silent: Whether to suppress the exceptions and just return a boolean. + :rtype: bool + :return: Whether the string is valid XML in UTF-8. + """ + try: + etree.fromstring(string.encode("utf-8")) + except UnicodeEncodeError: + if not silent: + raise + return False + except etree.XMLSyntaxError: + if not silent: + raise + return False + return True diff --git a/Exscript/workqueue/__init__.py b/Exscript/workqueue/__init__.py index 23191bc3..711dd7a3 100644 --- a/Exscript/workqueue/__init__.py +++ b/Exscript/workqueue/__init__.py @@ -20,11 +20,11 @@ # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -from __future__ import absolute_import -from .workqueue import WorkQueue -from .task import Task +import inspect + from .pipeline import Pipeline +from .task import Task +from .workqueue import WorkQueue -import inspect __all__ = [name for name, obj in list(locals().items()) if not (name.startswith('_') or inspect.ismodule(obj))] diff --git a/Exscript/workqueue/job.py b/Exscript/workqueue/job.py index 3ed4744e..143b53c9 100644 --- a/Exscript/workqueue/job.py +++ b/Exscript/workqueue/job.py @@ -1,5 +1,3 @@ -from builtins import str -from builtins import object # # Copyright (C) 2010-2017 Samuel Abels # The MIT License (MIT) @@ -22,12 +20,13 @@ # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +import multiprocessing import sys import threading -import multiprocessing from copy import copy from functools import partial from multiprocessing import Pipe + from Exscript.util.impl import serializeable_sys_exc_info @@ -93,7 +92,7 @@ def start(self, pipe): Process = _make_process_class(multiprocessing.Process, 'Process') -class Job(object): +class Job: __slots__ = ('id', 'func', 'name', diff --git a/Exscript/workqueue/mainloop.py b/Exscript/workqueue/mainloop.py index b588d6d5..be95bbf7 100644 --- a/Exscript/workqueue/mainloop.py +++ b/Exscript/workqueue/mainloop.py @@ -20,13 +20,16 @@ # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -from __future__ import print_function, absolute_import -from builtins import next -import threading +import logging import multiprocessing +import threading + from ..util.event import Event from .job import Job +_logger = logging.getLogger(__name__) +_logger.addHandler(logging.NullHandler()) + # See http://bugs.python.org/issue1731717 multiprocessing.process._cleanup = lambda: None @@ -35,6 +38,7 @@ class MainLoop(threading.Thread): def __init__(self, collection, job_cls): threading.Thread.__init__(self) + self._logger = _logger self.job_init_event = Event() self.job_started_event = Event() self.job_error_event = Event() @@ -43,12 +47,13 @@ def __init__(self, collection, job_cls): self.queue_empty_event = Event() self.collection = collection self.job_cls = job_cls - self.debug = 5 + self.debug = 0 self.daemon = True - def _dbg(self, level, msg): - if self.debug >= level: - print(msg) + def _dbg(self, level, msg, exc_info=None): + if level > self.debug: + return + self._logger.debug(msg, exc_info=exc_info) def enqueue(self, function, name, times, data): job = Job(function, name, times, data) @@ -94,30 +99,30 @@ def get_queue_length(self): def _on_job_completed(self, job, exc_info): # This function is called in a sub-thread, so we need to be # careful that we are not in a lock while sending an event. - self._dbg(1, 'Job "%s" called completed()' % job.name) + self._dbg(4, 'Job "%s" called completed()' % job.name) try: # Notify listeners of the error # *before* removing the job from the queue. # This is because wait_until_done() depends on # get_queue_length() being 0, and we don't want a listener - # to get a signal from a queue that already already had + # to get a signal from a queue that already had # wait_until_done() completed. if exc_info: - self._dbg(1, 'Error in job "%s"' % job.name) + self._dbg(3, 'Error in job "%s"' % job.name) job.failures += 1 self.job_error_event(job.child, exc_info) if job.failures >= job.times: - self._dbg(1, 'Job "%s" finally failed' % job.name) + self._dbg(3, 'Job "%s" finally failed' % job.name) self.job_aborted_event(job.child) else: - self._dbg(1, 'Job "%s" succeeded.' % job.name) + self._dbg(3, 'Job "%s" succeeded.' % job.name) self.job_succeeded_event(job.child) finally: # Remove the watcher from the queue, and re-enque if needed. if exc_info and job.failures < job.times: - self._dbg(1, 'Restarting job "%s"' % job.name) + self._dbg(3, 'Restarting job "%s"' % job.name) job.start(self.job_cls, self._on_job_completed) self.job_started_event(job.child) else: @@ -136,5 +141,4 @@ def run(self): self.job_init_event(job) job.start(self.job_cls, self._on_job_completed) self.job_started_event(job.child) - self._dbg(1, 'Job "%s" started.' % job.name) - self._dbg(2, 'Main loop terminated.') + self._dbg(3, 'Job "%s" started.' % job.name) diff --git a/Exscript/workqueue/pipeline.py b/Exscript/workqueue/pipeline.py index 6a7353fc..49bc8d0a 100644 --- a/Exscript/workqueue/pipeline.py +++ b/Exscript/workqueue/pipeline.py @@ -1,4 +1,3 @@ -from builtins import object # # Copyright (C) 2010-2017 Samuel Abels # The MIT License (MIT) @@ -21,12 +20,12 @@ # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -from uuid import uuid4 from collections import deque from multiprocessing import Condition, RLock +from uuid import uuid4 -class Pipeline(object): +class Pipeline: """ A collection that is similar to Python's Queue object, except diff --git a/Exscript/workqueue/task.py b/Exscript/workqueue/task.py index 564077b4..350fc203 100644 --- a/Exscript/workqueue/task.py +++ b/Exscript/workqueue/task.py @@ -23,11 +23,10 @@ """ Represents a batch of enqueued actions. """ -from builtins import object from Exscript.util.event import Event -class Task(object): +class Task: """ Represents a batch of running actions. diff --git a/Exscript/workqueue/workqueue.py b/Exscript/workqueue/workqueue.py index 911feebc..35a500a1 100644 --- a/Exscript/workqueue/workqueue.py +++ b/Exscript/workqueue/workqueue.py @@ -20,15 +20,18 @@ # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -from __future__ import absolute_import -from builtins import object +import logging + from ..util.event import Event -from .job import Thread, Process -from .pipeline import Pipeline +from .job import Process, Thread from .mainloop import MainLoop +from .pipeline import Pipeline +_logger = logging.getLogger(__name__) +_logger.addHandler(logging.NullHandler()) -class WorkQueue(object): + +class WorkQueue: """ This class implements the asynchronous workqueue and is the main API @@ -37,20 +40,19 @@ class WorkQueue(object): def __init__(self, collection=None, - debug=0, max_threads=1, mode='threading'): """ Constructor. - :type debug: int - :param debug: The debug level. :type max_threads: int :param max_threads: The maximum number of concurrent threads. """ + self._logger = _logger if mode == 'threading': self.job_cls = Thread elif mode == 'multiprocessing': + self._logger.warning("Mode 'multiprocessing' is not recommended, see: https://github.com/knipknap/exscript/issues/230") self.job_cls = Process else: raise TypeError('invalid "mode" argument: ' + repr(mode)) @@ -65,7 +67,7 @@ def __init__(self, self.job_succeeded_event = Event() self.job_aborted_event = Event() self.queue_empty_event = Event() - self.debug = debug + self.debug = 0 self.main_loop = None self._init() @@ -84,16 +86,21 @@ def _check_if_ready(self): if self.main_loop is None: raise Exception('main loop is already destroyed') - def set_debug(self, debug=1): + def _dbg(self, level, msg, exc_info=None): + if level > self.debug: + return + self._logger.debug(msg, exc_info=exc_info) + + def set_debug(self, level): """ Set the debug level. - :type debug: int - :param debug: The debug level. + :type level: int + :param level: The debug level. """ self._check_if_ready() - self.debug = debug - self.main_loop.debug = debug + self.debug = level + self.main_loop.debug = level def get_max_threads(self): """ diff --git a/Makefile b/Makefile index 3d1ef7de..0815d420 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ NAME=exscript VERSION=`python setup.py --version | sed s/^v//` PREFIX=/usr/local/ BIN_DIR=$(PREFIX)/bin -SITE_DIR=$(PREFIX)`python -c "from __future__ import print_function; import sys; from distutils.sysconfig import get_python_lib; print(get_python_lib()[len(sys.prefix):])"` +SITE_DIR=$(PREFIX)`python -c "import sys; from distutils.sysconfig import get_python_lib; print(get_python_lib()[len(sys.prefix):])"` ################################################################### # Standard targets. diff --git a/README.md b/README.md index b569b3ab..8bdf49f5 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,6 @@ # Exscript + [![Build Status](https://travis-ci.org/knipknap/exscript.svg?branch=master)](https://travis-ci.org/knipknap/exscript) [![Coverage Status](https://coveralls.io/repos/github/knipknap/exscript/badge.svg?branch=master)](https://coveralls.io/github/knipknap/exscript?branch=master) [![Code Climate](https://lima.codeclimate.com/github/knipknap/exscript/badges/gpa.svg)](https://lima.codeclimate.com/github/knipknap/exscript) @@ -8,23 +9,27 @@ ## Summary -Exscript is a Python module and a template processor for automating network -connections over protocols such as Telnet or SSH. We attempt to create the -best possible set of tools for working with Telnet and SSH. +Exscript is a Python package and a template processor for automating network connections +CLI-driven over protocols such as Telnet, SSH or alternatively structured Netconf. We +attempt to create the best possible set of tools for working with Telnet, SSH and +Netconf. Exscript also provides a set of tools and functions for sysadmins, that simplify **regular expression matching**, **reporting** by email, **logging**, or **syslog** handling, **CSV parsing**, **ip address handling**, **template processing**, and many more. -Exscript may be used to automate sessions with routers from Cisco, Juniper, -OneAccess, Huawei, or any others. If you want to configures machines -running Linux/Unix, IOS, IOS-XR, JunOS, VRP, or any other operating system -that can be used with a terminal, Exscript is what you are looking for. +Exscript may be used to automate sessions with routers from Cisco, Juniper, OneAccess, +Huawei, or any others. If you want to configures machines running Linux/Unix, IOS, +IOS-XR, JunOS, VRP, or any other operating system that can be used with a CLI terminal, +Exscript is what you are looking for. Additionally any system that supports SSH Netconf +1.0 and 1.1 can be automated with Exscript. ## Do you need commercial support? -Exscript is supported by [Procedure 8](https://procedure8.com). Get in touch if you need anything! + +Exscript is supported by [Procedure 8](https://procedure8.com). Get in touch if you need +anything! ## Method 1: Using Exscript with Python @@ -59,21 +64,50 @@ start(accounts, hosts, do_something, max_threads=2) Create a file named `test.exscript` with the following content: -``` +```plaintext uname -a ``` To run this Exscript template, just start Exscript using the following command: -``` +```shell exscript test.exscript ssh://localhost ``` -Awesome fact: Just replace `ssh://` by `telnet://` and it should still work with Telnet devices. +Awesome fact: Just replace `ssh://` by `telnet://` and it should still work with Telnet +devices. This also works for Netconf, just use `netconf://` instead and create a valid +Netconf template: +```xml + + + +``` -## Documentation +The RPC body will be sent to the device, automatically encapsulated with proper +rpc-tags like: + +```xml + + +` +``` + +No need to handle the message-id and namespaces manually. The reply will be available as +plaintext (like with CLI) or a parsed XML tree element. -Full documentation is here: +## Release-Management + +Releases are created on demand without any strict interval. The versioning follows the +packaging guidance for Python and supports the following canonical-like tag pattern, +optionally with local version specifiers, for the build process. + +```plaintext +N.N(.N)?[{a|b|rc}N][.postN][.devN][+{a..z0..9}(.{a..z0..9})*] +``` + +See also: [Python version-specifiers](https://packaging.python.org/en/latest/specifications/version-specifiers) + +## Documentation -http://exscript.readthedocs.io +Full documentation is here: http://exscript.readthedocs.io diff --git a/demos/multiple/accounts.cfg b/demos/multiple/accounts.cfg index f95447e4..2689ce89 100644 --- a/demos/multiple/accounts.cfg +++ b/demos/multiple/accounts.cfg @@ -1,5 +1,5 @@ [account-pool] # The password must be encrypted using the following command: -# python -c 'import base64; print base64.b64encode("thepassword")' +# python3 -c 'import base64; print(base64.b64encode("thepassword".encode()).decode())' user1 = dGhlcGFzc3dvcmQ= user2 = dGhlcGFzc3dvcmQ= diff --git a/demos/multiple/multiple.py b/demos/multiple/multiple.py index 8c25bf4a..3c029439 100644 --- a/demos/multiple/multiple.py +++ b/demos/multiple/multiple.py @@ -1,6 +1,6 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 from Exscript import Host -from Exscript.util.file import get_hosts_from_file, get_accounts_from_file +from Exscript.util.file import get_accounts_from_file, get_hosts_from_file from Exscript.util.start import start @@ -16,7 +16,7 @@ def one(job, host, conn): # Execute a simple command. conn.execute('show ip int brie') - print "myvariable is", conn.get_host().get('myvariable') + print("myvariable is", host.get('myvariable')) def two(job, host, conn): diff --git a/demos/quickstart/quickstart.py b/demos/quickstart/quickstart.py index d46ffd51..1a251fa7 100644 --- a/demos/quickstart/quickstart.py +++ b/demos/quickstart/quickstart.py @@ -1,16 +1,14 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 from Exscript.util.match import any_match -from Exscript.util.template import eval_file from Exscript.util.start import quickstart def do_something(job, host, conn): conn.execute('ls -1') files = any_match(conn, r'(\S+)') - print "Files found:", files + print("Files found:", files) -# Open a connection (Telnet, by default) to each of the hosts, and run -# do_something(). To open the connection via SSH, you may prefix the -# hostname by the protocol, e.g.: 'ssh://hostname', 'telnet://hostname', -# etc. +# Open a connection (SSH2, by default) to each of the hosts, and run do_something(). To +# open the connection via Telnet or SSH Netconf, you may prefix the hostname by the +# protocol, e.g.: 'telnet://hostname', 'netconf://hostname', etc. quickstart(('localhost', 'otherhost'), do_something) diff --git a/demos/report/accounts.cfg b/demos/report/accounts.cfg index 6041fb09..2689ce89 100644 --- a/demos/report/accounts.cfg +++ b/demos/report/accounts.cfg @@ -1,5 +1,5 @@ [account-pool] -# The password is encrypted using the following command: -# python -c 'import base64; print base64.b64encode("thepassword")' +# The password must be encrypted using the following command: +# python3 -c 'import base64; print(base64.b64encode("thepassword".encode()).decode())' user1 = dGhlcGFzc3dvcmQ= user2 = dGhlcGFzc3dvcmQ= diff --git a/demos/report/report.py b/demos/report/report.py index f781ecc7..5e2cb1f6 100644 --- a/demos/report/report.py +++ b/demos/report/report.py @@ -1,8 +1,8 @@ -#!/usr/bin/env python -from Exscript import Queue, Logger -from Exscript.util.log import log_to +#!/usr/bin/env python3 +from Exscript import Logger, Queue from Exscript.util.decorator import autologin -from Exscript.util.file import get_hosts_from_file, get_accounts_from_file +from Exscript.util.file import get_accounts_from_file, get_hosts_from_file +from Exscript.util.log import log_to from Exscript.util.report import status, summarize logger = Logger() # Logs everything to memory. @@ -20,11 +20,11 @@ def do_something(job, host, conn): # Run do_something on each of the hosts. The given accounts are used # round-robin. "verbose=0" instructs the queue to not generate any # output on stdout. -queue = Queue(verbose=5, max_threads=5) +queue = Queue(verbose=0, max_threads=5) queue.add_account(accounts) # Adds one or more accounts. queue.run(hosts, do_something) # Asynchronously enqueues all hosts. queue.shutdown() # Waits until all hosts are completed. # Print a short report. -print status(logger) -print summarize(logger) +print(status(logger)) +print(summarize(logger)) diff --git a/demos/simple/simple.py b/demos/simple/simple.py index eac9d514..e1c963ec 100644 --- a/demos/simple/simple.py +++ b/demos/simple/simple.py @@ -1,5 +1,6 @@ -from Exscript.util.interact import read_login +#!/usr/bin/env python3 from Exscript.protocols import SSH2 +from Exscript.util.interact import read_login account = read_login() @@ -8,7 +9,8 @@ conn.login(account) conn.execute('ls -l') -print "Response was:", repr(conn.response) +# Fetch a clean response (no command echo, no NUL/BEL/BS, no ANSI codes). +print("Response was:", repr(conn.get_response())) -conn.send('exit\r') +# Close the session gracefully (driver sends logout command like 'exit') conn.close() diff --git a/demos/template/template.py b/demos/template/template.py index c1d022e9..dfd1c1b7 100644 --- a/demos/template/template.py +++ b/demos/template/template.py @@ -1,6 +1,6 @@ -#!/usr/bin/env python -from Exscript.util.template import eval_file +#!/usr/bin/env python3 from Exscript.util.start import quickstart +from Exscript.util.template import eval_file def do_something(job, host, conn): @@ -8,4 +8,4 @@ def do_something(job, host, conn): conn.execute('ls -1') eval_file(conn, 'template.exscript', foobar='hello-world') -quickstart('ssh://xpc3', do_something) +quickstart('ssh://localhost', do_something) diff --git a/requirements.txt b/requirements.txt index b5a4afae..d656c981 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ -future configparser +lxml +ncclient>=0.7,<1 +paramiko>=2.2,<5 pycryptodomex -paramiko>=1.17,<5 diff --git a/scripts/exscript b/scripts/exscript index cc2fe787..5c97a808 100755 --- a/scripts/exscript +++ b/scripts/exscript @@ -1,8 +1,5 @@ -#!/usr/bin/env python -from __future__ import print_function -from builtins import input -from builtins import str -from builtins import range +#!/usr/bin/env python3 +"""TODO: Add missing doctring.""" # Copyright (C) 2007-2010 Samuel Abels. # # This program is free software; you can redistribute it and/or modify @@ -17,206 +14,257 @@ from builtins import range # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -import sys +import argparse +import getpass +import importlib.util import os import re -import imp -import getpass +import sys +from pathlib import Path + import Exscript.protocols.drivers -from optparse import OptionParser, Option, OptionValueError -from Exscript import Queue, Account, PrivateKey, __version__ +from Exscript import Account, PrivateKey, Queue +from Exscript.logger import getConsoleLogger, setupLogging from Exscript.protocols import protocol_map from Exscript.util import template -from Exscript.util.sigint import SigIntWatcher -from Exscript.util.cast import to_list, to_host +from Exscript.util.cast import to_host +from Exscript.util.decorator import autologin +from Exscript.util.file import ( + get_accounts_from_file, + get_hosts_from_csv, + get_hosts_from_file, + load_lib, +) from Exscript.util.interact import get_login from Exscript.util.log import log_to_file -from Exscript.util.file import get_accounts_from_file, \ - get_hosts_from_file, \ - get_hosts_from_csv, \ - load_lib -from Exscript.util.decorator import autologin +from Exscript.util.sigint import SigIntWatcher + +_logger = getConsoleLogger("exscript-cli") + +bracket_expression = r"\{([^\]]*)\}" +define_expression = rf"^\S+=(?:\S+|{bracket_expression})$" +bracket_expression_re = re.compile(rf"^{bracket_expression}$") +define_expression_re = re.compile(define_expression) -bracket_expression_re = re.compile(r'^\{([^\]]*)\}$') def expand_bracket(value_name, value): + """TODO: Add missing doctring.""" match = bracket_expression_re.match(value) if match is None: return value - prompt = match.group(1) or 'a value for "%s"' % value_name - if prompt.startswith('!'): - value = getpass.getpass('Please enter %s: ' % prompt[1:]) + prompt = match.group(1) or f"a value for '{value_name}' " + if prompt.startswith("!"): + value = getpass.getpass(f"Please enter {prompt.lstrip('!')}: ") else: - value = eval(input('Please enter %s: ' % prompt)) + value = input(f"Please enter {prompt.lstrip('#')}: ") + if prompt.startswith("#"): + try: + value = eval(value) + except (SyntaxError, NameError): + pass return value + def expand_host_variables(host): + """TODO: Add missing doctring.""" # Define host-specific variables. for key, value in list(host.get_all().items()): if isinstance(value, str): value = expand_bracket(key, value) - elif hasattr(value, '__iter__'): + elif hasattr(value, "__iter__"): for n in range(len(value)): if not isinstance(value[n], str): continue value[n] = expand_bracket(key, value[n]) host.set(key, value) + def run(): + """TODO: Add missing doctring.""" # Parse options. - options, args = parser.parse_args(sys.argv) - args.pop(0) + args = parser.parse_args(sys.argv[1:]) def mkhost(name): - return to_host(name, - default_protocol = options.protocol, - default_domain = options.default_domain) + return to_host( + name, + default_protocol=args.protocol, + default_domain=args.default_domain, + ) def mkhosts_from_file(filename): - return get_hosts_from_file(options.hosts, - default_protocol = options.protocol, - default_domain = options.default_domain) + return get_hosts_from_file( + args.hosts, + default_protocol=args.protocol, + default_domain=args.default_domain, + ) def mkhosts_from_csv(filename): - return get_hosts_from_csv(options.csv_hosts, - default_protocol = options.protocol, - default_domain = options.default_domain) + return get_hosts_from_csv( + args.csv_hosts, + default_protocol=args.protocol, + default_domain=args.default_domain, + ) + + # Setup package logging. + setupLogging( + package=bool(args.verbose), + parser=bool(args.parser_verbose), + protocol=bool(args.protocol_verbose), + ) - # Extract the hostnames out of the command line arguments. - filename = None - if not options.execute: + # Read the Exscript. + script_extra_content = "" # init for later use + if args.execute: + script_content = args.execute + else: + if not args.filename: + parser.error("Missing template filename") try: - filename = args.pop(0) - except IndexError: - parser.error('Missing filename') - hosts = [mkhost(h) for h in args] + with open(args.filename, "r") as fp: + script_content = fp.read() + except IOError as e: + parser.error(str(e)) + + # Extract the hostnames out of the command line arguments. + if args.execute and args.filename: + args.host = [args.filename] + args.host + hosts = [mkhost(h) for h in args.host] # If a filename containing hostnames AND VARIABLES was given, read it. - if options.csv_hosts: + if args.csv_hosts: try: - csv_hosts = mkhosts_from_csv(options.csv_hosts) + path = Path(os.path.expandvars(args.csv_hosts)).expanduser() + csv_hosts = mkhosts_from_csv(path) except IOError as e: parser.error(str(e)) if not csv_hosts: - print('Warning: %s is empty.' % options.csv_hosts) + print(f"Warning: {args.csv_hosts} is empty.") hosts += csv_hosts # If a filename containing hostnames was given, read it. - if options.hosts: + if args.hosts: try: - txt_hosts = mkhosts_from_file(options.hosts) + path = Path(os.path.expandvars(args.hosts)).expanduser() + txt_hosts = mkhosts_from_file(path) except IOError as e: parser.error(str(e)) except ValueError as e: parser.error(e) if not txt_hosts: - print('Warning: %s is empty.' % options.hosts) + print(f"Warning: {args.hosts} is empty.") hosts += txt_hosts # If a filename of an Exscript driver was given, import it. - if options.load_driver: - print('Searching drivers in %s...' % options.load_driver) - name = os.path.splitext(os.path.basename(options.load_driver))[0] + if args.load_driver: + path = Path(os.path.expandvars(args.load_driver)).expanduser() + print(f"Searching drivers in {args.load_driver}...") try: - driver_module = imp.load_source(name, options.load_driver) + spec = importlib.util.spec_from_file_location(path.stem, path) + driver_module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(driver_module) except IOError as e: parser.error(str(e)) - except Exception as e: - sys.stderr.write('Error in driver ' + options.load_driver + '\n') + except Exception: + sys.stderr.write(f"Error in driver {args.load_driver}\n") raise for name, obj in list(driver_module.__dict__.items()): if Exscript.protocols.drivers.isdriver(obj): Exscript.protocols.drivers.add_driver(obj) - print('Driver', repr(name), 'added.') + print(f"Driver {repr(name)} added.") # Make sure that all mandatory options are present. if not hosts: - parser.error('No hosts to connect to') + parser.error("No hosts to connect to") # Set host options. for host in hosts: - host.set_option('debug', options.protocol_verbose) - host.set_option('verify_fingerprint', not options.ssh_auto_verify) - - # Read the Exscript. - if options.execute: - script_content = options.execute - else: - with open(filename, 'r') as fp: - script_content = fp.read() + host.set_option("debug", args.protocol_verbose) + if args.ssh_disabled_algs: + ssh_disabled_algs = {} + for algs_map in args.ssh_disabled_algs.split(";"): + key, algs = algs_map.split("=", maxsplit=1) + ssh_disabled_algs[key] = algs.split(",") + host.set_option("disabled_algorithms", ssh_disabled_algs) + if args.ssh_preferred_auth: + host.set_option("preferred_authentications", args.ssh_preferred_auth.split(",")) + host.set_option("verify_fingerprint", not args.ssh_auto_verify) # Prepare the code that is executed after the user script has completed. - #FIXME: Move into the library, then use read_template_from_file instead of read_template(). - if not options.no_auto_logout: - script_content += r''' + if not args.no_auto_logout: + script_extra_content += r""" ## Exscript generated commands. ## - {if connection.guess_os() is "vrp"} - {connection.sendline("quit")} - {else} - {connection.sendline("exit")} - {end}''' + {connection.close()}""" # Load extra functions. functions = {} - if options.lib: + if args.lib: try: - functions = load_lib(options.lib) + functions = load_lib(args.lib) except IOError as e: parser.error(str(e)) # Test whether the template compiles. + template.debug = args.parser_verbose vars = functions.copy() - vars.update(options.define) - if filename: - vars['__filename__'] = filename + vars.update(args.define) + if args.filename: + vars["__filename__"] = args.filename vars.update(hosts[0].get_all()) try: template.test(script_content, **vars) + template.test(script_extra_content, **vars) except Exception as e: - if options.parser_verbose > 0: + if args.parser_verbose > 0: raise parser.error(str(e)) + script_content += script_extra_content + # NOTE: Disable debugging again until Parser logging is more than just dumping! + template.debug = 0 # Create Exscript. - queue = Queue(domain = options.default_domain, - mode = 'multiprocessing', - host_driver = options.use_driver, - verbose = options.verbose, - max_threads = options.connections) + # Switch to 'threading' mode to match API default. + # See also: https://github.com/knipknap/exscript/issues/230 + queue = Queue( + domain=args.default_domain, + mode="threading", + host_driver=args.use_driver, + verbose=args.verbose, + max_threads=args.connections, + ) default_pool = queue.account_manager.default_pool # Read the account pool file. try: - if options.account_pool: - accounts = get_accounts_from_file(options.account_pool) + if args.account_pool: + path = Path(os.path.expandvars(args.account_pool)).expanduser() + accounts = get_accounts_from_file(path) default_pool.add_account(accounts) except IOError as e: parser.error(str(e)) - if options.account_pool and default_pool.n_accounts() == 0: - msg = r'WARNING: No accounts found in account pool file (%s)!' - print(msg % options.account_pool) + if args.account_pool and default_pool.n_accounts() == 0: + print(f"WARNING: No accounts found in account pool file ({args.account_pool})!") # Read username and password. - if options.non_interactive or default_pool.n_accounts() > 0: - user = None - password = None + if args.non_interactive or default_pool.n_accounts() > 0: + user = None + password = None password2 = None else: try: user, password = get_login() except KeyboardInterrupt: sys.exit(1) - if options.authorization: - prompt = 'Please enter your authorization password: ' + if args.authorization: + prompt = "Please enter your authorization password: " password2 = getpass.getpass(prompt) else: password2 = password # Read the SSH key. - if options.ssh_key: - print("Reading key from", options.ssh_key) + if args.ssh_key: + print(f"Reading key from {args.ssh_key}") try: - key = PrivateKey.from_file(options.ssh_key, password) + key = PrivateKey.from_file(args.ssh_key, password) except ValueError as e: parser.error(str(e)) if user is None: @@ -226,135 +274,151 @@ def run(): # Add the account to the pool. if user is not None: - account = Account(user, password, password2, key = key) + account = Account(user, password, password2, key=key) default_pool.add_account(account) # Ask for values of marked variables. for host in hosts: expand_host_variables(host) - for key, val in list(options.define.items()): - options.define[key] = expand_bracket(key, val) + for key, val in list(args.define.items()): + args.define[key] = expand_bracket(key, val) # Choose the template processing type. - tmpl_vars = options.define - if options.no_prompt: + tmpl_vars = args.define + if args.no_prompt: + def function(job, host, conn, **kwargs): kwargs.update(functions) kwargs.update(tmpl_vars) + if args.filename: + kwargs["__filename__"] = args.filename kwargs.update(host.get_all()) - return template.paste(conn, script_content, **kwargs) + return template.paste(conn, script_content, args.no_escaping, **kwargs) + else: - strip = not options.no_strip + def function(job, host, conn, **kwargs): kwargs.update(functions) kwargs.update(tmpl_vars) + if args.filename: + kwargs["__filename__"] = args.filename kwargs.update(host.get_all()) - return template.eval(conn, script_content, strip, **kwargs) + return template.eval( + conn, script_content, not args.no_strip, args.no_escaping, **kwargs + ) # Wrap the template processor such that the login procedure is automated. - if not options.no_authentication: - attempts = options.retry_login + 1 - decorator = autologin(attempts = attempts) - function = decorator(function) + if not args.no_authentication: + attempts = args.retry_login + 1 + decorator = autologin(attempts=attempts) + function = decorator(function) # Wrap the template processor such that logging is enabled. - if options.logdir: + if args.logdir: # Create the log directory. - if not os.path.exists(options.logdir): - print('Creating log directory (%s)...' % options.logdir) + if not os.path.exists(args.logdir): + print(f"Creating log directory ({args.logdir})...") try: - os.makedirs(options.logdir) + os.makedirs(args.logdir) except IOError as e: parser.error(str(e)) # Enable logging. - mode = options.overwrite_logs and 'w' or 'a' - decorator = log_to_file(options.logdir, mode, options.delete_logs) - function = decorator(function) + mode = args.overwrite_logs and "w" or "a" + decorator = log_to_file(args.logdir, mode, args.delete_logs) + function = decorator(function) # Wait until the specified time. We are using os.system because # it allows for specifying absolute times, not just the number # of seconds like Python's time.sleep(). - if options.sleep: - print("Waiting for %s..." % options.sleep, end='', flush=True) - os.system('sleep %s' % options.sleep) + if args.sleep: + print(f"Waiting for {args.sleep}...", end="", flush=True) + os.system(f"sleep {args.sleep}") print("time expired, starting script.") # Run the template. - for host in hosts: - queue.run(host, function, attempts = options.retry + 1) + queue.run(hosts, function, attempts=args.retry + 1) queue.join() failed = queue.failed - queue.destroy() + queue.destroy(force=True) return failed -# Define command line option value types. -def check_assignment(option, opt, value): - if not re.match(r'\S+=\S+', value): - raise OptionValueError('option %s: invalid value: %r' % (opt, value)) - return value -def check_protocol(option, opt, value): - if value not in protocol_map: - raise OptionValueError('option %s: invalid value: %r' % (opt, value)) +# Define command line option value types. +def assignment(value, pattern=define_expression_re): + """TODO: Add missing doctring.""" + if not pattern.match(value): + raise argparse.ArgumentTypeError(f"invalid value: {value}") return value -class AssignmentOption(Option): - # Define a new 'assignment' type for the '--define' command line option. - TYPES = Option.TYPES + ('assignment', 'protocol') - TYPE_CHECKER = Option.TYPE_CHECKER.copy() - TYPE_CHECKER['assignment'] = check_assignment - TYPE_CHECKER['protocol'] = check_protocol - # Define a new store action that parses 'assignment' values. - ACTIONS = Option.ACTIONS + ('store_dict',) - STORE_ACTIONS = Option.STORE_ACTIONS + ('store_dict',) - TYPED_ACTIONS = Option.TYPED_ACTIONS + ('store_dict',) +class DefineAction(argparse.Action): + """TODO: Add missing doctring.""" - def take_action(self, action, dest, opt, value, values, parser): - if action == "store_dict": + def __call__(self, parser, namespace, values, option_string=None): + """TODO: Add missing doctring.""" + if type(values) == str: + values = [values] + d = getattr(namespace, self.dest) + for value in values: left, right = value.split("=") - values.ensure_value(dest, {})[left] = right - return - Option.take_action(self, - action, - dest, - opt, - value, - values, - parser) + if left.startswith('__') and left.endswith('__'): + parser.error('Assignment to internal variable %s' % repr(left)) + d[left] = right + setattr(namespace, self.dest, d) + # Define the command line syntax. -usage = '%prog [options] filename [hostname [hostname ...]]' -usage += '\nCopyright (C) 2007-2010 by Samuel Abels.' -parser = OptionParser(usage = usage, - version = __version__, - option_class = AssignmentOption) - -parser.add_option('--account-pool', - dest = 'account_pool', - metavar = 'FILE', - help = ''' +parser = argparse.ArgumentParser(epilog="\N{COPYRIGHT SIGN} 2007-2023 by Samuel Abels.") + +parser.add_argument( + "filename", + default=None, + nargs="?", + help=""" +Filename of the template to be executed. Ignored with +"-e/--execute" and taken as first positional host. +""".strip(), +) + +parser.add_argument( + "host", + default=list(), + nargs="*", + help=""" +List of hostnames. +""".strip(), +) + +parser.add_argument( + "--account-pool", + default=None, + metavar="FILE", + help=""" Reads the user/password combination from the given file instead of prompting on the command line. The file may also contain more than one user/password combination, in which case the accounts are used round robin. -'''.strip()) - -parser.add_option('--connections', '-c', - dest = 'connections', - type = 'int', - metavar = 'NUM', - default = 1, - help = ''' +""".strip(), +) + +parser.add_argument( + "-c", + "--connections", + type=int, + metavar="NUM", + default=1, + help=""" Maximum number of concurrent connections. NUM is a number between 1 and 20, default is 1. -'''.strip()) - -parser.add_option('--csv-hosts', - dest = 'csv_hosts', - metavar = 'FILE', - help = ''' +""".strip(), +) + +parser.add_argument( + "--csv-hosts", + default=None, + metavar="FILE", + help=""" Loads a list of hostnames and definitions from the given file. The first line of the file must have the column headers in the following syntax: "hostname [variable] [variable] ...", where @@ -363,226 +427,315 @@ the fields are separated by tabs, "hostname" is the keyword column is accessed in the script. The following lines contain the hostname in the first column, and the values of the variables in the following columns. -'''.strip()) - -parser.add_option('--define', '-d', - dest = 'define', - type = 'assignment', - action = 'store_dict', - default = {}, - metavar = 'PAIR', - help = ''' +""".strip(), +) + +parser.add_argument( + "-d", + "--define", + type=assignment, + action=DefineAction, + default=dict(), + metavar="PAIR", + help=""" Defines a variable that is passed to the script. PAIR has the following syntax: STRING=STRING. -'''.strip()) - -parser.add_option('--default-domain', - dest = 'default_domain', - metavar = 'STRING', - default = '', - help = ''' +Use a prompt with the following syntax: VAR={"PROMPT"}. +Use a password prompt with the following syntax: VAR={\\!"PROMPT"}. +Evaluate the value of a prompt with the following syntax: VAR={#"PROMPT"}. +""".strip(), +) + +parser.add_argument( + "--default-domain", + metavar="STRING", + default="", + help=""" The IP domain name that is used if a given hostname has no domain appended. -'''.strip()) - -parser.add_option('--delete-logs', - dest = 'delete_logs', - action = 'store_true', - default = False, - help = 'Delete logs of successful operations when done.') - -parser.add_option('--lib', - dest = 'lib', - metavar = 'FILE', - help = ''' +""".strip(), +) + +parser.add_argument( + "--delete-logs", + action="store_true", + default=False, + help="Delete logs of successful operations when done.", +) + +parser.add_argument( + "--no-escaping", + action="store_true", + default=False, + help=""" +Disable additional evaluation of escaped control characters (NUL/BEL/BS/ESC/FF/VT), +octal (\\ooo), hexadecimal (\\xhh) and unicode (\\uhhhh, \\Uhhhhhhhh) characters +in the script. Evaluation of escaped CR, LF and TAB characters in strings is always +enabled. This is irrelevant for RPC bodies in Netconf as XML does not support +escaping (see XML entity syntax instead). +""".strip(), +) + +parser.add_argument( + "--lib", + default=None, + metavar="FILE", + help=""" A python file containing a __lib__ dictionary that points to extra functions to be made available in the Exscript template. -'''.strip()) - -parser.add_option('--load-driver', - dest = 'load_driver', - metavar = 'FILE', - help = ''' +""".strip(), +) + +parser.add_argument( + "--load-driver", + default=None, + metavar="FILE", + help=""" A python file containing an Exscript driver class. -'''.strip()) - -parser.add_option('--use-driver', - dest = 'use_driver', - default = None, - help = ''' +""".strip(), +) + +parser.add_argument( + "--use-driver", + default=None, + metavar="NAME", + help=""" The name of an exscript driver to use for all connections (overriding os detection). -'''.strip()) - -parser.add_option('--execute', '-e', - dest = 'execute', - metavar = 'EXSCRIPT', - help = 'Interprets the given string as the script.') - -parser.add_option('--hosts', - dest = 'hosts', - metavar = 'FILE', - help = ''' +""".strip(), +) + +parser.add_argument( + "-e", + "--execute", + default=None, + metavar="EXSCRIPT", + help=""" +Interprets the given string as the script. +""".strip(), +) + +parser.add_argument( + "--hosts", + default=None, + metavar="FILE", + help=""" Loads a list of hostnames from the given file (one host per line). -'''.strip()) - -parser.add_option('--non-interactive', '-i', - dest = 'non_interactive', - action = 'store_true', - default = False, - help = ''' +""".strip(), +) + +parser.add_argument( + "-i", + "--non-interactive", + action="store_true", + default=False, + help=""" Do not ask for a username or password, but still try to authenticate. May be used if the login credentials are passed as part of a URI formatted hostname. -'''.strip()) - -parser.add_option('--logdir', '-l', - dest = 'logdir', - metavar = 'DIR', - help = ''' +""".strip(), +) + +parser.add_argument( + "-l", + "--logdir", + default=None, + metavar="DIR", + help=""" Logs any communication into the directory with the given name. Each filename consists of the hostname with ".log" appended. Errors are written to a separate file, where the filename consists of the hostname with ".log.error" appended. -'''.strip()) - -parser.add_option('--no-authentication', '-n', - dest = 'no_authentication', - action = 'store_true', - default = False, - help = ''' +""".strip(), +) + +parser.add_argument( + "-n", + "--no-authentication", + action="store_true", + default=False, + help=""" When given, the automatic authentication procedure is skipped. Implies -i. -'''.strip()) - -parser.add_option('--authorization', - dest = 'authorization', - action = 'store_true', - default = False, - help = ''' +""".strip(), +) + +parser.add_argument( + "--authorization", + action="store_true", + default=False, + help=""" Ask for an authorization password in addition to the authentication password. -'''.strip()) - -parser.add_option('--no-auto-logout', - dest = 'no_auto_logout', - action = 'store_true', - default = False, - help = ''' +""".strip(), +) + +parser.add_argument( + "--no-auto-logout", + action="store_true", + default=False, + help=""" Do not attempt to execute the exit or quit command at the end of a script. -'''.strip()) - -parser.add_option('--no-prompt', - dest = 'no_prompt', - action = 'store_true', - default = False, - help = ''' +""".strip(), +) + +parser.add_argument( + "--no-prompt", + action="store_true", + default=False, + help=""" Do not wait for a prompt anywhere (except during the authentication procedure). Note that this will also cause Exscript to disable commands that require a prompt, such as "extract". -'''.strip()) - -parser.add_option('--no-strip', - dest = 'no_strip', - action = 'store_true', - default = False, - help = 'Do not strip the first line of each response.') - -parser.add_option('--overwrite-logs', - dest = 'overwrite_logs', - action = 'store_true', - default = False, - help = ''' +""".strip(), +) + +parser.add_argument( + "--no-strip", + action="store_true", + default=False, + help=""" +Do not strip the command echo from each response. This is irrelevant for RPC +bodies as Netconf does not echo commands. +""".strip(), +) + +parser.add_argument( + "--overwrite-logs", + action="store_true", + default=False, + help=""" Instructs Exscript to overwrite existing logfiles. The default is to append the output if a log already exists. -'''.strip()) +""".strip(), +) protocols = list(protocol_map.keys()) protocols.sort() -parser.add_option('--protocol', '-p', - dest = 'protocol', - type = 'protocol', - metavar = 'STRING', - default = 'telnet', - help = ''' +parser.add_argument( + "-p", + "--protocol", + metavar="STRING", + default="ssh", + choices=protocols, + help=f""" Specify which protocol to use to connect to the remote host. -Allowed values for STRING include: %s. -The default protocol is telnet. -'''.strip() % ', '.join(protocols)) - -parser.add_option('--retry', - dest = 'retry', - type = 'int', - metavar = 'NUM', - default = 0, - help = ''' +Allowed values for STRING include: {", ".join(protocols)}. +The default protocol is ssh. +""".strip(), +) + +parser.add_argument( + "--retry", + type=int, + metavar="NUM", + default=0, + help=""" Defines the number of retries per host on failure. Default is 0. -'''.strip()) - -parser.add_option('--retry-login', - dest = 'retry_login', - type = 'int', - metavar = 'NUM', - default = 0, - help = ''' +""".strip(), +) + +parser.add_argument( + "--retry-login", + type=int, + metavar="NUM", + default=0, + help=""" Defines the number of retries per host on login failure. Default is 0. -'''.strip()) - -parser.add_option('--sleep', - dest = 'sleep', - metavar = 'TIME', - default = '', - help = ''' +""".strip(), +) + +parser.add_argument( + "--sleep", + metavar="TIME", + default="", + help=""" Waits for the specified time before running the script. TIME is a timespec as specified by the 'sleep' Unix command. -'''.strip()) - -parser.add_option('--ssh-auto-verify', - dest = 'ssh_auto_verify', - action = 'store_true', - default = False, - help = ''' -Automatically confirms the 'Host key changed' SSH error +""".strip(), +) + +parser.add_argument( + "--ssh-auto-verify", + action="store_true", + default=False, + help=""" +Automatically confirms the 'Host key changed' SSH error message with 'yes'. Highly insecure and not recommended. -'''.strip()) - -parser.add_option('--ssh-key', - dest = 'ssh_key', - metavar = 'FILE', - help = ''' +""".strip(), +) + +parser.add_argument( + "--ssh-disabled-algs", + default="", + metavar="STRING", + help=""" +Specify a mapping of disabled SSH algorithms according to Paramiko's +SSHClient/Transport constructor. The syntax on the CLI goes like this: +"ciphers=aes128-cbc,3des-cbc;pubkeys=ssh-rsa,ssh-dss". This will be +parsed into a mapping as expected by the Paramiko constructor. Please +refer to the "paramiko.Transport" documentation for details. +""".strip(), +) + +parser.add_argument( + "--ssh-preferred-auth", + default="", + metavar="STRING", + help=""" +Specify a comma-separated list of preferred SSH authentication +types in the preferred order. This is equivalent to the +"-o PreferredAuthentications" option of the openssh command +line client and the corresponding SSH config option. +The default is "publickey,keyboard-interactive,password" which +is derived from the SSH config default value. +""".strip(), +) + +parser.add_argument( + "--ssh-key", + default=None, + metavar="FILE", + help=""" Specify a key file that is passed to the SSH client. This is equivalent to using the "-i" parameter of the openssh command line client. -'''.strip()) - -parser.add_option('--verbose', '-v', - dest = 'verbose', - type = 'int', - metavar = 'NUM', - default = 1, - help = ''' +""".strip(), +) + +parser.add_argument( + "-v", + "--verbose", + type=int, + metavar="NUM", + default=1, + help=""" Print out debug information about the queue. NUM is a number between 0 (min) and 5 (max). Default is 1. -'''.strip()) - -parser.add_option('--parser-verbose', '-V', - dest = 'parser_verbose', - type = 'int', - metavar = 'NUM', - default = 0, - help = ''' +""".strip(), +) + +parser.add_argument( + "-V", + "--parser-verbose", + type=int, + metavar="NUM", + default=0, + help=""" Print out debug information about the Exscript template parser. NUM is a number between 0 (min) and 5 (max). Default is 0. -'''.strip()) - -parser.add_option('--protocol-verbose', - dest = 'protocol_verbose', - type = 'int', - metavar = 'NUM', - default = 0, - help = ''' +""".strip(), +) + +parser.add_argument( + "--protocol-verbose", + type=int, + metavar="NUM", + default=0, + help=""" Print out debug information about the network activity. -NUM is a number between 0 (min) and 5 (max). Default is 1. -'''.strip()) +NUM is a number between 0 (min) and 5 (max). Default is 0. +""".strip(), +) -if __name__ == '__main__': +if __name__ == "__main__": SigIntWatcher() failed = run() if failed != 0: - sys.exit('error: %d actions failed.' % failed) + sys.exit(f"error: {failed} actions failed.") diff --git a/scripts/otp b/scripts/otp index b9e84631..f1974882 100755 --- a/scripts/otp +++ b/scripts/otp @@ -1,26 +1,16 @@ -#!/usr/bin/env python -from __future__ import print_function -from builtins import input -from builtins import str -from builtins import range -import sys -import os +#!/usr/bin/env python3 +import argparse import getpass -from optparse import OptionParser, Option, OptionValueError -from Exscript import __version__ + from Exscript.util.crypt import otp # Define the command line syntax. -usage = '%prog sequence salt' -parser = OptionParser(usage=usage, version=__version__) - -options, args = parser.parse_args(sys.argv) -args.pop(0) - -try: - seq, salt = args -except ValueError: - parser.error('invalid number of arguments') +parser = argparse.ArgumentParser() +parser.add_argument('sequence', help='the sequence number') +parser.add_argument('salt', help='the salt value') +args = parser.parse_args() +seq = args.sequence +salt = args.salt try: seq = int(seq) diff --git a/setup.py b/setup.py index 57d1edb8..6d23f756 100644 --- a/setup.py +++ b/setup.py @@ -1,20 +1,21 @@ import sys -import os -from setuptools import setup, find_packages + +from setuptools import find_packages, setup + sys.path.insert(0, 'Exscript') from version import __version__ # Import the project description from the README. descr = ''' Exscript is a Python module and a template processor for automating network -connections over protocols such as Telnet or SSH. We attempt to create the -best possible set of tools for working with Telnet and SSH. +connections over protocols such as Telnet or SSH/Netconf. We attempt to create +the best possible set of tools for working with Telnet and SSH/Netconf. '''.strip() # Run the setup. setup(name = 'Exscript', version = __version__, - description = 'Automating Telnet and SSH', + description = 'Automating Telnet and SSH/Netconf', long_description = descr, author = 'Samuel Abels', author_email = 'knipknap@gmail.com', @@ -23,11 +24,24 @@ package_data = {}, packages = find_packages(), scripts = ['scripts/exscript', 'scripts/otp'], - install_requires = ['future', 'configparser', 'paramiko>=1.17,<5', 'pycryptodomex'], - extras_require = {}, + install_requires = [ + 'configparser', + 'lxml', + 'ncclient>=0.7,<1', + 'paramiko>=2.2,<5', + 'pycryptodomex', + ], + extras_require = { + 'otel': [ + 'opentelemetry-api>=1.12', + 'opentelemetry-sdk>=1.12', + 'opentelemetry-instrumentation>=0.42b0', + ], + }, keywords = ' '.join(['exscript', 'telnet', 'ssh', + 'netconf', 'network', 'networking', 'automate', @@ -38,7 +52,11 @@ 'Development Status :: 5 - Production/Stable', 'Intended Audience :: Developers', 'License :: OSI Approved :: MIT License', - 'Programming Language :: Python', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', + 'Programming Language :: Python :: 3.13', + 'Programming Language :: Python :: 3.14', 'Topic :: Security', 'Topic :: Software Development :: Libraries', 'Topic :: Software Development :: Libraries :: Python Modules' diff --git a/tests/Exscript/AccountPoolTest.py b/tests/Exscript/AccountPoolTest.py index f8a40e55..a4cc80bb 100644 --- a/tests/Exscript/AccountPoolTest.py +++ b/tests/Exscript/AccountPoolTest.py @@ -1,8 +1,8 @@ -from builtins import range +import os.path +import re import sys import unittest -import re -import os.path + sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..')) from Exscript import Account diff --git a/tests/Exscript/HostTest.py b/tests/Exscript/HostTest.py index 6cbbaea4..626ac6e5 100644 --- a/tests/Exscript/HostTest.py +++ b/tests/Exscript/HostTest.py @@ -1,14 +1,15 @@ -from builtins import str +import os.path +import re import sys import unittest -import re -import os.path + sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..')) -from Exscript import Host, Account -from Exscript.util.url import Url from util.urlTest import urls +from Exscript import Account, Host +from Exscript.util.url import Url + class HostTest(unittest.TestCase): CORRELATE = Host @@ -19,7 +20,7 @@ def setUp(self): def testConstructor(self): host = Host('localhost') - self.assertEqual(host.get_protocol(), 'telnet') + self.assertEqual(host.get_protocol(), 'ssh') host = Host('localhost', default_protocol='foo') self.assertEqual(host.get_protocol(), 'foo') @@ -50,8 +51,8 @@ def testGetDict(self): host.set_address('foo2') self.assertEqual(host.get_dict(), {'hostname': 'foo', 'address': 'foo2', - 'protocol': 'telnet', - 'port': 23}) + 'protocol': 'ssh', + 'port': 22}) def testSetAddress(self): self.host.set_protocol('dummy') @@ -83,7 +84,7 @@ def testGetName(self): pass # Tested in testSetName(). def testSetProtocol(self): - self.assertEqual(self.host.get_protocol(), 'telnet') + self.assertEqual(self.host.get_protocol(), 'ssh') self.host.set_protocol('dummy') self.assertEqual(self.host.get_protocol(), 'dummy') @@ -107,9 +108,9 @@ def testGetOptions(self): pass # Tested in testSetOption(). def testSetTcpPort(self): - self.assertEqual(self.host.get_tcp_port(), 23) - self.host.set_protocol('ssh') - self.assertEqual(self.host.get_tcp_port(), 23) + self.assertEqual(self.host.get_tcp_port(), 22) + self.host.set_protocol('telnet') + self.assertEqual(self.host.get_tcp_port(), 22) self.host.set_tcp_port(123) self.assertEqual(self.host.get_tcp_port(), 123) diff --git a/tests/Exscript/LogTest.py b/tests/Exscript/LogTest.py index 96c34ab6..bda761cb 100644 --- a/tests/Exscript/LogTest.py +++ b/tests/Exscript/LogTest.py @@ -1,13 +1,14 @@ -from builtins import str +import os.path +import re import sys import unittest -import re -import os.path + sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..')) +from util.reportTest import FakeError + from Exscript import Host from Exscript.logger import Log -from util.reportTest import FakeError class LogTest(unittest.TestCase): diff --git a/tests/Exscript/LogfileTest.py b/tests/Exscript/LogfileTest.py index 20da4e8b..33609f54 100644 --- a/tests/Exscript/LogfileTest.py +++ b/tests/Exscript/LogfileTest.py @@ -1,13 +1,15 @@ -from builtins import str +import os.path +import re import sys import unittest -import re -import os.path + sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..')) -from tempfile import mkdtemp from shutil import rmtree +from tempfile import mkdtemp + from LogTest import LogTest + from Exscript.logger import Logfile diff --git a/tests/Exscript/LoggerTest.py b/tests/Exscript/LoggerTest.py index f75e6ea5..cff696cc 100644 --- a/tests/Exscript/LoggerTest.py +++ b/tests/Exscript/LoggerTest.py @@ -1,19 +1,20 @@ -from builtins import next -from builtins import str +import os +import re import sys import unittest -import re -import os + sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..')) import gc from itertools import islice -from tempfile import mkdtemp from shutil import rmtree -from Exscript.logger import Log, Logger +from tempfile import mkdtemp + from LogTest import FakeError from util.reportTest import FakeJob +from Exscript.logger import Log, Logger + def count(iterable): return sum(1 for _ in iterable) diff --git a/tests/Exscript/PrivateKeyTest.py b/tests/Exscript/PrivateKeyTest.py index 76b34b85..892c8f89 100644 --- a/tests/Exscript/PrivateKeyTest.py +++ b/tests/Exscript/PrivateKeyTest.py @@ -19,19 +19,22 @@ def testConstructor(self): self.key = PrivateKey() self.assertRaises(TypeError, PrivateKey, 'foo') PrivateKey('rsa') - PrivateKey('dss') + PrivateKey('ecdsa') + PrivateKey('ed25519') def testFromFile(self): - self.key = PrivateKey.from_file(self.filename, self.password, 'dss') - self.assertEqual(self.key.get_type(), 'dss') + self.key = PrivateKey.from_file(self.filename, self.password, 'ecdsa') + self.assertEqual(self.key.get_type(), 'ecdsa') self.assertEqual(self.filename, self.key.get_filename()) self.assertEqual(self.password, self.key.get_password()) def testGetType(self): self.testConstructor() self.assertEqual(self.key.get_type(), 'rsa') - self.key = PrivateKey('dss') - self.assertEqual(self.key.get_type(), 'dss') + self.key = PrivateKey('ecdsa') + self.assertEqual(self.key.get_type(), 'ecdsa') + self.key = PrivateKey('ed25519') + self.assertEqual(self.key.get_type(), 'ed25519') def testGetFilename(self): self.testConstructor() diff --git a/tests/Exscript/QueueTest.py b/tests/Exscript/QueueTest.py index fddcf480..0842bcac 100644 --- a/tests/Exscript/QueueTest.py +++ b/tests/Exscript/QueueTest.py @@ -1,26 +1,30 @@ -from builtins import object +import logging +import os.path +import re import sys import unittest -import re -import os.path import warnings + sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..')) warnings.simplefilter('ignore', DeprecationWarning) +import ctypes import shutil import time -import ctypes from functools import partial -from tempfile import mkdtemp from multiprocessing import Value from multiprocessing.managers import BaseManager -from Exscript import Queue, Account, AccountPool, FileLogger -from Exscript.protocols import Protocol, Dummy +from tempfile import mkdtemp + +from Exscript import Account, AccountPool, FileLogger, Queue from Exscript.interpreter.exception import FailException +from Exscript.protocols import Dummy, Protocol from Exscript.util.decorator import bind from Exscript.util.log import log_to +logger = logging.getLogger('TestSuite.QueueTest') +logger.addHandler(logging.NullHandler()) def count_calls(job, data, **kwargs): assert hasattr(job, 'start') @@ -72,7 +76,7 @@ def raise_if_not_myprotocol(job, host, conn): raise Exception('not a MyProtocol instance') -class Log(object): +class Log: data = '' def write(self, data): @@ -137,6 +141,10 @@ def assertVerbosity(self, channel, expected): msg = repr(expected) + ' not in ' + repr(data) self.assertTrue(expected in data, msg) + # IMPORTANT: If you see fails of this test, it might be related to the + # Logmanager as it uses Process, not Thread. Mixing both is not recommended + # and false-positive fails have been seen with stuck old processes. So check + # `ps -ef -u` on your system for any old test processes and kill them. def testConstructor(self): self.assertEqual(1, self.queue.get_max_threads()) @@ -158,8 +166,10 @@ def testConstructor(self): (5, 2, ('[', ''), ('[', 'tb'), ('[', 'tb')), ) for level, max_threads, with_simple, with_error, with_fatal in levels: - # print("S:", level, max_threads, with_simple, with_error,) - # with_fatal + msg = 'Testing verbose=%s with max_threads=%s (%s)' + + # with_simple + logger.debug(msg, level, max_threads, 'simple') stdout, stderr = with_simple self.createQueue(verbose=level, max_threads=max_threads) self.queue.run('dummy://mytest', say_hello) @@ -167,8 +177,8 @@ def testConstructor(self): self.assertVerbosity(self.out, stdout) self.assertVerbosity(self.err, stderr) - # print("E:", level, max_threads, with_simple, with_error,) - # with_fatal + # with_error + logger.debug(msg, level, max_threads, 'error') stdout, stderr = with_error self.createQueue(verbose=level, max_threads=max_threads) self.queue.run('dummy://mytest', error) @@ -176,8 +186,8 @@ def testConstructor(self): self.assertVerbosity(self.out, stdout) self.assertVerbosity(self.err, stderr) - # print("F:", level, max_threads, with_simple, with_error,) # with_fatal + logger.debug(msg, level, max_threads, 'fatal') stdout, stderr = with_fatal self.createQueue(verbose=level, max_threads=max_threads) self.queue.run('dummy://mytest', fatal_error) @@ -357,7 +367,7 @@ def write(data, value, *args): self.queue.destroy() # The 'dummy' job should run first, so the value must - # be overwritten by the other process. + # be overwritten by the other job. self.assertEqual(data.value, 1) def testPriorityRunOrRaise(self): @@ -404,15 +414,14 @@ def testEnqueue(self): self.queue.destroy() self.assertEqual(data.value, 10) - # FIXME: Not a method test; this should probably be elsewhere. - def testLogging(self): + def testQueueLogging(self): task = self.startTask() while not task.is_completed(): - time.sleep(.1) + time.sleep(.2) # The following function is not synchronous with the above, so add # a timeout to avoid races. - time.sleep(.1) + time.sleep(.2) self.assertTrue(self.queue.is_completed()) logfiles = os.listdir(self.tempdir) @@ -424,14 +433,17 @@ def testLogging(self): content = fp.read() -class QueueTestMultiProcessing(QueueTest): - mode = 'multiprocessing' +# Mode 'multiprocessing' is not recommended and officially not supported with +# threads in parallel, see https://github.com/knipknap/exscript/issues/230 +# +# class QueueTestMultiProcessing(QueueTest): +# mode = 'multiprocessing' def suite(): loader = unittest.TestLoader() suite1 = loader.loadTestsFromTestCase(QueueTest) - suite2 = loader.loadTestsFromTestCase(QueueTestMultiProcessing) - return unittest.TestSuite((suite1, suite2)) + # suite2 = loader.loadTestsFromTestCase(QueueTestMultiProcessing) + return unittest.TestSuite((suite1,)) if __name__ == '__main__': unittest.TextTestRunner(verbosity=2).run(suite()) diff --git a/tests/Exscript/TemplateTest.py b/tests/Exscript/TemplateTest.py index d79c0668..cce7f019 100644 --- a/tests/Exscript/TemplateTest.py +++ b/tests/Exscript/TemplateTest.py @@ -1,24 +1,23 @@ -from __future__ import print_function -from builtins import object +import os +import re import sys import unittest -import re -import os + dirname = os.path.dirname(__file__) sys.path.insert(0, os.path.join(dirname, '..', '..')) -from Exscript import Queue, Account, Logger, protocols +from Exscript import Account, Host, Logger, Queue, protocols +from Exscript.emulators import IOSEmulator +from Exscript.protocols import Dummy from Exscript.util import template from Exscript.util.decorator import bind from Exscript.util.log import log_to from Exscript.util.report import format -from Exscript.protocols import Dummy -from Exscript.emulators import IOSEmulator test_dir = os.path.join(os.path.dirname(dirname), 'templates') -class Log(object): +class Log: data = '' def collect(self, data): @@ -27,13 +26,12 @@ def collect(self, data): def dummy_cb(job, host, conn, template_test): - # Warning: Assertions raised in this function happen in a subprocess! + # Warning: Assertions raised in this function happen in a thread of the queue! # Create a log object. log = Log() conn.data_received_event.connect(log.collect) # Connect and load the test template. - conn.connect(host.get_address(), host.get_tcp_port()) test_name = host.get_address() if host.get_protocol() == 'ios': dirname = os.path.join(test_dir, test_name) @@ -50,12 +48,8 @@ def dummy_cb(job, host, conn, template_test): print(log.data) raise log.data = log.data.replace('\r\n', '\n') - if sys.version_info[0] == 2: - with open(expected) as fp: - expected_data = fp.read() - else: - with open(expected, newline='') as fp: - expected_data = fp.read() + with open(expected, newline='') as fp: + expected_data = fp.read() #open(expected, 'w').write(log.data) #open('output', 'w').write(log.data) #open('exp', 'w').write(expected_data) @@ -67,6 +61,7 @@ class IOSDummy(Dummy): def __init__(self, *args, **kwargs): device = IOSEmulator('dummy', strict=False) Dummy.__init__(self, device=device, **kwargs) + protocols.protocol_map['ios'] = IOSDummy @@ -84,12 +79,14 @@ def tearDown(self): def testTemplates(self): callback = bind(log_to(self.logger)(dummy_cb), self) - for test in os.listdir(test_dir): + for test in sorted(os.listdir(test_dir)): pseudo = os.path.join(test_dir, test, 'pseudodev.py') if os.path.exists(pseudo): - self.queue.run('pseudo://' + pseudo, callback) + host = Host('pseudo://' + pseudo) else: - self.queue.run('ios://' + test, callback) + host = Host('ios://' + test) + host.set_option('debug', 5) # protocol debugging + self.queue.run(host, callback) self.queue.shutdown() # Unfortunately, unittest.TestCase does not fail if self.assert() diff --git a/tests/Exscript/account_pool.cfg b/tests/Exscript/account_pool.cfg index b49c668b..569c9bb5 100644 --- a/tests/Exscript/account_pool.cfg +++ b/tests/Exscript/account_pool.cfg @@ -1,4 +1,4 @@ [account-pool] -abc=a123 -def=b456 -ghi=c789 +abc=YTEyM8O8 +def=YjQ1NsO8 +ghi=Yzc4OcO8 diff --git a/tests/Exscript/emulators/IOSEmulatorTest.py b/tests/Exscript/emulators/IOSEmulatorTest.py index 904e874a..f5fe5857 100644 --- a/tests/Exscript/emulators/IOSEmulatorTest.py +++ b/tests/Exscript/emulators/IOSEmulatorTest.py @@ -1,14 +1,15 @@ -from __future__ import absolute_import +import os.path +import re import sys import unittest -import re -import os.path + sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..', '..')) -from .VirtualDeviceTest import VirtualDeviceTest from Exscript.emulators import IOSEmulator from Exscript.emulators.iosemu import iosbanner +from .VirtualDeviceTest import VirtualDeviceTest + class IOSEmulatorTest(VirtualDeviceTest): CORRELATE = IOSEmulator @@ -28,7 +29,7 @@ def testAddCommand(self): response = cs.do('show version') self.assertTrue(response.startswith( - 'Cisco Internetwork Operating'), response) + '\nCisco Internetwork Operating'), response) def suite(): diff --git a/tests/Exscript/emulators/VirtualDeviceTest.py b/tests/Exscript/emulators/VirtualDeviceTest.py index d5a5dcb6..5152a2bc 100644 --- a/tests/Exscript/emulators/VirtualDeviceTest.py +++ b/tests/Exscript/emulators/VirtualDeviceTest.py @@ -44,7 +44,7 @@ def sayhello(cmd): self.assertEqual(cs.do('hi2'), 'hello') def testAddCommandsFromFile(self): - pass # FIXME + pass # TODO: Implement missing test. def testInit(self): cs = self.cls('myhost', diff --git a/tests/Exscript/emulators/run_suite.py b/tests/Exscript/emulators/run_suite.py index d2e47d3b..0520fafd 100755 --- a/tests/Exscript/emulators/run_suite.py +++ b/tests/Exscript/emulators/run_suite.py @@ -1,13 +1,17 @@ -#!/usr/bin/env python -from __future__ import print_function +#!/usr/bin/env python3 +import fnmatch +import functools +import glob +import logging import os +import re import sys +import time import unittest -import glob -import fnmatch -import re -from inspect import isfunction, ismodule, isclass +from inspect import isclass, isfunction, ismodule +logger = logging.getLogger('TestSuite') +logger.addHandler(logging.NullHandler()) def uppercase(match): return match.group(1).upper() @@ -15,6 +19,75 @@ def uppercase(match): correlated = dict() +def setup_logging(filename=None): + logformat = "%(levelname)s: %(name)s (%(threadName)s) %(message)s" + loglevel = logging.DEBUG + logger_names = [ + "Exscript", + "Exscript.interpreter.parser", + "Exscript.parselib.lexer", + "Exscript.protocols", + "Exscript.servers", + # Netconf Client + "ncclient.capabilities", + "ncclient.devices", + "ncclient.manager", + "ncclient.operations", + "ncclient.transport.session", + "ncclient.transport.ssh", + # Paramiko + "paramiko.transport", + # Tests may also log. + "TestSuite", + ] + formatter = logging.Formatter(logformat) + formatter.converter = time.gmtime + if filename is None: + handler = logging.StreamHandler(sys.stderr) + else: + handler = logging.FileHandler(filename, mode="w", encoding="utf8") + handler.setFormatter(formatter) + handler.setLevel(loglevel) + for name in logger_names: + logger = logging.getLogger(name) + logger.setLevel(logging.DEBUG) + logger.addHandler(handler) + logger.propagate = False + + +def log_suites(suites): + for item in suites: + if isinstance(item, unittest.TestSuite): + log_suites(item) # suites are iterable + elif isinstance(item, unittest.TestCase): + testname = item.id() + method_names = ( + 'setUpClass', + 'setUp', + testname.split('.')[-1], + 'tearDown', + 'tearDownClass', + ) + for name in method_names: + method = getattr(item, name, None) + if not method: + continue + msg = 'Running %s (%s)' % (name, testname) + setattr(item, name, log_call(msg)(method)) + else: + raise TypeError("Unexpected type: %s" % type(item)) + + +def log_call(msg): + def decorator(func): + @functools.wraps(func) + def wrapper(*args, **kwargs): + logger.debug(msg) + return func(*args, **kwargs) + return wrapper + return decorator + + def correlate_class(theclass): """ Checks the given testcase for missing test methods. @@ -74,6 +147,7 @@ def find(dirname, pattern): def load_suite(files): modules = [os.path.splitext(f)[0] for f in files] + modules.sort() all_suites = [] for name in modules: name = name.lstrip('.').lstrip('/').replace('/', '.') @@ -83,6 +157,7 @@ def load_suite(files): if correlated: sys.stderr.write('Error: Untested methods found.\n') sys.exit(1) + log_suites(all_suites) return unittest.TestSuite(all_suites) @@ -107,4 +182,7 @@ def recursive_suite(): sys.exit(1) # Run. + logfile = os.path.splitext(__file__)[0] + '.log' + setup_logging(logfile) unittest.TextTestRunner(verbosity=verbosity).run(recursive_suite()) + print('Logfile with Exscript and TestSuite activity: %s' % logfile) diff --git a/tests/Exscript/memtest.py b/tests/Exscript/memtest.py index 651a81bb..dabdd673 100644 --- a/tests/Exscript/memtest.py +++ b/tests/Exscript/memtest.py @@ -1,12 +1,12 @@ -from __future__ import print_function # This script is not meant to provide a fully automated test, it's # merely a hack/starting point for investigating memory consumption # manually. The behavior also depends heavily on the version of meliae. import gc import re + +from Exscript import Account, Host, Queue from Exscript.protocols import connect from Exscript.util.decorator import bind -from Exscript import Queue, Account, Host objnames = ('count_calls',) follow_modules = False @@ -31,9 +31,11 @@ def foobar(): # Test memory consumption. from meliae import scanner + gc.collect() scanner.dump_all_objects("test.dump") from meliae import loader + om = loader.load('test.dump') om.remove_expensive_references() om.collapse_instance_dicts() diff --git a/tests/Exscript/memtest_host.py b/tests/Exscript/memtest_host.py index 1f14b7ef..a4a8518a 100644 --- a/tests/Exscript/memtest_host.py +++ b/tests/Exscript/memtest_host.py @@ -1,10 +1,8 @@ -from __future__ import print_function -from builtins import str -from builtins import range # This script is not meant to provide a fully automated test, it's # merely a hack/starting point for investigating memory consumption # manually. The behavior also depends heavily on the version of meliae. -from meliae import scanner, loader +from meliae import loader, scanner + from Exscript import Account, Host hostlist = [Host(str(i)) for i in range(1, 10000)] diff --git a/tests/Exscript/protocols/DummyTest.py b/tests/Exscript/protocols/DummyTest.py index 823cead3..1211ae30 100644 --- a/tests/Exscript/protocols/DummyTest.py +++ b/tests/Exscript/protocols/DummyTest.py @@ -1,21 +1,21 @@ -from __future__ import absolute_import -from builtins import str +import os.path +import re import sys import unittest -import re -import os.path + sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..', '..')) -from .ProtocolTest import ProtocolTest from Exscript.emulators import VirtualDevice from Exscript.protocols import Dummy +from .ProtocolTest import ProtocolTest + class DummyTest(ProtocolTest): CORRELATE = Dummy def createProtocol(self): - self.protocol = Dummy(device=self.device, timeout=1) + self.protocol = Dummy(device=self.device, timeout=1, debug=5) def testConstructor(self): self.assertIsInstance(self.protocol, Dummy) @@ -26,7 +26,7 @@ def testIsDummy(self): def _create_dummy_and_eat_banner(self, device, port=None): protocol = Dummy(device=device) protocol.connect(device.hostname, port) - self.assertEqual(str(protocol.buffer), '') + self.assertEqual(str(protocol._buffer), '') self.assertEqual(protocol.response, None) protocol.expect(re.compile(re.escape(self.banner))) self.assertEqual(protocol.response, '') @@ -36,17 +36,17 @@ def testDummy(self): # Test simple instance with banner. protocol = Dummy(device=self.device) protocol.connect('testhost') - self.assertEqual(str(protocol.buffer), '') + self.assertEqual(str(protocol._buffer), '') self.assertEqual(protocol.response, None) protocol.close() # Test login. protocol = Dummy(device=self.device) protocol.connect('testhost') - self.assertEqual(str(protocol.buffer), '') + self.assertEqual(str(protocol._buffer), '') self.assertEqual(protocol.response, None) protocol.login(self.account, flush=False) - self.assertEqual(protocol.buffer.tail(len(self.prompt)), self.prompt) + self.assertEqual(protocol._buffer.tail(len(self.prompt)), self.prompt) protocol.close() # Test login with user prompt. @@ -54,9 +54,9 @@ def testDummy(self): echo=True, login_type=VirtualDevice.LOGIN_TYPE_USERONLY) protocol = self._create_dummy_and_eat_banner(device) - self.assertEqual(str(protocol.buffer), 'User: ') + self.assertEqual(str(protocol._buffer), 'User: ') protocol.login(self.account, flush=False) - self.assertEqual(protocol.buffer.tail(len(self.prompt)), self.prompt) + self.assertEqual(protocol._buffer.tail(len(self.prompt)), self.prompt) protocol.close() # Test login with password prompt. @@ -64,9 +64,9 @@ def testDummy(self): echo=True, login_type=VirtualDevice.LOGIN_TYPE_PASSWORDONLY) protocol = self._create_dummy_and_eat_banner(device) - self.assertEqual(str(protocol.buffer), 'Password: ') + self.assertEqual(str(protocol._buffer), 'Password: ') protocol.login(self.account, flush=False) - self.assertEqual(protocol.buffer.tail(len(self.prompt)), self.prompt) + self.assertEqual(protocol._buffer.tail(len(self.prompt)), self.prompt) protocol.close() # Test login without user/password prompt. @@ -74,7 +74,7 @@ def testDummy(self): echo=True, login_type=VirtualDevice.LOGIN_TYPE_NONE) protocol = self._create_dummy_and_eat_banner(device) - self.assertEqual(str(protocol.buffer), self.prompt) + self.assertEqual(str(protocol._buffer), self.prompt) protocol.close() # Test login with user prompt and wait parameter. @@ -82,10 +82,10 @@ def testDummy(self): echo=True, login_type=VirtualDevice.LOGIN_TYPE_USERONLY) protocol = self._create_dummy_and_eat_banner(device) - self.assertEqual(str(protocol.buffer), 'User: ') + self.assertEqual(str(protocol._buffer), 'User: ') protocol.login(self.account) - self.assertEqual(str(protocol.buffer), '') - self.assertEqual(protocol.response, self.user + '\r') + self.assertEqual(str(protocol._buffer), '') + self.assertEqual(protocol.response, self.user) protocol.close() # Test login with password prompt and wait parameter. @@ -93,18 +93,18 @@ def testDummy(self): echo=True, login_type=VirtualDevice.LOGIN_TYPE_PASSWORDONLY) protocol = self._create_dummy_and_eat_banner(device) - self.assertEqual(str(protocol.buffer), 'Password: ') + self.assertEqual(str(protocol._buffer), 'Password: ') protocol.login(self.account) - self.assertEqual(str(protocol.buffer), '') - self.assertEqual(protocol.response, self.password + '\r') + self.assertEqual(str(protocol._buffer), '') + self.assertEqual(protocol.response, self.password) protocol.close() # Test login with port number. protocol = self._create_dummy_and_eat_banner(device, 1234) - self.assertEqual(str(protocol.buffer), 'Password: ') + self.assertEqual(str(protocol._buffer), 'Password: ') protocol.login(self.account) - self.assertEqual(str(protocol.buffer), '') - self.assertEqual(protocol.response, self.password + '\r') + self.assertEqual(str(protocol._buffer), '') + self.assertEqual(protocol.response, self.password) protocol.close() # Test a custom response. @@ -119,14 +119,14 @@ def testDummy(self): protocol.connect('testhost') protocol.expect(re.compile(re.escape(self.banner))) self.assertEqual(protocol.response, '') - self.assertEqual(str(protocol.buffer), self.prompt) + self.assertEqual(str(protocol._buffer), self.prompt) protocol.expect_prompt() - self.assertEqual(str(protocol.buffer), '') + self.assertEqual(str(protocol._buffer), '') self.assertEqual(protocol.response, self.hostname) protocol.execute('testcommand') expected = 'testcommand\rhello world\r\n' + self.hostname self.assertEqual(protocol.response, expected) - self.assertEqual(str(protocol.buffer), '') + self.assertEqual(str(protocol._buffer), '') protocol.close() diff --git a/tests/Exscript/protocols/OsGuesserTest.py b/tests/Exscript/protocols/OsGuesserTest.py index 69c01256..0610672b 100644 --- a/tests/Exscript/protocols/OsGuesserTest.py +++ b/tests/Exscript/protocols/OsGuesserTest.py @@ -1,11 +1,12 @@ +import os +import re import sys import unittest -import re -import os + sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..', '..')) -from Exscript.protocols.osguesser import OsGuesser from Exscript.protocols import drivers +from Exscript.protocols.osguesser import OsGuesser class OsGuesserTest(unittest.TestCase): @@ -61,7 +62,7 @@ def testSetFromMatch(self): def testGet(self): pass # See testSet(). - def testDataReceived(self): + def testCheckReceivedData(self): dirname = os.path.dirname(__file__) banner_dir = os.path.join(dirname, 'banners') for file in os.listdir(banner_dir): @@ -75,18 +76,27 @@ def testDataReceived(self): banner = fp.read().rstrip('\n') osg = OsGuesser() for char in banner: - osg.data_received(char, False) + osg.check_received_data(char, False) self.assertEqual(osg.get('os'), osname) - def testProtocolInfo(self): + def testCheckProtocolInfo(self): osg = OsGuesser() - osg.protocol_info('') + osg.check_protocol_info('') self.assertEqual(osg.get('os'), "unknown") - osg.protocol_info(None) + osg.check_protocol_info(None) self.assertEqual(osg.get('os'), "unknown") - osg.protocol_info("SSH-2.0-Cisco-1.25") + osg.check_protocol_info("SSH-2.0-Cisco-1.25") self.assertEqual(osg.get('os'), "ios") - pass + + def testCheckNetconfCapabilities(self): + osg = OsGuesser() + osg.check_netconf_capabilities([ + 'urn:ietf:params:netconf:base:1.0', + 'urn:ietf:params:netconf:capability:candidate:1.0', + 'urn:ietf:params:netconf:capability:confirmed-commit:1.0', + 'http://xml.juniper.net/netconf/junos/1.0' + ]) + self.assertEqual(osg.get('os'), "junos") def suite(): diff --git a/tests/Exscript/protocols/ProtocolTest.py b/tests/Exscript/protocols/ProtocolTest.py index c20445d1..fe7ed323 100644 --- a/tests/Exscript/protocols/ProtocolTest.py +++ b/tests/Exscript/protocols/ProtocolTest.py @@ -1,20 +1,24 @@ -from future import standard_library -standard_library.install_aliases() -from builtins import str +import os +import re import sys +import tempfile import unittest -import re -import os.path +import unittest.mock + sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..', '..')) import time from functools import partial -from configparser import RawConfigParser + from Exscript import Account, PrivateKey from Exscript.emulators import VirtualDevice -from Exscript.protocols.exception import TimeoutException, \ - InvalidCommandException, ExpectCancelledException from Exscript.protocols import drivers +from Exscript.protocols.exception import ( + ExpectCancelledException, + InvalidCommandException, + ProtocolException, + TimeoutException, +) from Exscript.protocols.protocol import Protocol @@ -39,11 +43,13 @@ def setUp(self): if self.daemon is not None: self.daemon.start() time.sleep(.2) + # LIFO cleanup order! + self.addCleanup(self.daemon.join) + self.addCleanup(self.daemon.exit) self.createProtocol() def tearDown(self): - if self.protocol.__class__ != Protocol: - self.protocol.close(True) + self.protocol.close(force=True) if self.daemon is not None: self.daemon.exit() self.daemon.join() @@ -63,7 +69,7 @@ def createDaemon(self): pass def createProtocol(self): - self.protocol = Protocol(timeout=1) + self.protocol = Protocol(timeout=1, debug=5) def disableDriver(self): self.assertIn(drivers.driver_map, 'ios') @@ -143,6 +149,14 @@ def testPrompts(self): def testConstructor(self): self.assertIsInstance(self.protocol, Protocol) + def testMandatorySubclassAttributes(self): + self.assertTrue(self.protocol._name is not None) + self.assertTrue(self.protocol._name.isalpha()) + self.assertTrue(self.protocol._name.islower()) + if self.protocol.__class__ != Protocol: + self.assertTrue(self.protocol._default_port is not None) + self.assertTrue(0 <= self.protocol._default_port <= 65535) + def testCopy(self): self.assertEqual(self.protocol, self.protocol.__copy__()) @@ -196,12 +210,33 @@ def _test_prompt_setter(self, getter, setter): self.assertTrue(hasattr(initial_regex[0], 'groups')) my_re = re.compile(r'% username') + + # Extends an implicit copy of the initial pattern list. + try: + setter(my_re, extend=True) # not all setters have "extend" parameter + except TypeError: + pass + else: + regex = getter() + self.assertIsInstance(regex, list) + self.assertEqual(regex, initial_regex + [my_re]) + setter(my_re) regex = getter() self.assertIsInstance(regex, list) self.assertTrue(hasattr(regex[0], 'groups')) self.assertEqual(regex[0], my_re) + # Extends the manually set pattern list. + try: + setter(my_re, extend=True) # not all setters have "extend" parameter + except TypeError: + pass + else: + regex = getter() + self.assertIsInstance(regex, list) + self.assertEqual(regex, [my_re, my_re]) + setter() regex = getter() self.assertEqual(regex, initial_regex) @@ -392,7 +427,6 @@ def testAppAuthorize2(self): return self.doProtocolAuthenticate(flush=True) self.doAppAuthenticate(flush=True) - response = self.protocol.response # At this point app_authorize should fail because the buffer is # empty due to flush=True above. In other words, app_authorize @@ -429,6 +463,90 @@ def testAutoAppAuthorize(self): def testIsAppAuthorized(self): pass # see testAppAuthorize() + def testGetHost(self): + self.assertTrue(self.protocol.get_host() is None) + if self.protocol.__class__ == Protocol: + return + self.doConnect() + self.assertEqual(self.protocol.get_host(), self.hostname) + + def testGetProtocol(self): + expected = type(self.protocol).__name__.lower().strip('0123456789') + self.assertTrue(self.protocol.get_protocol() == expected) + + def testGetResponse(self): + # No response yet — should return empty string. + self.protocol.response = None + self.assertEqual('', self.protocol.get_response()) + self.protocol.response = '' + self.assertEqual('', self.protocol.get_response()) + + # strip_command: first line (command echo) is removed. + self.protocol.response = 'show version\nCisco IOS 15.1\n' + self.assertEqual('Cisco IOS 15.1\n', self.protocol.get_response()) + # strip_command: command echo only, no output — returns empty string. + self.protocol.response = 'show version' + self.assertEqual('', self.protocol.get_response()) + # strip_command=False: first line is kept. + self.protocol.response = 'show version\nCisco IOS 15.1\n' + self.assertEqual( + 'show version\nCisco IOS 15.1\n', + self.protocol.get_response(strip_command=False), + ) + + # strip_chars: CR, NUL, BEL, BS are removed. + self.protocol.response = 'cmd\r\nline one\x00\x07\x08\r\nline two\r\n' + self.assertEqual('line one\nline two\n', self.protocol.get_response()) + # strip_chars='': disables all char stripping. + self.assertEqual( + 'line one\x00\x07\x08\nline two\n', + self.protocol.get_response(strip_chars=''), + ) + + # strip_ansi: CSI color/cursor sequences are removed. + self.protocol.response = 'cmd\n\x1b[1;32mgreen text\x1b[0m\n' + self.assertEqual('green text\n', self.protocol.get_response()) + # strip_ansi: full CSI with intermediate bytes is handled. + self.protocol.response = 'cmd\n\x1b[ Msome text\n' + self.assertEqual('some text\n', self.protocol.get_response()) + # strip_ansi: lone ESC is removed. + self.protocol.response = 'cmd\nstray\x1btext\n' + self.assertEqual('straytext\n', self.protocol.get_response()) + # strip_ansi=False: ANSI sequences are preserved. + self.protocol.response = 'cmd\n\x1b[32mtext\x1b[0m\n' + self.assertEqual( + '\x1b[32mtext\x1b[0m\n', + self.protocol.get_response(strip_ansi=False), + ) + + # Combined: real-world router output with CR, ANSI, and command echo. + self.protocol.response = ( + 'show ip int brief\r\n' + '\x1b[1mInterface\x1b[0m\tIP-Address\r\n' + 'GigabitEthernet0 10.0.0.1\r\n' + ) + self.assertEqual( + 'Interface\tIP-Address\nGigabitEthernet0 10.0.0.1\n', + self.protocol.get_response(), + ) + + def testCleanupResponse(self): + # Tested indirectly through testGetResponse(). + pass + + def testGuessOs(self): + self.assertEqual('unknown', self.protocol.guess_os()) + # Other tests can not work on the abstract base. + if self.protocol.__class__ == Protocol: + return + self.doConnect() + self.assertEqual('unknown', self.protocol.guess_os()) + self.protocol.login(self.account) + self.assertTrue(self.protocol.is_protocol_authenticated()) + self.assertTrue(self.protocol.is_app_authenticated()) + self.assertTrue(self.protocol.is_app_authorized()) + self.assertEqual('shell', self.protocol.guess_os()) + def testSend(self): # Test can not work on the abstract base. if self.protocol.__class__ == Protocol: @@ -436,14 +554,14 @@ def testSend(self): return self.doLogin() self.protocol.execute('ls') - - self.protocol.send('df\r') self.assertTrue(self.protocol.response is not None) self.assertTrue(self.protocol.response.startswith('ls')) + self.protocol.send('df\r') + self.assertTrue(self.protocol.response is None) + self.protocol.send('exit\r') - self.assertTrue(self.protocol.response is not None) - self.assertTrue(self.protocol.response.startswith('ls')) + self.assertTrue(self.protocol.response is None) def testExecute(self): # Test can not work on the abstract base. @@ -468,14 +586,13 @@ def testWaitfor(self): self.assertRaises(Exception, self.protocol.waitfor, 'ls') return self.doLogin() - oldresponse = self.protocol.response self.protocol.send('ls\r') - self.assertEqual(oldresponse, self.protocol.response) self.protocol.waitfor(re.compile(r'[\r\n]')) - self.assertNotEqual(oldresponse, self.protocol.response) - oldresponse = self.protocol.response + self.assertTrue(self.protocol.response is not None) + self.assertTrue(self.protocol.response.startswith('ls')) self.protocol.waitfor(re.compile(r'[\r\n]')) - self.assertEqual(oldresponse, self.protocol.response) + self.assertTrue(self.protocol.response is not None) + self.assertTrue(self.protocol.response.startswith('ls')) def testExpect(self): # Test can not work on the abstract base. @@ -483,11 +600,10 @@ def testExpect(self): self.assertRaises(Exception, self.protocol.expect, 'ls') return self.doLogin() - oldresponse = self.protocol.response self.protocol.send('ls\r') - self.assertEqual(oldresponse, self.protocol.response) self.protocol.expect(re.compile(r'[\r\n]')) - self.assertNotEqual(oldresponse, self.protocol.response) + self.assertTrue(self.protocol.response is not None) + self.assertTrue(self.protocol.response.startswith('ls')) def testExpectPrompt(self): # Test can not work on the abstract base. @@ -495,11 +611,10 @@ def testExpectPrompt(self): self.assertRaises(Exception, self.protocol.expect, 'ls') return self.doLogin() - oldresponse = self.protocol.response self.protocol.send('ls\r') - self.assertEqual(oldresponse, self.protocol.response) self.protocol.expect_prompt() - self.assertNotEqual(oldresponse, self.protocol.response) + self.assertTrue(self.protocol.response is not None) + self.assertTrue(self.protocol.response.startswith('ls')) def testAddMonitor(self): # Set the monitor callback up. @@ -510,11 +625,11 @@ def monitor_cb(thedata, *args, **kwargs): self.protocol.add_monitor('abc', partial(monitor_cb, data)) # Simulate some non-matching data. - self.protocol.buffer.append(u'aaa') + self.protocol._buffer.append(u'aaa') self.assertEqual(data, {}) # Simulate some matching data. - self.protocol.buffer.append(u'abc') + self.protocol._buffer.append(u'abc') self.assertEqual(len(data.get('args')), 3) self.assertEqual(data.get('args')[0], self.protocol) self.assertEqual(data.get('args')[1], 0) @@ -525,15 +640,15 @@ def testGetBuffer(self): # Test can not work on the abstract base. if self.protocol.__class__ == Protocol: return - self.assertEqual(str(self.protocol.buffer), '') + self.assertEqual(str(self.protocol._buffer), '') self.doLogin() # Depending on whether the connected host sends a banner, # the buffer may or may not contain anything now. - before = str(self.protocol.buffer) + before = str(self.protocol._buffer) self.protocol.send('ls\r') self.protocol.waitfor(self.protocol.get_prompt()) - self.assertNotEqual(str(self.protocol.buffer), before) + self.assertNotEqual(str(self.protocol._buffer), before) def _cancel_cb(self, data): self.protocol.cancel_expect() @@ -543,10 +658,8 @@ def testCancelExpect(self): if self.protocol.__class__ == Protocol: return self.doLogin() - oldresponse = self.protocol.response self.protocol.data_received_event.connect(self._cancel_cb) self.protocol.send('ls\r') - self.assertEqual(oldresponse, self.protocol.response) self.assertRaises(ExpectCancelledException, self.protocol.expect, 'notgoingtohappen') @@ -558,31 +671,246 @@ def testInteract(self): return # Can't really be tested. - def testClose(self): - if self.protocol.__class__ != Protocol: - self.doConnect() - self.protocol.close(True) + def testSwitchDeviceContext(self): + # Stack is empty at construction time. + self.assertEqual(self.protocol._device_context_stack, []) + + # Set up initial device state including OS guesser to verify capture/restore. + # Simulate guesser state that would have been built up during a session. + self.protocol._os_guesser._info = {'os': ('ios', 80)} + self.protocol._os_guesser._auth_buffer = 'banner data' + self.protocol._os_guesser._protocol_buffer = 'SSH-2.0-OpenSSH' + self.protocol._os_guesser._response_buffer = 'router#' + self.protocol._auto_driver = drivers.driver_map[self.protocol.guess_os()] + self.protocol._app_authenticated = True + self.protocol._app_authorized = True + self.protocol._last_account = self.account + self.protocol._manual_prompt_re = [re.compile(r'router#')] + self.protocol._manual_error_re = [re.compile(r'% Error')] + self.protocol._manual_user_re = [re.compile(r'Username:')] + self.protocol._manual_password_re = [re.compile(r'Password:')] + self.protocol._manual_login_error_re = [re.compile(r'Login failed')] + self.protocol.set_driver('junos') + + # -- Switch to a remote hop (default: reset_auth=True) -- + self.protocol.switch_device_context() + self.assertEqual(len(self.protocol._device_context_stack), 1) + snapshot = self.protocol._device_context_stack[0] + + # Original state captured correctly — auth. + self.assertEqual(snapshot['app_authenticated'], True) + self.assertEqual(snapshot['app_authorized'], True) + self.assertIs(snapshot['last_account'], self.account) + # Original state captured correctly — guesser. + self.assertEqual(snapshot['guesser_info'], {'os': ('ios', 80)}) + self.assertEqual(snapshot['guesser_auth_buffer'], 'banner data') + self.assertEqual(snapshot['guesser_protocol_buffer'], 'SSH-2.0-OpenSSH') + self.assertEqual(snapshot['guesser_response_buffer'], 'router#') + # Original state captured correctly — driver. + self.assertIsNotNone(snapshot['auto_driver']) + self.assertEqual(snapshot['auto_driver'].name, 'ios') + self.assertIsNotNone(snapshot['manual_driver']) + self.assertEqual(snapshot['manual_driver'].name, 'junos') + # Original state captured correctly — patterns. + self.assertIsNotNone(snapshot['manual_prompt_re']) + self.assertIsNotNone(snapshot['manual_error_re']) + self.assertIsNotNone(snapshot['manual_user_re']) + self.assertIsNotNone(snapshot['manual_password_re']) + self.assertIsNotNone(snapshot['manual_login_error_re']) + + # New context: auth reset. + self.assertFalse(self.protocol.is_app_authenticated()) + self.assertFalse(self.protocol.is_app_authorized()) + self.assertIsNone(self.protocol._last_account) + # New context: guesser reset — info back to unknown, all buffers empty. + self.assertEqual(self.protocol._os_guesser._info, {'os': ('unknown', 0)}) + self.assertEqual(self.protocol._os_guesser._auth_buffer, '') + self.assertEqual(self.protocol._os_guesser._protocol_buffer, '') + self.assertEqual(self.protocol._os_guesser._response_buffer, '') + # New context: manual driver and all patterns cleared. + self.assertIsNone(self.protocol._manual_driver) + self.assertIsNone(self.protocol._manual_prompt_re) + self.assertIsNone(self.protocol._manual_error_re) + self.assertIsNone(self.protocol._manual_user_re) + self.assertIsNone(self.protocol._manual_password_re) + self.assertIsNone(self.protocol._manual_login_error_re) + # New context: auto driver follows guesser (unknown → generic). + self.assertEqual(self.protocol.get_driver().name, 'generic') - def testGetHost(self): - self.assertTrue(self.protocol.get_host() is None) + # -- Nested switch: local shell (reset_auth=False, no driver param) -- + hop_account = Account('hop_user', password='hop_pass') + self.protocol._app_authenticated = True # simulate auth on the hop + self.protocol._app_authorized = True + self.protocol._last_account = hop_account + self.protocol.switch_device_context(reset_auth=False) + self.assertEqual(len(self.protocol._device_context_stack), 2) + + # Auth flags are preserved when reset_auth=False. + self.assertTrue(self.protocol.is_app_authenticated()) + self.assertTrue(self.protocol.is_app_authorized()) + # last_account is preserved when reset_auth=False (same device, same account). + self.assertIs(self.protocol._last_account, hop_account) + # Manual driver auto-cleared on switch; caller sets it explicitly if needed. + self.assertIsNone(self.protocol._manual_driver) + self.protocol.set_driver('shell') + self.assertEqual(self.protocol.get_driver().name, 'shell') + + # -- Restore to the first hop context -- + self.protocol.restore_device_context() + self.assertEqual(len(self.protocol._device_context_stack), 1) + + # Auth was True in the first-hop context when the nested switch was called + # (we simulated a successful login on the hop before entering the local shell). + self.assertTrue(self.protocol.is_app_authenticated()) + self.assertTrue(self.protocol.is_app_authorized()) + self.assertIs(self.protocol._last_account, hop_account) + # First-hop context had a reset guesser. + self.assertEqual(self.protocol._os_guesser._info, {'os': ('unknown', 0)}) + self.assertEqual(self.protocol._os_guesser._auth_buffer, '') + self.assertEqual(self.protocol._os_guesser._protocol_buffer, '') + self.assertEqual(self.protocol._os_guesser._response_buffer, '') + # First-hop context had auto driver (generic) and no manual driver/patterns. + self.assertIsNone(self.protocol._manual_driver) + self.assertIsNone(self.protocol._manual_prompt_re) + self.assertIsNone(self.protocol._manual_error_re) + self.assertIsNone(self.protocol._manual_user_re) + self.assertIsNone(self.protocol._manual_password_re) + self.assertIsNone(self.protocol._manual_login_error_re) + + # -- Restore to the original context -- + self.protocol.restore_device_context() + self.assertEqual(self.protocol._device_context_stack, []) + + # Auth restored. + self.assertTrue(self.protocol.is_app_authenticated()) + self.assertTrue(self.protocol.is_app_authorized()) + self.assertIs(self.protocol._last_account, self.account) + # Guesser restored. + self.assertEqual(self.protocol._os_guesser._info, {'os': ('ios', 80)}) + self.assertEqual(self.protocol._os_guesser._auth_buffer, 'banner data') + self.assertEqual(self.protocol._os_guesser._protocol_buffer, 'SSH-2.0-OpenSSH') + self.assertEqual(self.protocol._os_guesser._response_buffer, 'router#') + # Driver restored (manual junos takes effect). + self.assertEqual(self.protocol.get_driver().name, 'junos') + # Patterns restored. + self.assertIsNotNone(self.protocol._manual_prompt_re) + self.assertIsNotNone(self.protocol._manual_error_re) + self.assertIsNotNone(self.protocol._manual_user_re) + self.assertIsNotNone(self.protocol._manual_password_re) + self.assertIsNotNone(self.protocol._manual_login_error_re) + + # -- restore_device_context() raises when the stack is empty -- + self.assertRaises(ProtocolException, self.protocol.restore_device_context) + + def testRestoreDeviceContext(self): + pass # Tested in full detail by testSwitchDeviceContext(). + + def testClose(self): if self.protocol.__class__ == Protocol: return - self.doConnect() - self.assertEqual(self.protocol.get_host(), self.hostname) + self.doLogin() + self.protocol.execute('ls') + driver = self.protocol.get_driver() + if self.protocol._logout.__func__ is Protocol._logout: + # Base _logout delegates to driver.logout(), which a forced close() must not call. + with unittest.mock.patch.object(driver, 'logout') as mock_driver_logout: + self.protocol.close(force=True) + mock_driver_logout.assert_not_called() + else: + # Subclass overrides _logout — verify it is not called by close(). + with unittest.mock.patch.object(self.protocol, '_logout') as mock_logout: + self.protocol.close(force=True) + mock_logout.assert_not_called() + self.assertEqual(0, self.protocol._buffer.size()) + self.assertEqual('unknown', self.protocol.guess_os()) + self.assertEqual(0, self.protocol.stdout.tell()) + self.assertFalse(self.protocol.connected) + self.assertTrue(self.protocol.response is not None) - def testGuessOs(self): + def testGracefulClose(self): + if self.protocol.__class__ == Protocol: + return + self.doLogin() + self.protocol.execute('ls') + driver = self.protocol.get_driver() + if self.protocol._logout.__func__ is Protocol._logout: + # Base _logout delegates to driver.logout() — verify the full chain. + with unittest.mock.patch.object(driver, 'logout') as mock_driver_logout: + self.protocol.close(force=False) + mock_driver_logout.assert_called_once() + else: + # Subclass overrides _logout — verify it is called by close(). + with unittest.mock.patch.object(self.protocol, '_logout') as mock_logout: + self.protocol.close(force=False) + mock_logout.assert_called_once() + self.assertEqual(0, self.protocol._buffer.size()) self.assertEqual('unknown', self.protocol.guess_os()) - # Other tests can not work on the abstract base. + self.assertEqual(0, self.protocol.stdout.tell()) + self.assertFalse(self.protocol.connected) + self.assertTrue(self.protocol.response is not None) + + def testReconnect(self): + # Test can not work on the abstract base. if self.protocol.__class__ == Protocol: return - self.doConnect() + + # Assign a logfile before the first connect. + # No FD should be open yet — the file is only opened on connect(). + fd, logpath = tempfile.mkstemp(suffix='.log') + os.close(fd) + self.addCleanup(os.unlink, logpath) + self.protocol.logfile = logpath + self.assertIsNone(self.protocol._log) + + # --- First session --- + self.doLogin() + self.assertTrue(self.protocol._log is not None) # FD opened by connect() + self.protocol.execute('ls') + first_response = self.protocol.response + self.assertTrue(first_response is not None) + self.assertTrue(first_response.startswith('ls')) + self.assertTrue(self.protocol.is_app_authenticated()) + self.assertTrue(self.protocol.is_app_authorized()) + self.assertEqual('shell', self.protocol.guess_os()) + + self.protocol.close(force=True) + + # close() resets session state; response and logfile path are preserved. + self.assertFalse(self.protocol._connected) + self.assertIsNone(self.protocol._log) # FD closed by _reset() + self.assertEqual(logpath, self.protocol.logfile) # path survives close + self.assertEqual(0, self.protocol._buffer.size()) self.assertEqual('unknown', self.protocol.guess_os()) - self.protocol.login(self.account) - self.assertTrue(self.protocol.is_protocol_authenticated()) + self.assertFalse(self.protocol.is_app_authenticated()) + self.assertFalse(self.protocol.is_app_authorized()) + self.assertEqual(first_response, self.protocol.response) + + # First session data must have been flushed to disk. + with open(logpath) as f: + first_log_size = len(f.read()) + self.assertGreater(first_log_size, 0) + + # --- Second session on the same instance (reconnect) --- + self.doLogin() + self.assertTrue(self.protocol._log is not None) # FD reopened by connect() + self.protocol.execute('ls') + second_response = self.protocol.response + self.assertTrue(second_response is not None) + self.assertTrue(second_response.startswith('ls')) self.assertTrue(self.protocol.is_app_authenticated()) self.assertTrue(self.protocol.is_app_authorized()) self.assertEqual('shell', self.protocol.guess_os()) + self.protocol.close(force=True) + + self.assertIsNone(self.protocol._log) + self.assertEqual(logpath, self.protocol.logfile) + + # Second session appended to the same logfile. + with open(logpath) as f: + second_log_size = len(f.read()) + self.assertGreater(second_log_size, first_log_size) + def suite(): return unittest.TestLoader().loadTestsFromTestCase(ProtocolTest) diff --git a/tests/Exscript/protocols/SSH2Test.py b/tests/Exscript/protocols/SSH2Test.py index e57591f8..0316690c 100644 --- a/tests/Exscript/protocols/SSH2Test.py +++ b/tests/Exscript/protocols/SSH2Test.py @@ -1,14 +1,15 @@ -from __future__ import absolute_import +import os.path +import re import sys import unittest -import re -import os.path + sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..', '..')) -from .ProtocolTest import ProtocolTest -from Exscript.servers import sshd, SSHd -from Exscript.protocols import SSH2 from Exscript import PrivateKey +from Exscript.protocols import SSH2 +from Exscript.servers import SSHd, sshd + +from .ProtocolTest import ProtocolTest keyfile = os.path.join(os.path.dirname(__file__), 'id_rsa') key = PrivateKey.from_file(keyfile) @@ -19,9 +20,10 @@ class SSH2Test(ProtocolTest): def createDaemon(self): self.daemon = SSHd(self.hostname, self.port, self.device, key=key) + self.daemon.dbg = 5 def createProtocol(self): - self.protocol = SSH2(timeout=1) + self.protocol = SSH2(timeout=1, debug=5) def testConstructor(self): self.assertIsInstance(self.protocol, SSH2) diff --git a/tests/Exscript/protocols/TelnetTest.py b/tests/Exscript/protocols/TelnetTest.py index 99d1e054..d0687c9f 100644 --- a/tests/Exscript/protocols/TelnetTest.py +++ b/tests/Exscript/protocols/TelnetTest.py @@ -1,15 +1,16 @@ -from __future__ import absolute_import -import sys -import unittest -import re import os +import re +import sys import time +import unittest + sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..')) -from .ProtocolTest import ProtocolTest from Exscript import Account -from Exscript.servers import Telnetd from Exscript.protocols import Telnet +from Exscript.servers import Telnetd + +from .ProtocolTest import ProtocolTest class TelnetTest(ProtocolTest): @@ -17,9 +18,10 @@ class TelnetTest(ProtocolTest): def createDaemon(self): self.daemon = Telnetd(self.hostname, self.port, self.device) + self.daemon.dbg = 5 def createProtocol(self): - self.protocol = Telnet(timeout=1) + self.protocol = Telnet(timeout=1, debug=5) def testConstructor(self): self.assertIsInstance(self.protocol, Telnet) diff --git a/tests/Exscript/protocols/run_suite.py b/tests/Exscript/protocols/run_suite.py index d2e47d3b..0520fafd 100755 --- a/tests/Exscript/protocols/run_suite.py +++ b/tests/Exscript/protocols/run_suite.py @@ -1,13 +1,17 @@ -#!/usr/bin/env python -from __future__ import print_function +#!/usr/bin/env python3 +import fnmatch +import functools +import glob +import logging import os +import re import sys +import time import unittest -import glob -import fnmatch -import re -from inspect import isfunction, ismodule, isclass +from inspect import isclass, isfunction, ismodule +logger = logging.getLogger('TestSuite') +logger.addHandler(logging.NullHandler()) def uppercase(match): return match.group(1).upper() @@ -15,6 +19,75 @@ def uppercase(match): correlated = dict() +def setup_logging(filename=None): + logformat = "%(levelname)s: %(name)s (%(threadName)s) %(message)s" + loglevel = logging.DEBUG + logger_names = [ + "Exscript", + "Exscript.interpreter.parser", + "Exscript.parselib.lexer", + "Exscript.protocols", + "Exscript.servers", + # Netconf Client + "ncclient.capabilities", + "ncclient.devices", + "ncclient.manager", + "ncclient.operations", + "ncclient.transport.session", + "ncclient.transport.ssh", + # Paramiko + "paramiko.transport", + # Tests may also log. + "TestSuite", + ] + formatter = logging.Formatter(logformat) + formatter.converter = time.gmtime + if filename is None: + handler = logging.StreamHandler(sys.stderr) + else: + handler = logging.FileHandler(filename, mode="w", encoding="utf8") + handler.setFormatter(formatter) + handler.setLevel(loglevel) + for name in logger_names: + logger = logging.getLogger(name) + logger.setLevel(logging.DEBUG) + logger.addHandler(handler) + logger.propagate = False + + +def log_suites(suites): + for item in suites: + if isinstance(item, unittest.TestSuite): + log_suites(item) # suites are iterable + elif isinstance(item, unittest.TestCase): + testname = item.id() + method_names = ( + 'setUpClass', + 'setUp', + testname.split('.')[-1], + 'tearDown', + 'tearDownClass', + ) + for name in method_names: + method = getattr(item, name, None) + if not method: + continue + msg = 'Running %s (%s)' % (name, testname) + setattr(item, name, log_call(msg)(method)) + else: + raise TypeError("Unexpected type: %s" % type(item)) + + +def log_call(msg): + def decorator(func): + @functools.wraps(func) + def wrapper(*args, **kwargs): + logger.debug(msg) + return func(*args, **kwargs) + return wrapper + return decorator + + def correlate_class(theclass): """ Checks the given testcase for missing test methods. @@ -74,6 +147,7 @@ def find(dirname, pattern): def load_suite(files): modules = [os.path.splitext(f)[0] for f in files] + modules.sort() all_suites = [] for name in modules: name = name.lstrip('.').lstrip('/').replace('/', '.') @@ -83,6 +157,7 @@ def load_suite(files): if correlated: sys.stderr.write('Error: Untested methods found.\n') sys.exit(1) + log_suites(all_suites) return unittest.TestSuite(all_suites) @@ -107,4 +182,7 @@ def recursive_suite(): sys.exit(1) # Run. + logfile = os.path.splitext(__file__)[0] + '.log' + setup_logging(logfile) unittest.TextTestRunner(verbosity=verbosity).run(recursive_suite()) + print('Logfile with Exscript and TestSuite activity: %s' % logfile) diff --git a/tests/Exscript/run_suite.py b/tests/Exscript/run_suite.py index d2e47d3b..0520fafd 100755 --- a/tests/Exscript/run_suite.py +++ b/tests/Exscript/run_suite.py @@ -1,13 +1,17 @@ -#!/usr/bin/env python -from __future__ import print_function +#!/usr/bin/env python3 +import fnmatch +import functools +import glob +import logging import os +import re import sys +import time import unittest -import glob -import fnmatch -import re -from inspect import isfunction, ismodule, isclass +from inspect import isclass, isfunction, ismodule +logger = logging.getLogger('TestSuite') +logger.addHandler(logging.NullHandler()) def uppercase(match): return match.group(1).upper() @@ -15,6 +19,75 @@ def uppercase(match): correlated = dict() +def setup_logging(filename=None): + logformat = "%(levelname)s: %(name)s (%(threadName)s) %(message)s" + loglevel = logging.DEBUG + logger_names = [ + "Exscript", + "Exscript.interpreter.parser", + "Exscript.parselib.lexer", + "Exscript.protocols", + "Exscript.servers", + # Netconf Client + "ncclient.capabilities", + "ncclient.devices", + "ncclient.manager", + "ncclient.operations", + "ncclient.transport.session", + "ncclient.transport.ssh", + # Paramiko + "paramiko.transport", + # Tests may also log. + "TestSuite", + ] + formatter = logging.Formatter(logformat) + formatter.converter = time.gmtime + if filename is None: + handler = logging.StreamHandler(sys.stderr) + else: + handler = logging.FileHandler(filename, mode="w", encoding="utf8") + handler.setFormatter(formatter) + handler.setLevel(loglevel) + for name in logger_names: + logger = logging.getLogger(name) + logger.setLevel(logging.DEBUG) + logger.addHandler(handler) + logger.propagate = False + + +def log_suites(suites): + for item in suites: + if isinstance(item, unittest.TestSuite): + log_suites(item) # suites are iterable + elif isinstance(item, unittest.TestCase): + testname = item.id() + method_names = ( + 'setUpClass', + 'setUp', + testname.split('.')[-1], + 'tearDown', + 'tearDownClass', + ) + for name in method_names: + method = getattr(item, name, None) + if not method: + continue + msg = 'Running %s (%s)' % (name, testname) + setattr(item, name, log_call(msg)(method)) + else: + raise TypeError("Unexpected type: %s" % type(item)) + + +def log_call(msg): + def decorator(func): + @functools.wraps(func) + def wrapper(*args, **kwargs): + logger.debug(msg) + return func(*args, **kwargs) + return wrapper + return decorator + + def correlate_class(theclass): """ Checks the given testcase for missing test methods. @@ -74,6 +147,7 @@ def find(dirname, pattern): def load_suite(files): modules = [os.path.splitext(f)[0] for f in files] + modules.sort() all_suites = [] for name in modules: name = name.lstrip('.').lstrip('/').replace('/', '.') @@ -83,6 +157,7 @@ def load_suite(files): if correlated: sys.stderr.write('Error: Untested methods found.\n') sys.exit(1) + log_suites(all_suites) return unittest.TestSuite(all_suites) @@ -107,4 +182,7 @@ def recursive_suite(): sys.exit(1) # Run. + logfile = os.path.splitext(__file__)[0] + '.log' + setup_logging(logfile) unittest.TextTestRunner(verbosity=verbosity).run(recursive_suite()) + print('Logfile with Exscript and TestSuite activity: %s' % logfile) diff --git a/tests/Exscript/servers/SSHdTest.py b/tests/Exscript/servers/SSHdTest.py index 4a5fde16..e5e4125a 100644 --- a/tests/Exscript/servers/SSHdTest.py +++ b/tests/Exscript/servers/SSHdTest.py @@ -1,13 +1,14 @@ -from __future__ import absolute_import +import os +import re import sys import unittest -import re -import os + sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..')) -from .ServerTest import ServerTest -from Exscript.servers import SSHd from Exscript.protocols import SSH2 +from Exscript.servers import SSHd + +from .ServerTest import ServerTest class SSHdTest(ServerTest): diff --git a/tests/Exscript/servers/ServerTest.py b/tests/Exscript/servers/ServerTest.py index 9d0dc3b4..7c473fbf 100644 --- a/tests/Exscript/servers/ServerTest.py +++ b/tests/Exscript/servers/ServerTest.py @@ -1,14 +1,15 @@ -from builtins import str +import os +import re import sys import unittest -import re -import os + sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..')) import time + from Exscript import Account -from Exscript.servers.server import Server from Exscript.emulators import VirtualDevice +from Exscript.servers.server import Server class ServerTest(unittest.TestCase): diff --git a/tests/Exscript/servers/TelnetdTest.py b/tests/Exscript/servers/TelnetdTest.py index df9fb98d..c26f26d9 100644 --- a/tests/Exscript/servers/TelnetdTest.py +++ b/tests/Exscript/servers/TelnetdTest.py @@ -1,13 +1,14 @@ -from __future__ import absolute_import +import os +import re import sys import unittest -import re -import os + sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..')) -from .ServerTest import ServerTest -from Exscript.servers import Telnetd from Exscript.protocols import Telnet +from Exscript.servers import Telnetd + +from .ServerTest import ServerTest class TelnetdTest(ServerTest): diff --git a/tests/Exscript/servers/run_suite.py b/tests/Exscript/servers/run_suite.py index d2e47d3b..0520fafd 100755 --- a/tests/Exscript/servers/run_suite.py +++ b/tests/Exscript/servers/run_suite.py @@ -1,13 +1,17 @@ -#!/usr/bin/env python -from __future__ import print_function +#!/usr/bin/env python3 +import fnmatch +import functools +import glob +import logging import os +import re import sys +import time import unittest -import glob -import fnmatch -import re -from inspect import isfunction, ismodule, isclass +from inspect import isclass, isfunction, ismodule +logger = logging.getLogger('TestSuite') +logger.addHandler(logging.NullHandler()) def uppercase(match): return match.group(1).upper() @@ -15,6 +19,75 @@ def uppercase(match): correlated = dict() +def setup_logging(filename=None): + logformat = "%(levelname)s: %(name)s (%(threadName)s) %(message)s" + loglevel = logging.DEBUG + logger_names = [ + "Exscript", + "Exscript.interpreter.parser", + "Exscript.parselib.lexer", + "Exscript.protocols", + "Exscript.servers", + # Netconf Client + "ncclient.capabilities", + "ncclient.devices", + "ncclient.manager", + "ncclient.operations", + "ncclient.transport.session", + "ncclient.transport.ssh", + # Paramiko + "paramiko.transport", + # Tests may also log. + "TestSuite", + ] + formatter = logging.Formatter(logformat) + formatter.converter = time.gmtime + if filename is None: + handler = logging.StreamHandler(sys.stderr) + else: + handler = logging.FileHandler(filename, mode="w", encoding="utf8") + handler.setFormatter(formatter) + handler.setLevel(loglevel) + for name in logger_names: + logger = logging.getLogger(name) + logger.setLevel(logging.DEBUG) + logger.addHandler(handler) + logger.propagate = False + + +def log_suites(suites): + for item in suites: + if isinstance(item, unittest.TestSuite): + log_suites(item) # suites are iterable + elif isinstance(item, unittest.TestCase): + testname = item.id() + method_names = ( + 'setUpClass', + 'setUp', + testname.split('.')[-1], + 'tearDown', + 'tearDownClass', + ) + for name in method_names: + method = getattr(item, name, None) + if not method: + continue + msg = 'Running %s (%s)' % (name, testname) + setattr(item, name, log_call(msg)(method)) + else: + raise TypeError("Unexpected type: %s" % type(item)) + + +def log_call(msg): + def decorator(func): + @functools.wraps(func) + def wrapper(*args, **kwargs): + logger.debug(msg) + return func(*args, **kwargs) + return wrapper + return decorator + + def correlate_class(theclass): """ Checks the given testcase for missing test methods. @@ -74,6 +147,7 @@ def find(dirname, pattern): def load_suite(files): modules = [os.path.splitext(f)[0] for f in files] + modules.sort() all_suites = [] for name in modules: name = name.lstrip('.').lstrip('/').replace('/', '.') @@ -83,6 +157,7 @@ def load_suite(files): if correlated: sys.stderr.write('Error: Untested methods found.\n') sys.exit(1) + log_suites(all_suites) return unittest.TestSuite(all_suites) @@ -107,4 +182,7 @@ def recursive_suite(): sys.exit(1) # Run. + logfile = os.path.splitext(__file__)[0] + '.log' + setup_logging(logfile) unittest.TextTestRunner(verbosity=verbosity).run(recursive_suite()) + print('Logfile with Exscript and TestSuite activity: %s' % logfile) diff --git a/tests/Exscript/util/bufferTest.py b/tests/Exscript/util/bufferTest.py index dedf4e26..9fb2c3d6 100644 --- a/tests/Exscript/util/bufferTest.py +++ b/tests/Exscript/util/bufferTest.py @@ -1,13 +1,13 @@ -from __future__ import unicode_literals -from builtins import str +import os +import re import sys import unittest -import re -import os + sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..', '..')) -from tempfile import TemporaryFile from functools import partial +from tempfile import TemporaryFile + from Exscript.util.buffer import MonitoredBuffer diff --git a/tests/Exscript/util/decoratorTest.py b/tests/Exscript/util/decoratorTest.py index 27ef53a8..16b4bc12 100644 --- a/tests/Exscript/util/decoratorTest.py +++ b/tests/Exscript/util/decoratorTest.py @@ -1,21 +1,22 @@ -from builtins import object +import os.path +import re import sys import unittest -import re -import os.path + sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..', '..')) from functools import partial +from multiprocessing import Value + import Exscript.util.decorator -from Exscript.util.impl import get_label -from Exscript import Host, Account +from Exscript import Account, Host from Exscript.protocols import Protocol from Exscript.protocols.exception import LoginFailure +from Exscript.util.impl import get_label from util.reportTest import FakeJob -from multiprocessing import Value -class FakeConnection(object): +class FakeConnection: def __init__(self, os=None): self.os = os diff --git a/tests/Exscript/util/fileTest.py b/tests/Exscript/util/fileTest.py index 16820707..7d2de588 100644 --- a/tests/Exscript/util/fileTest.py +++ b/tests/Exscript/util/fileTest.py @@ -13,7 +13,7 @@ ('user3', 'password3'), ('user4', 'password4')] -hosts = ['localhost', '1.2.3.4', 'ssh://test', 'ssh1://another:23'] +hosts = ['localhost', '1.2.3.4', 'ssh://test', 'telnet://another:23'] expected_hosts = ['localhost', '1.2.3.4', 'test', 'another'] diff --git a/tests/Exscript/util/ipv4Test.py b/tests/Exscript/util/ipv4Test.py index bd75ed0d..f531f76f 100644 --- a/tests/Exscript/util/ipv4Test.py +++ b/tests/Exscript/util/ipv4Test.py @@ -1,8 +1,8 @@ -from builtins import range +import os.path +import re import sys import unittest -import re -import os.path + sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..', '..')) import Exscript.util.ipv4 @@ -57,7 +57,7 @@ def testInt2Ip(self): self.assertEqual(int2ip(ip2int(ip)), ip) def testPfxlen2MaskInt(self): - from Exscript.util.ipv4 import pfxlen2mask_int, int2ip + from Exscript.util.ipv4 import int2ip, pfxlen2mask_int self.assertEqual(int2ip(pfxlen2mask_int(32)), '255.255.255.255') self.assertEqual(int2ip(pfxlen2mask_int(31)), '255.255.255.254') self.assertEqual(int2ip(pfxlen2mask_int(30)), '255.255.255.252') @@ -140,8 +140,9 @@ def testIsPrivate(self): self.assertFalse(is_private('192.169.0.0')) def testSort(self): - from Exscript.util.ipv4 import sort import random + + from Exscript.util.ipv4 import sort ip_list = ['0.0.0.0', '0.0.0.255', '1.2.3.4', diff --git a/tests/Exscript/util/mailTest.py b/tests/Exscript/util/mailTest.py index ea0fc020..c3972824 100644 --- a/tests/Exscript/util/mailTest.py +++ b/tests/Exscript/util/mailTest.py @@ -105,11 +105,11 @@ def testAddBcc(self): def testGetBcc(self): pass # see testSetBcc() - def testGetReceipients(self): + def testGetRecipients(self): self.mail.set_to('test1') self.mail.set_cc('test2') self.mail.set_bcc('test3') - self.assertEqual(self.mail.get_receipients(), + self.assertEqual(self.mail.get_recipients(), ['test1', 'test2', 'test3']) def testSetSubject(self): diff --git a/tests/Exscript/util/reportTest.py b/tests/Exscript/util/reportTest.py index 182a25f2..4e19b130 100644 --- a/tests/Exscript/util/reportTest.py +++ b/tests/Exscript/util/reportTest.py @@ -1,24 +1,23 @@ -from builtins import str -from builtins import object +import os +import re import sys import unittest -import re -import os + sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..', '..')) import traceback + import Exscript.util.report -from Exscript import Host -from Exscript import Logger +from Exscript import Host, Logger from Exscript.util.event import Event from Exscript.workqueue import WorkQueue -class FakeQueue(object): +class FakeQueue: workqueue = WorkQueue() -class FakeJob(object): +class FakeJob: def __init__(self, name='fake'): self.function = lambda x: None @@ -39,6 +38,7 @@ class reportTest(unittest.TestCase): def setUp(self): self.logger = Logger() self.n_actions = 0 + self.maxDiff = None def createLog(self): self.n_actions += 1 @@ -104,12 +104,9 @@ def testFormat(self): ------------------- fake1 fake3'''.strip() % file - expected_py3 = expected.replace('FakeError\n', - 'util.reportTest.FakeError\n') - if sys.version_info[0] < 3: - self.assertEqual(format(self.logger), expected) - else: - self.assertEqual(format(self.logger), expected_py3) + expected = expected.replace('FakeError\n', + 'util.reportTest.FakeError\n') + self.assertEqual(format(self.logger), expected) def suite(): diff --git a/tests/Exscript/util/run_suite.py b/tests/Exscript/util/run_suite.py index d2e47d3b..0520fafd 100755 --- a/tests/Exscript/util/run_suite.py +++ b/tests/Exscript/util/run_suite.py @@ -1,13 +1,17 @@ -#!/usr/bin/env python -from __future__ import print_function +#!/usr/bin/env python3 +import fnmatch +import functools +import glob +import logging import os +import re import sys +import time import unittest -import glob -import fnmatch -import re -from inspect import isfunction, ismodule, isclass +from inspect import isclass, isfunction, ismodule +logger = logging.getLogger('TestSuite') +logger.addHandler(logging.NullHandler()) def uppercase(match): return match.group(1).upper() @@ -15,6 +19,75 @@ def uppercase(match): correlated = dict() +def setup_logging(filename=None): + logformat = "%(levelname)s: %(name)s (%(threadName)s) %(message)s" + loglevel = logging.DEBUG + logger_names = [ + "Exscript", + "Exscript.interpreter.parser", + "Exscript.parselib.lexer", + "Exscript.protocols", + "Exscript.servers", + # Netconf Client + "ncclient.capabilities", + "ncclient.devices", + "ncclient.manager", + "ncclient.operations", + "ncclient.transport.session", + "ncclient.transport.ssh", + # Paramiko + "paramiko.transport", + # Tests may also log. + "TestSuite", + ] + formatter = logging.Formatter(logformat) + formatter.converter = time.gmtime + if filename is None: + handler = logging.StreamHandler(sys.stderr) + else: + handler = logging.FileHandler(filename, mode="w", encoding="utf8") + handler.setFormatter(formatter) + handler.setLevel(loglevel) + for name in logger_names: + logger = logging.getLogger(name) + logger.setLevel(logging.DEBUG) + logger.addHandler(handler) + logger.propagate = False + + +def log_suites(suites): + for item in suites: + if isinstance(item, unittest.TestSuite): + log_suites(item) # suites are iterable + elif isinstance(item, unittest.TestCase): + testname = item.id() + method_names = ( + 'setUpClass', + 'setUp', + testname.split('.')[-1], + 'tearDown', + 'tearDownClass', + ) + for name in method_names: + method = getattr(item, name, None) + if not method: + continue + msg = 'Running %s (%s)' % (name, testname) + setattr(item, name, log_call(msg)(method)) + else: + raise TypeError("Unexpected type: %s" % type(item)) + + +def log_call(msg): + def decorator(func): + @functools.wraps(func) + def wrapper(*args, **kwargs): + logger.debug(msg) + return func(*args, **kwargs) + return wrapper + return decorator + + def correlate_class(theclass): """ Checks the given testcase for missing test methods. @@ -74,6 +147,7 @@ def find(dirname, pattern): def load_suite(files): modules = [os.path.splitext(f)[0] for f in files] + modules.sort() all_suites = [] for name in modules: name = name.lstrip('.').lstrip('/').replace('/', '.') @@ -83,6 +157,7 @@ def load_suite(files): if correlated: sys.stderr.write('Error: Untested methods found.\n') sys.exit(1) + log_suites(all_suites) return unittest.TestSuite(all_suites) @@ -107,4 +182,7 @@ def recursive_suite(): sys.exit(1) # Run. + logfile = os.path.splitext(__file__)[0] + '.log' + setup_logging(logfile) unittest.TextTestRunner(verbosity=verbosity).run(recursive_suite()) + print('Logfile with Exscript and TestSuite activity: %s' % logfile) diff --git a/tests/Exscript/util/startTest.py b/tests/Exscript/util/startTest.py index 13b88ef5..b184fcd0 100644 --- a/tests/Exscript/util/startTest.py +++ b/tests/Exscript/util/startTest.py @@ -10,7 +10,7 @@ def count_calls(job, host, conn, data, **kwargs): - # Warning: Assertions raised in this function happen in a subprocess! + # Assertions in this function fail the job in its thread, not the test. assert kwargs.get('testarg') == 1 assert isinstance(conn, Exscript.protocols.Protocol) data.value += 1 diff --git a/tests/Exscript/util/syslogTest.py b/tests/Exscript/util/syslogTest.py index 8bfbff5b..ec72a55a 100644 --- a/tests/Exscript/util/syslogTest.py +++ b/tests/Exscript/util/syslogTest.py @@ -1,6 +1,5 @@ import sys import unittest -import re import os.path sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..', '..')) @@ -12,8 +11,8 @@ class castTest(unittest.TestCase): def testNetlog(self): from Exscript.util.syslog import netlog - # FIXME: dont really know how to test this; we'd need to know - # how to open a local syslog server. + # NOTE: No idea how to test netlog. + # How to open a local syslog server? Mocking? def suite(): diff --git a/tests/Exscript/util/ttyTest.py b/tests/Exscript/util/ttyTest.py index abc62fc1..33abeb0b 100644 --- a/tests/Exscript/util/ttyTest.py +++ b/tests/Exscript/util/ttyTest.py @@ -1,12 +1,13 @@ -from builtins import int +import os +import re import sys import unittest -import re -import os + sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..', '..')) import tempfile -from subprocess import Popen, PIPE +from subprocess import PIPE, Popen + import Exscript.util.tty diff --git a/tests/Exscript/util/urlTest.py b/tests/Exscript/util/urlTest.py index 97ef8c55..b9594625 100644 --- a/tests/Exscript/util/urlTest.py +++ b/tests/Exscript/util/urlTest.py @@ -1,8 +1,8 @@ -from builtins import str +import os.path +import re import sys import unittest -import re -import os.path + sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..', '..')) from Exscript.util.url import Url @@ -10,47 +10,47 @@ urls = [ # No protocol. ('testhost', - 'telnet://testhost:23'), + 'ssh://testhost:22'), ('testhost?myvar=testvalue', - 'telnet://testhost:23?myvar=testvalue'), + 'ssh://testhost:22?myvar=testvalue'), # No protocol + empty user. ('@testhost', - 'telnet://@testhost:23'), + 'ssh://@testhost:22'), ('@testhost?myvar=testvalue', - 'telnet://@testhost:23?myvar=testvalue'), + 'ssh://@testhost:22?myvar=testvalue'), # No protocol + user. ('user@testhost', - 'telnet://user@testhost:23'), + 'ssh://user@testhost:22'), ('user:password@testhost', - 'telnet://user:password@testhost:23'), + 'ssh://user:password@testhost:22'), ('user:password:password2@testhost', - 'telnet://user:password:password2@testhost:23'), + 'ssh://user:password:password2@testhost:22'), # No protocol + empty password 1. ('user:@testhost', - 'telnet://user:@testhost:23'), + 'ssh://user:@testhost:22'), ('user::password2@testhost', - 'telnet://user::password2@testhost:23'), + 'ssh://user::password2@testhost:22'), (':@testhost', - 'telnet://:@testhost:23'), + 'ssh://:@testhost:22'), # No protocol + empty password 2. ('user:password:@testhost', - 'telnet://user:password:@testhost:23'), + 'ssh://user:password:@testhost:22'), ('user::@testhost', - 'telnet://user::@testhost:23'), + 'ssh://user::@testhost:22'), ('::@testhost', - 'telnet://::@testhost:23'), + 'ssh://::@testhost:22'), (':password:@testhost', - 'telnet://:password:@testhost:23'), + 'ssh://:password:@testhost:22'), # Protocol. - ('ssh1://testhost', - 'ssh1://testhost:22'), - ('ssh1://testhost?myvar=testvalue', - 'ssh1://testhost:22?myvar=testvalue'), + ('telnet://testhost', + 'telnet://testhost:23'), + ('telnet://testhost?myvar=testvalue', + 'telnet://testhost:23?myvar=testvalue'), # Protocol + empty user. ('ssh://@testhost', @@ -81,8 +81,8 @@ 'ssh://user:password@testhost:22?myvar=testvalue&myvar2=test+2'), # Encoding. - ('foo://%27M%7B7Zk:%27%2FM%7B7Zyk:C7%26Rt%3Ea@ULM-SZRC1:23', - 'foo://%27M%7B7Zk:%27%2FM%7B7Zyk:C7%26Rt%3Ea@ULM-SZRC1:23'), + ('foo://%27M%7B7Zk:%27%2FM%7B7Zyk:C7%26Rt%3Ea@ULM-SZRC1:22', + 'foo://%27M%7B7Zk:%27%2FM%7B7Zyk:C7%26Rt%3Ea@ULM-SZRC1:22'), # Pseudo protocol. ('pseudo://../my/path', diff --git a/tests/Exscript/util/weakmethodTest.py b/tests/Exscript/util/weakmethodTest.py index 5eaec351..f2ef6f95 100644 --- a/tests/Exscript/util/weakmethodTest.py +++ b/tests/Exscript/util/weakmethodTest.py @@ -1,14 +1,14 @@ -from builtins import object +import os +import re import sys import unittest -import re -import os + sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..', '..')) -from Exscript.util.weakmethod import ref, WeakMethod, DeadMethodCalled +from Exscript.util.weakmethod import DeadMethodCalled, WeakMethod, ref -class TestClass(object): +class TestClass: def callback(self, *args, **kwargs): self.args = args diff --git a/tests/Exscript/workqueue/JobTest.py b/tests/Exscript/workqueue/JobTest.py index ea39ebcf..5e948928 100644 --- a/tests/Exscript/workqueue/JobTest.py +++ b/tests/Exscript/workqueue/JobTest.py @@ -1,16 +1,16 @@ -from future import standard_library -standard_library.install_aliases() -import sys -import unittest -import re import os.path +import re +import sys import threading +import unittest + sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..', '..')) from multiprocessing import Pipe -from tempfile import NamedTemporaryFile from pickle import dumps, loads -from Exscript.workqueue.job import Thread, Process, Job +from tempfile import NamedTemporaryFile + +from Exscript.workqueue.job import Job, Process, Thread def do_nothing(job): diff --git a/tests/Exscript/workqueue/MainLoopTest.py b/tests/Exscript/workqueue/MainLoopTest.py index f0e1fd71..2c5effe4 100644 --- a/tests/Exscript/workqueue/MainLoopTest.py +++ b/tests/Exscript/workqueue/MainLoopTest.py @@ -1,14 +1,14 @@ -from builtins import range -import sys -import unittest -import re import os.path +import re +import sys import threading +import unittest + sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..', '..')) +from Exscript.workqueue.job import Process from Exscript.workqueue.mainloop import MainLoop from Exscript.workqueue.pipeline import Pipeline -from Exscript.workqueue.job import Process class MainLoopTest(unittest.TestCase): diff --git a/tests/Exscript/workqueue/PipelineTest.py b/tests/Exscript/workqueue/PipelineTest.py index 534467d8..472c5ea0 100644 --- a/tests/Exscript/workqueue/PipelineTest.py +++ b/tests/Exscript/workqueue/PipelineTest.py @@ -1,17 +1,16 @@ -from future import standard_library -standard_library.install_aliases() -from builtins import next -import sys -import unittest -import re import os.path +import re +import sys import threading import time +import unittest + sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..', '..')) import time -from threading import Thread from multiprocessing import Value +from threading import Thread + from Exscript.workqueue import Pipeline diff --git a/tests/Exscript/workqueue/WorkQueueTest.py b/tests/Exscript/workqueue/WorkQueueTest.py index dde3aff5..4c15da43 100644 --- a/tests/Exscript/workqueue/WorkQueueTest.py +++ b/tests/Exscript/workqueue/WorkQueueTest.py @@ -1,15 +1,16 @@ -from builtins import range -import sys -import unittest -import re import os.path +import re +import sys import threading import time +import unittest + sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..', '..')) import random import time -from multiprocessing import Value, Lock +from multiprocessing import Lock, Value + from Exscript.workqueue import WorkQueue lock = Lock() @@ -98,7 +99,6 @@ def testEnqueueOrIgnore(self): # Stress testing from testEnqueue() not repeated here. def testPriorityEnqueue(self): - # Well, this test sucks. self.wq.pause() self.assertEqual(0, self.wq.get_length()) id = self.wq.priority_enqueue(nop) diff --git a/tests/Exscript/workqueue/run_suite.py b/tests/Exscript/workqueue/run_suite.py index d2e47d3b..0520fafd 100755 --- a/tests/Exscript/workqueue/run_suite.py +++ b/tests/Exscript/workqueue/run_suite.py @@ -1,13 +1,17 @@ -#!/usr/bin/env python -from __future__ import print_function +#!/usr/bin/env python3 +import fnmatch +import functools +import glob +import logging import os +import re import sys +import time import unittest -import glob -import fnmatch -import re -from inspect import isfunction, ismodule, isclass +from inspect import isclass, isfunction, ismodule +logger = logging.getLogger('TestSuite') +logger.addHandler(logging.NullHandler()) def uppercase(match): return match.group(1).upper() @@ -15,6 +19,75 @@ def uppercase(match): correlated = dict() +def setup_logging(filename=None): + logformat = "%(levelname)s: %(name)s (%(threadName)s) %(message)s" + loglevel = logging.DEBUG + logger_names = [ + "Exscript", + "Exscript.interpreter.parser", + "Exscript.parselib.lexer", + "Exscript.protocols", + "Exscript.servers", + # Netconf Client + "ncclient.capabilities", + "ncclient.devices", + "ncclient.manager", + "ncclient.operations", + "ncclient.transport.session", + "ncclient.transport.ssh", + # Paramiko + "paramiko.transport", + # Tests may also log. + "TestSuite", + ] + formatter = logging.Formatter(logformat) + formatter.converter = time.gmtime + if filename is None: + handler = logging.StreamHandler(sys.stderr) + else: + handler = logging.FileHandler(filename, mode="w", encoding="utf8") + handler.setFormatter(formatter) + handler.setLevel(loglevel) + for name in logger_names: + logger = logging.getLogger(name) + logger.setLevel(logging.DEBUG) + logger.addHandler(handler) + logger.propagate = False + + +def log_suites(suites): + for item in suites: + if isinstance(item, unittest.TestSuite): + log_suites(item) # suites are iterable + elif isinstance(item, unittest.TestCase): + testname = item.id() + method_names = ( + 'setUpClass', + 'setUp', + testname.split('.')[-1], + 'tearDown', + 'tearDownClass', + ) + for name in method_names: + method = getattr(item, name, None) + if not method: + continue + msg = 'Running %s (%s)' % (name, testname) + setattr(item, name, log_call(msg)(method)) + else: + raise TypeError("Unexpected type: %s" % type(item)) + + +def log_call(msg): + def decorator(func): + @functools.wraps(func) + def wrapper(*args, **kwargs): + logger.debug(msg) + return func(*args, **kwargs) + return wrapper + return decorator + + def correlate_class(theclass): """ Checks the given testcase for missing test methods. @@ -74,6 +147,7 @@ def find(dirname, pattern): def load_suite(files): modules = [os.path.splitext(f)[0] for f in files] + modules.sort() all_suites = [] for name in modules: name = name.lstrip('.').lstrip('/').replace('/', '.') @@ -83,6 +157,7 @@ def load_suite(files): if correlated: sys.stderr.write('Error: Untested methods found.\n') sys.exit(1) + log_suites(all_suites) return unittest.TestSuite(all_suites) @@ -107,4 +182,7 @@ def recursive_suite(): sys.exit(1) # Run. + logfile = os.path.splitext(__file__)[0] + '.log' + setup_logging(logfile) unittest.TextTestRunner(verbosity=verbosity).run(recursive_suite()) + print('Logfile with Exscript and TestSuite activity: %s' % logfile) diff --git a/tests/templates/builtinvars/test.exscript b/tests/templates/builtinvars/test.exscript index 65c9e551..06d39ebb 100644 --- a/tests/templates/builtinvars/test.exscript +++ b/tests/templates/builtinvars/test.exscript @@ -1,5 +1,5 @@ { fail "username: $__username__" if not __username__ matches /^\S+$/ fail "hostname: $__hostname__" if not __hostname__ matches /^\S+\/pseudodev\.py$/ - fail "hostname: $__filename__" if not __filename__ matches /^\S+\/test\.exscript$/ + fail "filename: $__filename__" if not __filename__ matches /^\S+\/test\.exscript$/ } diff --git a/tests/templates/iosdummy/expected b/tests/templates/iosdummy/expected index cf13d301..2d0d8c24 100644 --- a/tests/templates/iosdummy/expected +++ b/tests/templates/iosdummy/expected @@ -6,13 +6,58 @@ Escape character is '^]'. Unauthorized access prohibited! -dummy line 2 +dummy line 2 at 'dummy' port '/dev/vty0' from '12.34.56.78' Username: sab Password: dummy#enable +dummy#show version +Cisco Internetwork Operating System Software +IOS (tm) GS Software (C12KPRP-P-M), Version 12.0(32)SY6c, RELEASE SOFTWARE (fc3) +Technical Support: http://www.cisco.com/techsupport +Copyright (c) 1986-2008 by cisco Systems, Inc. +Compiled Mon 08-Sep-08 15:31 by leccese +Image text-base: 0x00010000, data-base: 0x055CD000 + +ROM: System Bootstrap, Version 12.0(20040128:214555) [assafb-PRP1P_20040101 1.8dev(2.83)] DEVELOPMENT SOFTWARE +BOOTLDR: GS Software (C12KPRP-P-M), Version 12.0(32)SY6c, RELEASE SOFTWARE (fc3) + + dummy uptime is 36 weeks, 1 day, 15 hours, 9 minutes +Uptime for this control processor is 36 weeks, 1 day, 14 hours, 30 minutes +System returned to ROM by reload at 03:32:54 MET Mon Feb 16 2009 +System restarted at 03:25:22 MET Tue Mar 10 2009 +System image file is "disk0:c12kprp-p-mz.120-32.SY6c.bin" + +cisco 12416/PRP (MPC7457) processor (revision 0x00) with 1048576K bytes of memory. +MPC7457 CPU at 1263Mhz, Rev 1.1, 512KB L2, 2048KB L3 Cache +Last reset from power-on +Channelized E1, Version 1.0. + +2 Route Processor Cards +2 Clock Scheduler Cards +3 Switch Fabric Cards +4 T1/E1 BITS controllers +1 Quad-port OC3c ATM controller (4 ATM). +2 16-port OC3 POS controllers (32 POS). +2 four-port OC12 POS controllers (8 POS). +2 twelve-port E3 controllers (24 E3). +1 Four Port Gigabit Ethernet/IEEE 802.3z controller (4 GigabitEthernet). +4 OC12 channelized to STS-12c/STM-4, STS-3c/STM-1 or DS-3/E3 controllers +4 ISE 10G SPA Interface Cards (12000-SIP-601) +3 Ethernet/IEEE 802.3 interface(s) +56 FastEthernet/IEEE 802.3 interface(s) +14 GigabitEthernet/IEEE 802.3 interface(s) +111 Serial network interface(s) +4 ATM network interface(s) +50 Packet over SONET network interface(s) +2043K bytes of non-volatile configuration memory. + +250880K bytes of ATA PCMCIA card at slot 0 (Sector size 512 bytes). +65536K bytes of Flash internal SIMM (Sector size 256K). +Configuration register is 0x2102 + dummy#show diag 10 SLOT 10 (RP/LC 0 ): 16 Port ISE Packet Over SONET OC-3c/STM-1 Single Mode/IR LC connector MAIN: type 79, 800-19733-08 rev A0 @@ -31,7 +76,7 @@ SLOT 10 (RP/LC 0 ): 16 Port ISE Packet Over SONET OC-3c/STM-1 Single Mode/IR LC ROM Monitor version 18.0 Fabric Downloader version used 7.1 (ROM version is 7.1) Primary clock is CSC 1 - Board is analyzed + Board is analyzed Board State is Line Card Enabled (IOS RUN ) Insertion time: 00:00:30 (36w1d ago) Processor Memory size: 1073741824 bytes @@ -39,17 +84,18 @@ SLOT 10 (RP/LC 0 ): 16 Port ISE Packet Over SONET OC-3c/STM-1 Single Mode/IR LC RX Packet Memory size: 268435456 bytes, Packet Memory pagesize: 16384 bytes 0 crashes since restart -dummy#show ip int brie Interface IP-Address OK? Method Status Protocol -Ethernet0 10.108.00.5 YES NVRAM up up -Ethernet1 unassigned YES unset administratively down down -Loopback0 10.108.200.5 YES NVRAM up up -Serial0 10.108.100.5 YES NVRAM up up -Serial1 10.108.40.5 YES NVRAM up up -Serial2 10.108.100.5 YES manual up up -Serial3 unassigned YES unset administratively down down +dummy#show ip int brie +Interface IP-Address OK? Method Status Protocol +Ethernet0 10.108.00.5 YES NVRAM up up +Ethernet1 unassigned YES unset administratively down down +Loopback0 10.108.200.5 YES NVRAM up up +Serial0 10.108.100.5 YES NVRAM up up +Serial1 10.108.40.5 YES NVRAM up up +Serial2 10.108.100.5 YES manual up up +Serial3 unassigned YES unset administratively down down dummy#show interface Ethernet0 -FastEthernet0/2 is administratively down, line protocol is down +FastEthernet0/2 is administratively down, line protocol is down Hardware is i82545, address is 0001.c9f4.c418 (bia 0001.c9f4.c418) MTU 1500 bytes, BW 100000 Kbit, DLY 100 usec, rely 255/255, load 1/255 Encapsulation ARPA, loopback not set @@ -75,7 +121,7 @@ FastEthernet0/2 is administratively down, line protocol is down 0 output buffer failures, 0 output buffers swapped out dummy#show interface Ethernet1 -FastEthernet0/2 is administratively down, line protocol is down +FastEthernet0/2 is administratively down, line protocol is down Hardware is i82545, address is 0001.c9f4.c418 (bia 0001.c9f4.c418) MTU 1500 bytes, BW 100000 Kbit, DLY 100 usec, rely 255/255, load 1/255 Encapsulation ARPA, loopback not set diff --git a/tests/templates/iosdummy/test.exscript b/tests/templates/iosdummy/test.exscript index 2c2659c2..c8033360 100644 --- a/tests/templates/iosdummy/test.exscript +++ b/tests/templates/iosdummy/test.exscript @@ -1,3 +1,5 @@ +show version + show diag $slot{ extract /Linecard\/Module:\s+(\S+)/ as type extract /^ (.+) memory size: (\d+), bytes/ as memtypes, memsizes diff --git a/tests/templates/netconf/expected b/tests/templates/netconf/expected new file mode 100644 index 00000000..39213ca3 --- /dev/null +++ b/tests/templates/netconf/expected @@ -0,0 +1,12 @@ +Welcome to dummy! +User: sab +Password: +dummy> + + abc + + +abc + +dummy> ! Data: abc +dummy> \ No newline at end of file diff --git a/tests/templates/netconf/pseudodev.py b/tests/templates/netconf/pseudodev.py new file mode 100644 index 00000000..cf771d73 --- /dev/null +++ b/tests/templates/netconf/pseudodev.py @@ -0,0 +1,15 @@ +# This test lacks real Netconf emulation but it's sufficient to test RPC parsing. +commands = ( +( +""" + + + abc + + +""".strip(), +""" +abc +""", +), +) diff --git a/tests/templates/netconf/test.exscript b/tests/templates/netconf/test.exscript new file mode 100644 index 00000000..efb186a4 --- /dev/null +++ b/tests/templates/netconf/test.exscript @@ -0,0 +1,11 @@ +# This test lacks real Netconf emulation but it's sufficient to test RPC parsing. + + + + abc + + + +{extract /([^<]*)<\/data>/ as test} +# Some CLI sugar... :D +! Data: $test diff --git a/tox.ini b/tox.ini index 3a68f4a2..e4c1d7e4 100644 --- a/tox.ini +++ b/tox.ini @@ -4,13 +4,14 @@ # and then run "tox" from this directory. [tox] -envlist = py27, py34 +envlist = py311, py312, py313, py314 [testenv] changedir = tests/Exscript commands = {envpython} ./run_suite.py 1 deps = - future configparser - paramiko - pycrypto + lxml + ncclient>=0.7,<1 + paramiko>=2.2,<5 + pycryptodomex diff --git a/version.sh b/version.sh index 93abf34e..011e23a9 100755 --- a/version.sh +++ b/version.sh @@ -1,50 +1,94 @@ #!/bin/sh # Tag revisions like this: # $ git tag -a -m "v0.2" v0.2 +# Supports also pre-/post-/dev-releases and local version identifiers: +# $ git tag -a -m "v0.2rc3.post5.dev7+local.1" v0.2rc3.post5.dev7+local.1 +# Enforcing canonical-like pattern with local version specifier: +# N.N(.N)?[{a|b|rc}N][.postN][.devN][+{a..z0..9}(.{a..z0..9})*] +# See https://packaging.python.org/en/latest/specifications/version-specifiers VERSION_IN=VERSION.in VERSION_FILE=Exscript/version.py +REGEX_LOCALSPEC='(\+[a-z0-9]+(\.[a-z0-9]+)?)' +REGEX_CANONICAL='((0|[1-9][0-9]*)\.(0|[1-9][0-9]*))(\.(0|[1-9][0-9]*))?((a|b|rc)(0|[1-9][0-9]*))?(\.post(0|[1-9][0-9]*))?(\.dev(0|[1-9][0-9]*))?'"${REGEX_LOCALSPEC}"'?' +REGEX_DESCRIBE='(\-([1-9][0-9]*)\-g[a-f0-9]+)' +ZERO_DESCRIBE='-0-g0000000' # Check that we are actually in a git managed project. -if [ ! -e .git -a -z "$1" ]; then +if [ ! -e .git ] && [ -z "$1" ]; then echo >&2 Not a git repository. exit 1 fi # Make sure that we have permission to modify the version file. -if [ -r $VERSION_FILE -a ! -w $VERSION_FILE ]; then - echo >&2 No permission to modify $VERSION_FILE. +if [ -r "${VERSION_FILE}" ] && [ ! -w "${VERSION_FILE}" ]; then + echo >&2 No permission to modify ${VERSION_FILE}. exit 1 fi -# By default, get the version number from "git describe". -if [ ! -z "$1" ]; then - VERSION=$1 +# If the --reset switch was given, reset the version number to 'DEVELOPMENT'. +if [ "$1" = "--reset" ]; then + VERSION='DEVELOPMENT' else - HEAD=`git log -1 --pretty=format:%H HEAD` - VERSION=`git describe $HEAD --tags --match "v[0-9]*" | sed 's/^v//;s/-[^\-]*$//;s/-/./' 2>/dev/null` - if [ -z "$VERSION" ]; then + # Use static version or get the version number from "git describe". + if [ -n "$1" ]; then + VERSION=$1 + else + HEAD=$(git log -1 --pretty=format:%H HEAD) + VERSION=$(git describe "${HEAD}" --tags --match "v[0-9]*") + fi + + # Remove leading 'v' as it is not allowed for canonical versioning. + VERSION=$(echo "${VERSION}" | sed -E 's/^v//') + + # Set implicit patch version to '.0' to always have a 'major.minor.patch' schema. + VERSION=$(echo "${VERSION}" | sed -E 's/^([0-9]+\.[0-9]+)(\.?[a-z]|\+[a-z0-9]|$)/\1.0\2/') + + # Extract delta count of commits from possible "describe" suffix: -3-g454be62 + # Afterwards remove the "describe" suffix from version. + DELTA=$(echo "${VERSION}" | (grep -Eio '('"${REGEX_DESCRIBE}"'$)' || echo "${ZERO_DESCRIBE}") | cut -f2 -d'-') + VERSION=$(echo "${VERSION}" | sed -E 's/'"${REGEX_DESCRIBE}"'$//') + + # Validations. + if [ -z "${VERSION}" ]; then echo >&2 No matching tag was found. exit 1 + elif ! (echo "${VERSION}" | grep -Eiq '^'"${REGEX_CANONICAL}"'$'); then + echo >&2 "Version '${VERSION}' does not match canonical-like pattern 'N.N(.N)?[{a|b|rc}N][.postN][.devN][+{a..z0..9}(.{a..z0..9})*]'." + echo >&2 "See also: https://packaging.python.org/en/latest/specifications/version-specifiers" + exit 1 fi -fi -# If the --reset switch was given, reset the version number to 'DEVELOPMENT'. -[ "$1" = "--reset" ] && VERSION='DEVELOPMENT' + # Extract local version specifier from version as his trailing number is optional. + LOCALSPEC=$(echo "${VERSION}" | grep -Eio '('"${REGEX_LOCALSPEC}"'$)') + VERSION=$(echo "${VERSION}" | sed -E 's/'"${REGEX_LOCALSPEC}"'$//') + + # Increment last number by commit delta. + if [ "${LOCALSPEC}" ] && (echo "${LOCALSPEC}" | grep -Eiq '[0-9]$'); then + INCREASED=$(( $(echo "${LOCALSPEC}" | grep -Eio '[0-9]+$') + DELTA )) + LOCALSPEC=$(echo "${LOCALSPEC}" | sed -E 's/[0-9]+$/'"${INCREASED}"'/') + else + INCREASED=$(( $(echo "${VERSION}" | grep -Eio '[0-9]+$') + DELTA )) + VERSION=$(echo "${VERSION}" | sed -E 's/[0-9]+$/'"${INCREASED}"'/') + fi + + # Restore full version string. + VERSION="${VERSION}${LOCALSPEC}" +fi # If there is no version file, we are already done. -echo Version is $VERSION -[ ! -r $VERSION_FILE ] && exit 0 +echo Version is "${VERSION}" +[ ! -r "${VERSION_FILE}" ] && exit 0 # Check whether the version file already contains this number, # and only touch it if there is a change to avoid changing # the timestamp. -VERSION_FILE_TMP=`mktemp` -cat $VERSION_IN | sed "s/@VERSION@/$VERSION/g" > $VERSION_FILE_TMP -if diff -q $VERSION_FILE_TMP $VERSION_FILE; then +VERSION_FILE_TMP=$(mktemp) +cat "${VERSION_IN}" | sed "s/@VERSION@/${VERSION}/g" > "${VERSION_FILE_TMP}" +if diff -q "${VERSION_FILE_TMP}" "${VERSION_FILE}"; then echo Version file unchanged. - rm $VERSION_FILE_TMP + rm "${VERSION_FILE_TMP}" exit 0 fi -mv $VERSION_FILE_TMP $VERSION_FILE +mv "${VERSION_FILE_TMP}" "${VERSION_FILE}" echo Version file updated. From 848dbac9dbb03643736498694bfd490a35d50961 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Fri, 30 Oct 2026 17:07:27 +0100 Subject: [PATCH 002/250] Add regression safety nets for the 3.0 modernization Known bugs become expected failures, so each fix must flip its test. A golden file of the OS-guessing results for the banner corpus shows any change of the detection as a diff, and tools/bench_bulk.py benchmarks the connection layer. --- tests/Exscript/KnownIssuesTest.py | 213 + .../Exscript/protocols/OsGuesserCorpusTest.py | 106 + .../Exscript/protocols/osguess-baseline.json | 3822 +++++++++++++++++ tools/bench_bulk.py | 180 + 4 files changed, 4321 insertions(+) create mode 100644 tests/Exscript/KnownIssuesTest.py create mode 100644 tests/Exscript/protocols/OsGuesserCorpusTest.py create mode 100644 tests/Exscript/protocols/osguess-baseline.json create mode 100755 tools/bench_bulk.py diff --git a/tests/Exscript/KnownIssuesTest.py b/tests/Exscript/KnownIssuesTest.py new file mode 100644 index 00000000..a8ea3661 --- /dev/null +++ b/tests/Exscript/KnownIssuesTest.py @@ -0,0 +1,213 @@ +""" +Regression net for known issues found during the 3.0 analysis. + +Every test describes the *correct* behavior and is marked as an expected +failure while the issue exists. Once a fix makes a test pass, unittest reports +an "unexpected success" and the suite fails, so the fix must remove the marker +and move the test next to the tests of the fixed module. +""" +import os +import socket +import sys +import tempfile +import threading +import time +import unittest +import uuid +from unittest import mock + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..')) + +import paramiko + +from Exscript import Account, PrivateKey +from Exscript.emulators import VirtualDevice +from Exscript.protocols import SSH2, Dummy, Telnet +from Exscript.protocols.drivers import driver_map, isam +from Exscript.servers import SSHd, Telnetd + +try: + from opentelemetry.sdk.trace import TracerProvider + from opentelemetry.sdk.trace.export import SimpleSpanProcessor + from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter + + from Exscript.contrib.otel import ExscriptInstrumentor +except ImportError: # the optional "otel" extra is not installed + ExscriptInstrumentor = None + +HOST = '127.0.0.1' +PASSWORD = 'S3cr3t-P@ss' +KEYFILE = os.path.join(os.path.dirname(__file__), 'protocols', 'id_rsa') + + +def free_port(): + """Returns a currently unused TCP port on the loopback interface.""" + with socket.socket() as sock: + sock.bind((HOST, 0)) + return sock.getsockname()[1] + + +class ServerMixin: + """Starts an Exscript test server for the test and stops it afterwards.""" + + def start_daemon(self, daemon_cls, device, **kwargs): + port = free_port() + daemon = daemon_cls(HOST, port, device, **kwargs) + daemon.start() + time.sleep(.2) + self.addCleanup(daemon.join) # cleanups run LIFO: exit() first + self.addCleanup(daemon.exit) + return port + + +class HostKeyTest(ServerMixin, unittest.TestCase): + + @unittest.expectedFailure # S1: known_hosts is only loaded after the check; no [host]:port lookup + def testKnownHostsMismatchIsRejected(self): + port = self.start_daemon(SSHd, VirtualDevice(HOST), key=PrivateKey.from_file(KEYFILE)) + other_key = paramiko.RSAKey.generate(2048) + home = tempfile.mkdtemp() + os.mkdir(os.path.join(home, '.ssh')) + with open(os.path.join(home, '.ssh', 'known_hosts'), 'w') as fp: + fp.write('[%s]:%d %s %s\n' % (HOST, port, other_key.get_name(), other_key.get_base64())) + + conn = SSH2(timeout=1, connect_rate_limit=0) + self.addCleanup(conn.close, force=True) + with mock.patch.dict(os.environ, {'HOME': home, 'USERPROFILE': home}): + with self.assertRaises(paramiko.BadHostKeyException): + conn.connect(HOST, port) + + +class ExpectTest(ServerMixin, unittest.TestCase): + """expect() must find a pattern regardless of how much data follows it.""" + + def setUp(self): + self.device = VirtualDevice(HOST, login_type=VirtualDevice.LOGIN_TYPE_NONE, strict=False) + self.device.add_command('show big', 'Please confirm: MARKER ' + 'x' * 2000) + + def expectMarker(self, conn, port=None): + self.addCleanup(conn.close, force=True) + conn.connect(HOST if port else 'dummy', port) + conn.protocol_authenticate(Account('user', 'password', needs_lock=False)) + conn.send('show big\r') + conn.expect('MARKER') + + def testDummy(self): + self.expectMarker(Dummy(device=self.device)) + + def testTelnet(self): + port = self.start_daemon(Telnetd, self.device) + self.expectMarker(Telnet(timeout=1, connect_rate_limit=0), port) + + @unittest.expectedFailure # SSH2 only searches the last 150 characters after each read + def testSSH2(self): + port = self.start_daemon(SSHd, self.device, key=PrivateKey.from_file(KEYFILE)) + conn = SSH2(timeout=1, connect_rate_limit=0, verify_fingerprint=False, + preferred_authentications=['password']) + self.expectMarker(conn, port) + + +class SecretHygieneTest(unittest.TestCase): + + @unittest.expectedFailure # S3: Protocol.send() logs passwords at debug level 4 + def testDebugLogRedactsPassword(self): + conn = Dummy(device=VirtualDevice('r1', echo=False, strict=False), debug=4) + with self.assertLogs('Exscript.protocols', level='DEBUG') as logs: + conn.connect('r1') + conn.login(Account('admin', PASSWORD, needs_lock=False)) + conn.close(force=True) + self.assertEqual([line for line in logs.output if PASSWORD in line], []) + + +@unittest.skipIf(ExscriptInstrumentor is None, 'requires the "otel" extra') +class OtelTest(unittest.TestCase): + + def setUp(self): + self.exporter = InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(self.exporter)) + instrumentor = ExscriptInstrumentor() + instrumentor.instrument(tracer_provider=provider, skip_dep_check=True) + self.addCleanup(instrumentor.uninstrument) + + def runSession(self, login): + device = VirtualDevice('r1', echo=False, strict=False) + device.add_command('show run', 'snmp-server community PRIVATE-COMMUNITY RW') + conn = Dummy(device=device) + conn.connect('r1') + login(conn, Account('admin', PASSWORD, needs_lock=False)) + conn.execute('show run') + conn.close(force=True) + + @unittest.expectedFailure # S2: the wrapper reads args[1] for app_account -> IndexError + def testLoginWithPositionalAccount(self): + self.runSession(lambda conn, account: conn.login(account)) + + @unittest.expectedFailure # S2: commands (incl. passwords) and responses are exported + def testNoSecretsOrDeviceOutputExportedByDefault(self): + self.runSession(lambda conn, account: conn.login(account=account)) + exported = [str(value) + for span in self.exporter.get_finished_spans() + for value in (span.attributes or {}).values()] + self.assertEqual([v for v in exported if PASSWORD in v or 'PRIVATE-COMMUNITY' in v], []) + + +class DriverTest(unittest.TestCase): + + @unittest.expectedFailure # assigned to self._error_re instead of self.error_re + def testIsamUsesItsErrorPatterns(self): + self.assertIs(driver_map['isam'].error_re, isam._error_re) + + @unittest.expectedFailure # any "Last login" line selects ericsson_ban, which never matches "$" + def testLastLoginLineDoesNotBreakLogin(self): + device = VirtualDevice('host', login_type=VirtualDevice.LOGIN_TYPE_NONE, strict=False, + banner='Welcome\nLast login: Mon Oct 26 10:00:00 2026 from 192.0.2.1\n') + device.set_prompt('user@host:~$ ') + device.add_command('uname', 'Linux') + conn = Dummy(device=device) + conn.connect('host') + conn.login(Account('user', 'password', needs_lock=False)) + conn.execute('uname') + self.assertEqual(conn.get_response().strip(), 'Linux') + + +class ConnectRateLimitTest(unittest.TestCase): + + def enforce(self, host, starts=None): + conn = SSH2(connect_rate_limit=60) # 1.1 s spacing per host + conn.host = host + conn._enforce_connect_rate_limit() + if starts is not None: + starts.append(time.monotonic()) + + @unittest.expectedFailure # the limiter sleeps while holding a process-wide lock + def testOtherHostsAreNotDelayed(self): + host = uuid.uuid4().hex + self.enforce(host + '-a') + waiting = threading.Thread(target=self.enforce, args=(host + '-a',)) + waiting.start() + time.sleep(.05) + start = time.monotonic() + self.enforce(host + '-b') + elapsed = time.monotonic() - start + waiting.join() + self.assertLess(elapsed, .5) + + @unittest.expectedFailure # concurrent connects to one host may start together + def testConcurrentConnectsToOneHostAreSpaced(self): + host = uuid.uuid4().hex + starts = [] + self.enforce(host, starts) + threads = [threading.Thread(target=self.enforce, args=(host, starts)) for _ in range(2)] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + starts.sort() + self.assertGreaterEqual(min(b - a for a, b in zip(starts, starts[1:])), 1.0) + + +def suite(): + return unittest.TestLoader().loadTestsFromModule(sys.modules[__name__]) +if __name__ == '__main__': + unittest.TextTestRunner(verbosity=2).run(suite()) diff --git a/tests/Exscript/protocols/OsGuesserCorpusTest.py b/tests/Exscript/protocols/OsGuesserCorpusTest.py new file mode 100644 index 00000000..da9051a9 --- /dev/null +++ b/tests/Exscript/protocols/OsGuesserCorpusTest.py @@ -0,0 +1,106 @@ +""" +Golden-file test for the OS guesser. + +Replays every banner of the corpus (``banners/``) through the guesser like a +session: protocol info first, the banner during authentication (one character +per read, the worst case), then generic command output and the last banner +line after authentication in different read sizes. The guessed OS and +confidence of every replay are compared with ``osguess-baseline.json``, so any +change of the guessing behavior shows up as a reviewable diff. + +After an intended change, regenerate the file with ``EXSCRIPT_UPDATE_GOLDEN=1`` +and review the diff. +""" +import json +import os +import sys +import unittest + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..', '..')) + +from Exscript.protocols.osguesser import OsGuesser + +DIRNAME = os.path.dirname(os.path.abspath(__file__)) +BANNER_DIR = os.path.join(DIRNAME, 'banners') +GOLDEN = os.path.join(DIRNAME, 'osguess-baseline.json') +PROTOCOL_INFOS = ['', 'SSH-2.0-Cisco-1.25', 'SSH-2.0-OpenSSH_9.6'] +READ_SIZES = [64, 65536] +GENERIC_OUTPUT = '\r\n'.join( + ['interface GigabitEthernet0/%d\r\n description uplink #%d to core\r\n' + ' ip address 10.0.%d.1 255.255.255.0\r\n!' % (i, i, i) for i in range(40)] + + ['# end of config', 'Building configuration...', 'Current configuration : 8231 bytes'] +) +PROMPT_ONLY = ['router#', 'switch>', 'RP/0/RSP0/CPU0:xr-pe1#', 'user@host:~$', 'root@host:~#', + '', 'A:sros-pe1#', '[admin@MikroTik] >', 'FGT60E #', 'core-sw01(config)#', + 'admin@junos-mx> '] + + +def feed(guesser, data, authenticated, size): + for i in range(0, len(data), size): + guesser.check_received_data(data[i:i + size], authenticated) + + +def snapshot(guesser): + return (dict(guesser._info), guesser._auth_buffer, + guesser._protocol_buffer, guesser._response_buffer) + + +def restore(state): + guesser = OsGuesser() + (guesser._info, guesser._auth_buffer, + guesser._protocol_buffer, guesser._response_buffer) = dict(state[0]), *state[1:] + return guesser + + +def replay(name, auth_data, protocol_info, after_auth, results): + """Runs the auth phase once, then the post-auth phase for every read size.""" + guesser = OsGuesser() + guesser.check_protocol_info(protocol_info) + feed(guesser, auth_data, False, 1) + state = snapshot(guesser) + for size in READ_SIZES: + guesser = restore(state) + feed(guesser, after_auth, True, size) + results['%s|%s|%d' % (name, protocol_info or '-', size)] = { + 'after_auth': list(state[0]['os']), + 'final': list(guesser._info['os']), + } + + +def run_corpus(): + results = {} + for name in sorted(os.listdir(BANNER_DIR)): + if name.startswith('.'): + continue + with open(os.path.join(BANNER_DIR, name), encoding='utf-8') as fp: + banner = fp.read().rstrip('\n') + last_line = banner.replace('\r', '\n').rstrip('\n').split('\n')[-1] + for info in PROTOCOL_INFOS: + replay(name, banner, info, '\r\n' + GENERIC_OUTPUT + '\r\n' + last_line, results) + for prompt in PROMPT_ONLY: + replay('prompt-only:' + prompt, 'Last login: Mon Oct 26 10:00:00 2026\r\n', + 'SSH-2.0-OpenSSH_9.6', '\r\n' + GENERIC_OUTPUT + '\r\n' + prompt, results) + return results + + +class OsGuesserCorpusTest(unittest.TestCase): + + def testCorpusMatchesBaseline(self): + results = run_corpus() + if os.environ.get('EXSCRIPT_UPDATE_GOLDEN'): + with open(GOLDEN, 'w', encoding='utf-8', newline='\n') as fp: + json.dump(results, fp, indent=1, sort_keys=True) + fp.write('\n') + self.skipTest('golden file regenerated: ' + GOLDEN) + with open(GOLDEN, encoding='utf-8') as fp: + expected = json.load(fp) + changed = sorted(key for key in expected.keys() | results.keys() + if expected.get(key) != results.get(key)) + details = ['%s: %s -> %s' % (key, expected.get(key), results.get(key)) for key in changed[:20]] + self.assertEqual(changed, [], 'OS guessing changed:\n' + '\n'.join(details)) + + +def suite(): + return unittest.TestLoader().loadTestsFromTestCase(OsGuesserCorpusTest) +if __name__ == '__main__': + unittest.TextTestRunner(verbosity=2).run(suite()) diff --git a/tests/Exscript/protocols/osguess-baseline.json b/tests/Exscript/protocols/osguess-baseline.json new file mode 100644 index 00000000..59e475af --- /dev/null +++ b/tests/Exscript/protocols/osguess-baseline.json @@ -0,0 +1,3822 @@ +{ + "ace.1|-|64": { + "after_auth": [ + "ace", + 90 + ], + "final": [ + "ace", + 90 + ] + }, + "ace.1|-|65536": { + "after_auth": [ + "ace", + 90 + ], + "final": [ + "ace", + 90 + ] + }, + "ace.1|SSH-2.0-Cisco-1.25|64": { + "after_auth": [ + "ace", + 90 + ], + "final": [ + "ace", + 90 + ] + }, + "ace.1|SSH-2.0-Cisco-1.25|65536": { + "after_auth": [ + "ace", + 90 + ], + "final": [ + "ace", + 90 + ] + }, + "ace.1|SSH-2.0-OpenSSH_9.6|64": { + "after_auth": [ + "ace", + 90 + ], + "final": [ + "ace", + 90 + ] + }, + "ace.1|SSH-2.0-OpenSSH_9.6|65536": { + "after_auth": [ + "ace", + 90 + ], + "final": [ + "ace", + 90 + ] + }, + "adtran.1|-|64": { + "after_auth": [ + "shell", + 20 + ], + "final": [ + "shell", + 20 + ] + }, + "adtran.1|-|65536": { + "after_auth": [ + "shell", + 20 + ], + "final": [ + "shell", + 20 + ] + }, + "adtran.1|SSH-2.0-Cisco-1.25|64": { + "after_auth": [ + "ios", + 80 + ], + "final": [ + "ios", + 80 + ] + }, + "adtran.1|SSH-2.0-Cisco-1.25|65536": { + "after_auth": [ + "ios", + 80 + ], + "final": [ + "ios", + 80 + ] + }, + "adtran.1|SSH-2.0-OpenSSH_9.6|64": { + "after_auth": [ + "shell", + 20 + ], + "final": [ + "shell", + 20 + ] + }, + "adtran.1|SSH-2.0-OpenSSH_9.6|65536": { + "after_auth": [ + "shell", + 20 + ], + "final": [ + "shell", + 20 + ] + }, + "adtran.2|-|64": { + "after_auth": [ + "ericsson_ban", + 20 + ], + "final": [ + "ericsson_ban", + 20 + ] + }, + "adtran.2|-|65536": { + "after_auth": [ + "ericsson_ban", + 20 + ], + "final": [ + "ericsson_ban", + 20 + ] + }, + "adtran.2|SSH-2.0-Cisco-1.25|64": { + "after_auth": [ + "ios", + 80 + ], + "final": [ + "ios", + 80 + ] + }, + "adtran.2|SSH-2.0-Cisco-1.25|65536": { + "after_auth": [ + "ios", + 80 + ], + "final": [ + "ios", + 80 + ] + }, + "adtran.2|SSH-2.0-OpenSSH_9.6|64": { + "after_auth": [ + "ericsson_ban", + 20 + ], + "final": [ + "ericsson_ban", + 20 + ] + }, + "adtran.2|SSH-2.0-OpenSSH_9.6|65536": { + "after_auth": [ + "ericsson_ban", + 20 + ], + "final": [ + "ericsson_ban", + 20 + ] + }, + "aironet.1|-|64": { + "after_auth": [ + "aironet", + 90 + ], + "final": [ + "aironet", + 90 + ] + }, + "aironet.1|-|65536": { + "after_auth": [ + "aironet", + 90 + ], + "final": [ + "aironet", + 90 + ] + }, + "aironet.1|SSH-2.0-Cisco-1.25|64": { + "after_auth": [ + "aironet", + 90 + ], + "final": [ + "aironet", + 90 + ] + }, + "aironet.1|SSH-2.0-Cisco-1.25|65536": { + "after_auth": [ + "aironet", + 90 + ], + "final": [ + "aironet", + 90 + ] + }, + "aironet.1|SSH-2.0-OpenSSH_9.6|64": { + "after_auth": [ + "aironet", + 90 + ], + "final": [ + "aironet", + 90 + ] + }, + "aironet.1|SSH-2.0-OpenSSH_9.6|65536": { + "after_auth": [ + "aironet", + 90 + ], + "final": [ + "aironet", + 90 + ] + }, + "aironet.2|-|64": { + "after_auth": [ + "aironet", + 90 + ], + "final": [ + "aironet", + 90 + ] + }, + "aironet.2|-|65536": { + "after_auth": [ + "aironet", + 90 + ], + "final": [ + "aironet", + 90 + ] + }, + "aironet.2|SSH-2.0-Cisco-1.25|64": { + "after_auth": [ + "aironet", + 90 + ], + "final": [ + "aironet", + 90 + ] + }, + "aironet.2|SSH-2.0-Cisco-1.25|65536": { + "after_auth": [ + "aironet", + 90 + ], + "final": [ + "aironet", + 90 + ] + }, + "aironet.2|SSH-2.0-OpenSSH_9.6|64": { + "after_auth": [ + "aironet", + 90 + ], + "final": [ + "aironet", + 90 + ] + }, + "aironet.2|SSH-2.0-OpenSSH_9.6|65536": { + "after_auth": [ + "aironet", + 90 + ], + "final": [ + "aironet", + 90 + ] + }, + "aix|-|64": { + "after_auth": [ + "aix", + 75 + ], + "final": [ + "aix", + 75 + ] + }, + "aix|-|65536": { + "after_auth": [ + "aix", + 75 + ], + "final": [ + "aix", + 75 + ] + }, + "aix|SSH-2.0-Cisco-1.25|64": { + "after_auth": [ + "ios", + 80 + ], + "final": [ + "ios", + 80 + ] + }, + "aix|SSH-2.0-Cisco-1.25|65536": { + "after_auth": [ + "ios", + 80 + ], + "final": [ + "ios", + 80 + ] + }, + "aix|SSH-2.0-OpenSSH_9.6|64": { + "after_auth": [ + "aix", + 75 + ], + "final": [ + "aix", + 75 + ] + }, + "aix|SSH-2.0-OpenSSH_9.6|65536": { + "after_auth": [ + "aix", + 75 + ], + "final": [ + "aix", + 75 + ] + }, + "arbor_peakflow.1|-|64": { + "after_auth": [ + "arbor_peakflow", + 97 + ], + "final": [ + "arbor_peakflow", + 97 + ] + }, + "arbor_peakflow.1|-|65536": { + "after_auth": [ + "arbor_peakflow", + 97 + ], + "final": [ + "arbor_peakflow", + 97 + ] + }, + "arbor_peakflow.1|SSH-2.0-Cisco-1.25|64": { + "after_auth": [ + "arbor_peakflow", + 97 + ], + "final": [ + "arbor_peakflow", + 97 + ] + }, + "arbor_peakflow.1|SSH-2.0-Cisco-1.25|65536": { + "after_auth": [ + "arbor_peakflow", + 97 + ], + "final": [ + "arbor_peakflow", + 97 + ] + }, + "arbor_peakflow.1|SSH-2.0-OpenSSH_9.6|64": { + "after_auth": [ + "arbor_peakflow", + 97 + ], + "final": [ + "arbor_peakflow", + 97 + ] + }, + "arbor_peakflow.1|SSH-2.0-OpenSSH_9.6|65536": { + "after_auth": [ + "arbor_peakflow", + 97 + ], + "final": [ + "arbor_peakflow", + 97 + ] + }, + "arbor_peakflow.2|-|64": { + "after_auth": [ + "arbor_peakflow", + 97 + ], + "final": [ + "arbor_peakflow", + 97 + ] + }, + "arbor_peakflow.2|-|65536": { + "after_auth": [ + "arbor_peakflow", + 97 + ], + "final": [ + "arbor_peakflow", + 97 + ] + }, + "arbor_peakflow.2|SSH-2.0-Cisco-1.25|64": { + "after_auth": [ + "arbor_peakflow", + 97 + ], + "final": [ + "arbor_peakflow", + 97 + ] + }, + "arbor_peakflow.2|SSH-2.0-Cisco-1.25|65536": { + "after_auth": [ + "arbor_peakflow", + 97 + ], + "final": [ + "arbor_peakflow", + 97 + ] + }, + "arbor_peakflow.2|SSH-2.0-OpenSSH_9.6|64": { + "after_auth": [ + "arbor_peakflow", + 97 + ], + "final": [ + "arbor_peakflow", + 97 + ] + }, + "arbor_peakflow.2|SSH-2.0-OpenSSH_9.6|65536": { + "after_auth": [ + "arbor_peakflow", + 97 + ], + "final": [ + "arbor_peakflow", + 97 + ] + }, + "arbor_peakflow.3|-|64": { + "after_auth": [ + "arbor_peakflow", + 97 + ], + "final": [ + "arbor_peakflow", + 97 + ] + }, + "arbor_peakflow.3|-|65536": { + "after_auth": [ + "arbor_peakflow", + 97 + ], + "final": [ + "arbor_peakflow", + 97 + ] + }, + "arbor_peakflow.3|SSH-2.0-Cisco-1.25|64": { + "after_auth": [ + "arbor_peakflow", + 97 + ], + "final": [ + "arbor_peakflow", + 97 + ] + }, + "arbor_peakflow.3|SSH-2.0-Cisco-1.25|65536": { + "after_auth": [ + "arbor_peakflow", + 97 + ], + "final": [ + "arbor_peakflow", + 97 + ] + }, + "arbor_peakflow.3|SSH-2.0-OpenSSH_9.6|64": { + "after_auth": [ + "arbor_peakflow", + 97 + ], + "final": [ + "arbor_peakflow", + 97 + ] + }, + "arbor_peakflow.3|SSH-2.0-OpenSSH_9.6|65536": { + "after_auth": [ + "arbor_peakflow", + 97 + ], + "final": [ + "arbor_peakflow", + 97 + ] + }, + "arbor_peakflow.4|-|64": { + "after_auth": [ + "arbor_peakflow", + 97 + ], + "final": [ + "arbor_peakflow", + 97 + ] + }, + "arbor_peakflow.4|-|65536": { + "after_auth": [ + "arbor_peakflow", + 97 + ], + "final": [ + "arbor_peakflow", + 97 + ] + }, + "arbor_peakflow.4|SSH-2.0-Cisco-1.25|64": { + "after_auth": [ + "arbor_peakflow", + 97 + ], + "final": [ + "arbor_peakflow", + 97 + ] + }, + "arbor_peakflow.4|SSH-2.0-Cisco-1.25|65536": { + "after_auth": [ + "arbor_peakflow", + 97 + ], + "final": [ + "arbor_peakflow", + 97 + ] + }, + "arbor_peakflow.4|SSH-2.0-OpenSSH_9.6|64": { + "after_auth": [ + "arbor_peakflow", + 97 + ], + "final": [ + "arbor_peakflow", + 97 + ] + }, + "arbor_peakflow.4|SSH-2.0-OpenSSH_9.6|65536": { + "after_auth": [ + "arbor_peakflow", + 97 + ], + "final": [ + "arbor_peakflow", + 97 + ] + }, + "arbor_peakflow.5|-|64": { + "after_auth": [ + "arbor_peakflow", + 97 + ], + "final": [ + "arbor_peakflow", + 97 + ] + }, + "arbor_peakflow.5|-|65536": { + "after_auth": [ + "arbor_peakflow", + 97 + ], + "final": [ + "arbor_peakflow", + 97 + ] + }, + "arbor_peakflow.5|SSH-2.0-Cisco-1.25|64": { + "after_auth": [ + "arbor_peakflow", + 97 + ], + "final": [ + "arbor_peakflow", + 97 + ] + }, + "arbor_peakflow.5|SSH-2.0-Cisco-1.25|65536": { + "after_auth": [ + "arbor_peakflow", + 97 + ], + "final": [ + "arbor_peakflow", + 97 + ] + }, + "arbor_peakflow.5|SSH-2.0-OpenSSH_9.6|64": { + "after_auth": [ + "arbor_peakflow", + 97 + ], + "final": [ + "arbor_peakflow", + 97 + ] + }, + "arbor_peakflow.5|SSH-2.0-OpenSSH_9.6|65536": { + "after_auth": [ + "arbor_peakflow", + 97 + ], + "final": [ + "arbor_peakflow", + 97 + ] + }, + "arbor_peakflow.6|-|64": { + "after_auth": [ + "arbor_peakflow", + 97 + ], + "final": [ + "arbor_peakflow", + 97 + ] + }, + "arbor_peakflow.6|-|65536": { + "after_auth": [ + "arbor_peakflow", + 97 + ], + "final": [ + "arbor_peakflow", + 97 + ] + }, + "arbor_peakflow.6|SSH-2.0-Cisco-1.25|64": { + "after_auth": [ + "arbor_peakflow", + 97 + ], + "final": [ + "arbor_peakflow", + 97 + ] + }, + "arbor_peakflow.6|SSH-2.0-Cisco-1.25|65536": { + "after_auth": [ + "arbor_peakflow", + 97 + ], + "final": [ + "arbor_peakflow", + 97 + ] + }, + "arbor_peakflow.6|SSH-2.0-OpenSSH_9.6|64": { + "after_auth": [ + "arbor_peakflow", + 97 + ], + "final": [ + "arbor_peakflow", + 97 + ] + }, + "arbor_peakflow.6|SSH-2.0-OpenSSH_9.6|65536": { + "after_auth": [ + "arbor_peakflow", + 97 + ], + "final": [ + "arbor_peakflow", + 97 + ] + }, + "aruba.1|-|64": { + "after_auth": [ + "aruba", + 88 + ], + "final": [ + "aruba", + 88 + ] + }, + "aruba.1|-|65536": { + "after_auth": [ + "aruba", + 88 + ], + "final": [ + "aruba", + 88 + ] + }, + "aruba.1|SSH-2.0-Cisco-1.25|64": { + "after_auth": [ + "aruba", + 88 + ], + "final": [ + "aruba", + 88 + ] + }, + "aruba.1|SSH-2.0-Cisco-1.25|65536": { + "after_auth": [ + "aruba", + 88 + ], + "final": [ + "aruba", + 88 + ] + }, + "aruba.1|SSH-2.0-OpenSSH_9.6|64": { + "after_auth": [ + "aruba", + 88 + ], + "final": [ + "aruba", + 88 + ] + }, + "aruba.1|SSH-2.0-OpenSSH_9.6|65536": { + "after_auth": [ + "aruba", + 88 + ], + "final": [ + "aruba", + 88 + ] + }, + "bigip.1|-|64": { + "after_auth": [ + "bigip", + 90 + ], + "final": [ + "bigip", + 90 + ] + }, + "bigip.1|-|65536": { + "after_auth": [ + "bigip", + 90 + ], + "final": [ + "bigip", + 90 + ] + }, + "bigip.1|SSH-2.0-Cisco-1.25|64": { + "after_auth": [ + "bigip", + 90 + ], + "final": [ + "bigip", + 90 + ] + }, + "bigip.1|SSH-2.0-Cisco-1.25|65536": { + "after_auth": [ + "bigip", + 90 + ], + "final": [ + "bigip", + 90 + ] + }, + "bigip.1|SSH-2.0-OpenSSH_9.6|64": { + "after_auth": [ + "bigip", + 90 + ], + "final": [ + "bigip", + 90 + ] + }, + "bigip.1|SSH-2.0-OpenSSH_9.6|65536": { + "after_auth": [ + "bigip", + 90 + ], + "final": [ + "bigip", + 90 + ] + }, + "brocade.1|-|64": { + "after_auth": [ + "brocade", + 90 + ], + "final": [ + "brocade", + 90 + ] + }, + "brocade.1|-|65536": { + "after_auth": [ + "brocade", + 90 + ], + "final": [ + "brocade", + 90 + ] + }, + "brocade.1|SSH-2.0-Cisco-1.25|64": { + "after_auth": [ + "brocade", + 90 + ], + "final": [ + "brocade", + 90 + ] + }, + "brocade.1|SSH-2.0-Cisco-1.25|65536": { + "after_auth": [ + "brocade", + 90 + ], + "final": [ + "brocade", + 90 + ] + }, + "brocade.1|SSH-2.0-OpenSSH_9.6|64": { + "after_auth": [ + "brocade", + 90 + ], + "final": [ + "brocade", + 90 + ] + }, + "brocade.1|SSH-2.0-OpenSSH_9.6|65536": { + "after_auth": [ + "brocade", + 90 + ], + "final": [ + "brocade", + 90 + ] + }, + "brocade.2|-|64": { + "after_auth": [ + "brocade", + 90 + ], + "final": [ + "brocade", + 90 + ] + }, + "brocade.2|-|65536": { + "after_auth": [ + "brocade", + 90 + ], + "final": [ + "brocade", + 90 + ] + }, + "brocade.2|SSH-2.0-Cisco-1.25|64": { + "after_auth": [ + "brocade", + 90 + ], + "final": [ + "brocade", + 90 + ] + }, + "brocade.2|SSH-2.0-Cisco-1.25|65536": { + "after_auth": [ + "brocade", + 90 + ], + "final": [ + "brocade", + 90 + ] + }, + "brocade.2|SSH-2.0-OpenSSH_9.6|64": { + "after_auth": [ + "brocade", + 90 + ], + "final": [ + "brocade", + 90 + ] + }, + "brocade.2|SSH-2.0-OpenSSH_9.6|65536": { + "after_auth": [ + "brocade", + 90 + ], + "final": [ + "brocade", + 90 + ] + }, + "brocade.3|-|64": { + "after_auth": [ + "brocade", + 90 + ], + "final": [ + "brocade", + 90 + ] + }, + "brocade.3|-|65536": { + "after_auth": [ + "brocade", + 90 + ], + "final": [ + "brocade", + 90 + ] + }, + "brocade.3|SSH-2.0-Cisco-1.25|64": { + "after_auth": [ + "brocade", + 90 + ], + "final": [ + "brocade", + 90 + ] + }, + "brocade.3|SSH-2.0-Cisco-1.25|65536": { + "after_auth": [ + "brocade", + 90 + ], + "final": [ + "brocade", + 90 + ] + }, + "brocade.3|SSH-2.0-OpenSSH_9.6|64": { + "after_auth": [ + "brocade", + 90 + ], + "final": [ + "brocade", + 90 + ] + }, + "brocade.3|SSH-2.0-OpenSSH_9.6|65536": { + "after_auth": [ + "brocade", + 90 + ], + "final": [ + "brocade", + 90 + ] + }, + "brocade.4|-|64": { + "after_auth": [ + "brocade", + 90 + ], + "final": [ + "brocade", + 90 + ] + }, + "brocade.4|-|65536": { + "after_auth": [ + "brocade", + 90 + ], + "final": [ + "brocade", + 90 + ] + }, + "brocade.4|SSH-2.0-Cisco-1.25|64": { + "after_auth": [ + "brocade", + 90 + ], + "final": [ + "brocade", + 90 + ] + }, + "brocade.4|SSH-2.0-Cisco-1.25|65536": { + "after_auth": [ + "brocade", + 90 + ], + "final": [ + "brocade", + 90 + ] + }, + "brocade.4|SSH-2.0-OpenSSH_9.6|64": { + "after_auth": [ + "brocade", + 90 + ], + "final": [ + "brocade", + 90 + ] + }, + "brocade.4|SSH-2.0-OpenSSH_9.6|65536": { + "after_auth": [ + "brocade", + 90 + ], + "final": [ + "brocade", + 90 + ] + }, + "cienasaos.1|-|64": { + "after_auth": [ + "cienasaos", + 90 + ], + "final": [ + "cienasaos", + 90 + ] + }, + "cienasaos.1|-|65536": { + "after_auth": [ + "cienasaos", + 90 + ], + "final": [ + "cienasaos", + 90 + ] + }, + "cienasaos.1|SSH-2.0-Cisco-1.25|64": { + "after_auth": [ + "cienasaos", + 90 + ], + "final": [ + "cienasaos", + 90 + ] + }, + "cienasaos.1|SSH-2.0-Cisco-1.25|65536": { + "after_auth": [ + "cienasaos", + 90 + ], + "final": [ + "cienasaos", + 90 + ] + }, + "cienasaos.1|SSH-2.0-OpenSSH_9.6|64": { + "after_auth": [ + "cienasaos", + 90 + ], + "final": [ + "cienasaos", + 90 + ] + }, + "cienasaos.1|SSH-2.0-OpenSSH_9.6|65536": { + "after_auth": [ + "cienasaos", + 90 + ], + "final": [ + "cienasaos", + 90 + ] + }, + "enterasys_wc|-|64": { + "after_auth": [ + "enterasys_wc", + 85 + ], + "final": [ + "enterasys_wc", + 85 + ] + }, + "enterasys_wc|-|65536": { + "after_auth": [ + "enterasys_wc", + 85 + ], + "final": [ + "enterasys_wc", + 85 + ] + }, + "enterasys_wc|SSH-2.0-Cisco-1.25|64": { + "after_auth": [ + "enterasys_wc", + 85 + ], + "final": [ + "enterasys_wc", + 85 + ] + }, + "enterasys_wc|SSH-2.0-Cisco-1.25|65536": { + "after_auth": [ + "enterasys_wc", + 85 + ], + "final": [ + "enterasys_wc", + 85 + ] + }, + "enterasys_wc|SSH-2.0-OpenSSH_9.6|64": { + "after_auth": [ + "enterasys_wc", + 85 + ], + "final": [ + "enterasys_wc", + 85 + ] + }, + "enterasys_wc|SSH-2.0-OpenSSH_9.6|65536": { + "after_auth": [ + "enterasys_wc", + 85 + ], + "final": [ + "enterasys_wc", + 85 + ] + }, + "enterasys|-|64": { + "after_auth": [ + "enterasys", + 80 + ], + "final": [ + "enterasys", + 80 + ] + }, + "enterasys|-|65536": { + "after_auth": [ + "enterasys", + 80 + ], + "final": [ + "enterasys", + 80 + ] + }, + "enterasys|SSH-2.0-Cisco-1.25|64": { + "after_auth": [ + "ios", + 80 + ], + "final": [ + "ios", + 80 + ] + }, + "enterasys|SSH-2.0-Cisco-1.25|65536": { + "after_auth": [ + "ios", + 80 + ], + "final": [ + "ios", + 80 + ] + }, + "enterasys|SSH-2.0-OpenSSH_9.6|64": { + "after_auth": [ + "enterasys", + 80 + ], + "final": [ + "enterasys", + 80 + ] + }, + "enterasys|SSH-2.0-OpenSSH_9.6|65536": { + "after_auth": [ + "enterasys", + 80 + ], + "final": [ + "enterasys", + 80 + ] + }, + "eos.1|-|64": { + "after_auth": [ + "junos", + 35 + ], + "final": [ + "junos", + 35 + ] + }, + "eos.1|-|65536": { + "after_auth": [ + "junos", + 35 + ], + "final": [ + "junos", + 35 + ] + }, + "eos.1|SSH-2.0-Cisco-1.25|64": { + "after_auth": [ + "ios", + 80 + ], + "final": [ + "ios", + 80 + ] + }, + "eos.1|SSH-2.0-Cisco-1.25|65536": { + "after_auth": [ + "ios", + 80 + ], + "final": [ + "ios", + 80 + ] + }, + "eos.1|SSH-2.0-OpenSSH_9.6|64": { + "after_auth": [ + "junos", + 35 + ], + "final": [ + "junos", + 35 + ] + }, + "eos.1|SSH-2.0-OpenSSH_9.6|65536": { + "after_auth": [ + "junos", + 35 + ], + "final": [ + "junos", + 35 + ] + }, + "eos.2|-|64": { + "after_auth": [ + "junos", + 35 + ], + "final": [ + "junos", + 35 + ] + }, + "eos.2|-|65536": { + "after_auth": [ + "junos", + 35 + ], + "final": [ + "junos", + 35 + ] + }, + "eos.2|SSH-2.0-Cisco-1.25|64": { + "after_auth": [ + "ios", + 80 + ], + "final": [ + "ios", + 80 + ] + }, + "eos.2|SSH-2.0-Cisco-1.25|65536": { + "after_auth": [ + "ios", + 80 + ], + "final": [ + "ios", + 80 + ] + }, + "eos.2|SSH-2.0-OpenSSH_9.6|64": { + "after_auth": [ + "junos", + 35 + ], + "final": [ + "junos", + 35 + ] + }, + "eos.2|SSH-2.0-OpenSSH_9.6|65536": { + "after_auth": [ + "junos", + 35 + ], + "final": [ + "junos", + 35 + ] + }, + "eos.3|-|64": { + "after_auth": [ + "junos", + 35 + ], + "final": [ + "junos", + 35 + ] + }, + "eos.3|-|65536": { + "after_auth": [ + "junos", + 35 + ], + "final": [ + "junos", + 35 + ] + }, + "eos.3|SSH-2.0-Cisco-1.25|64": { + "after_auth": [ + "ios", + 80 + ], + "final": [ + "ios", + 80 + ] + }, + "eos.3|SSH-2.0-Cisco-1.25|65536": { + "after_auth": [ + "ios", + 80 + ], + "final": [ + "ios", + 80 + ] + }, + "eos.3|SSH-2.0-OpenSSH_9.6|64": { + "after_auth": [ + "junos", + 35 + ], + "final": [ + "junos", + 35 + ] + }, + "eos.3|SSH-2.0-OpenSSH_9.6|65536": { + "after_auth": [ + "junos", + 35 + ], + "final": [ + "junos", + 35 + ] + }, + "eos.4|-|64": { + "after_auth": [ + "isam", + 20 + ], + "final": [ + "isam", + 20 + ] + }, + "eos.4|-|65536": { + "after_auth": [ + "isam", + 20 + ], + "final": [ + "isam", + 20 + ] + }, + "eos.4|SSH-2.0-Cisco-1.25|64": { + "after_auth": [ + "ios", + 80 + ], + "final": [ + "ios", + 80 + ] + }, + "eos.4|SSH-2.0-Cisco-1.25|65536": { + "after_auth": [ + "ios", + 80 + ], + "final": [ + "ios", + 80 + ] + }, + "eos.4|SSH-2.0-OpenSSH_9.6|64": { + "after_auth": [ + "isam", + 20 + ], + "final": [ + "isam", + 20 + ] + }, + "eos.4|SSH-2.0-OpenSSH_9.6|65536": { + "after_auth": [ + "isam", + 20 + ], + "final": [ + "isam", + 20 + ] + }, + "eos.5|-|64": { + "after_auth": [ + "junos", + 35 + ], + "final": [ + "junos", + 35 + ] + }, + "eos.5|-|65536": { + "after_auth": [ + "junos", + 35 + ], + "final": [ + "junos", + 35 + ] + }, + "eos.5|SSH-2.0-Cisco-1.25|64": { + "after_auth": [ + "ios", + 80 + ], + "final": [ + "ios", + 80 + ] + }, + "eos.5|SSH-2.0-Cisco-1.25|65536": { + "after_auth": [ + "ios", + 80 + ], + "final": [ + "ios", + 80 + ] + }, + "eos.5|SSH-2.0-OpenSSH_9.6|64": { + "after_auth": [ + "junos", + 35 + ], + "final": [ + "junos", + 35 + ] + }, + "eos.5|SSH-2.0-OpenSSH_9.6|65536": { + "after_auth": [ + "junos", + 35 + ], + "final": [ + "junos", + 35 + ] + }, + "ericsson_ban|-|64": { + "after_auth": [ + "ericsson_ban", + 90 + ], + "final": [ + "ericsson_ban", + 90 + ] + }, + "ericsson_ban|-|65536": { + "after_auth": [ + "ericsson_ban", + 90 + ], + "final": [ + "ericsson_ban", + 90 + ] + }, + "ericsson_ban|SSH-2.0-Cisco-1.25|64": { + "after_auth": [ + "ericsson_ban", + 90 + ], + "final": [ + "ericsson_ban", + 90 + ] + }, + "ericsson_ban|SSH-2.0-Cisco-1.25|65536": { + "after_auth": [ + "ericsson_ban", + 90 + ], + "final": [ + "ericsson_ban", + 90 + ] + }, + "ericsson_ban|SSH-2.0-OpenSSH_9.6|64": { + "after_auth": [ + "ericsson_ban", + 90 + ], + "final": [ + "ericsson_ban", + 90 + ] + }, + "ericsson_ban|SSH-2.0-OpenSSH_9.6|65536": { + "after_auth": [ + "ericsson_ban", + 90 + ], + "final": [ + "ericsson_ban", + 90 + ] + }, + "fortios.1|-|64": { + "after_auth": [ + "fortios", + 50 + ], + "final": [ + "fortios", + 50 + ] + }, + "fortios.1|-|65536": { + "after_auth": [ + "fortios", + 50 + ], + "final": [ + "fortios", + 50 + ] + }, + "fortios.1|SSH-2.0-Cisco-1.25|64": { + "after_auth": [ + "ios", + 80 + ], + "final": [ + "ios", + 80 + ] + }, + "fortios.1|SSH-2.0-Cisco-1.25|65536": { + "after_auth": [ + "ios", + 80 + ], + "final": [ + "ios", + 80 + ] + }, + "fortios.1|SSH-2.0-OpenSSH_9.6|64": { + "after_auth": [ + "fortios", + 50 + ], + "final": [ + "fortios", + 50 + ] + }, + "fortios.1|SSH-2.0-OpenSSH_9.6|65536": { + "after_auth": [ + "fortios", + 50 + ], + "final": [ + "fortios", + 50 + ] + }, + "fortios.2|-|64": { + "after_auth": [ + "fortios", + 50 + ], + "final": [ + "fortios", + 50 + ] + }, + "fortios.2|-|65536": { + "after_auth": [ + "fortios", + 50 + ], + "final": [ + "fortios", + 50 + ] + }, + "fortios.2|SSH-2.0-Cisco-1.25|64": { + "after_auth": [ + "ios", + 80 + ], + "final": [ + "ios", + 80 + ] + }, + "fortios.2|SSH-2.0-Cisco-1.25|65536": { + "after_auth": [ + "ios", + 80 + ], + "final": [ + "ios", + 80 + ] + }, + "fortios.2|SSH-2.0-OpenSSH_9.6|64": { + "after_auth": [ + "fortios", + 50 + ], + "final": [ + "fortios", + 50 + ] + }, + "fortios.2|SSH-2.0-OpenSSH_9.6|65536": { + "after_auth": [ + "fortios", + 50 + ], + "final": [ + "fortios", + 50 + ] + }, + "hp_pro_curve.1|-|64": { + "after_auth": [ + "hp_pro_curve", + 95 + ], + "final": [ + "hp_pro_curve", + 95 + ] + }, + "hp_pro_curve.1|-|65536": { + "after_auth": [ + "hp_pro_curve", + 95 + ], + "final": [ + "hp_pro_curve", + 95 + ] + }, + "hp_pro_curve.1|SSH-2.0-Cisco-1.25|64": { + "after_auth": [ + "hp_pro_curve", + 95 + ], + "final": [ + "hp_pro_curve", + 95 + ] + }, + "hp_pro_curve.1|SSH-2.0-Cisco-1.25|65536": { + "after_auth": [ + "hp_pro_curve", + 95 + ], + "final": [ + "hp_pro_curve", + 95 + ] + }, + "hp_pro_curve.1|SSH-2.0-OpenSSH_9.6|64": { + "after_auth": [ + "hp_pro_curve", + 95 + ], + "final": [ + "hp_pro_curve", + 95 + ] + }, + "hp_pro_curve.1|SSH-2.0-OpenSSH_9.6|65536": { + "after_auth": [ + "hp_pro_curve", + 95 + ], + "final": [ + "hp_pro_curve", + 95 + ] + }, + "hp_pro_curve.2|-|64": { + "after_auth": [ + "hp_pro_curve", + 95 + ], + "final": [ + "hp_pro_curve", + 95 + ] + }, + "hp_pro_curve.2|-|65536": { + "after_auth": [ + "hp_pro_curve", + 95 + ], + "final": [ + "hp_pro_curve", + 95 + ] + }, + "hp_pro_curve.2|SSH-2.0-Cisco-1.25|64": { + "after_auth": [ + "hp_pro_curve", + 95 + ], + "final": [ + "hp_pro_curve", + 95 + ] + }, + "hp_pro_curve.2|SSH-2.0-Cisco-1.25|65536": { + "after_auth": [ + "hp_pro_curve", + 95 + ], + "final": [ + "hp_pro_curve", + 95 + ] + }, + "hp_pro_curve.2|SSH-2.0-OpenSSH_9.6|64": { + "after_auth": [ + "hp_pro_curve", + 95 + ], + "final": [ + "hp_pro_curve", + 95 + ] + }, + "hp_pro_curve.2|SSH-2.0-OpenSSH_9.6|65536": { + "after_auth": [ + "hp_pro_curve", + 95 + ], + "final": [ + "hp_pro_curve", + 95 + ] + }, + "icotera|-|64": { + "after_auth": [ + "icotera", + 80 + ], + "final": [ + "icotera", + 80 + ] + }, + "icotera|-|65536": { + "after_auth": [ + "icotera", + 80 + ], + "final": [ + "icotera", + 80 + ] + }, + "icotera|SSH-2.0-Cisco-1.25|64": { + "after_auth": [ + "ios", + 80 + ], + "final": [ + "ios", + 80 + ] + }, + "icotera|SSH-2.0-Cisco-1.25|65536": { + "after_auth": [ + "ios", + 80 + ], + "final": [ + "ios", + 80 + ] + }, + "icotera|SSH-2.0-OpenSSH_9.6|64": { + "after_auth": [ + "icotera", + 80 + ], + "final": [ + "icotera", + 80 + ] + }, + "icotera|SSH-2.0-OpenSSH_9.6|65536": { + "after_auth": [ + "icotera", + 80 + ], + "final": [ + "icotera", + 80 + ] + }, + "ios.1|-|64": { + "after_auth": [ + "ios", + 60 + ], + "final": [ + "ios", + 60 + ] + }, + "ios.1|-|65536": { + "after_auth": [ + "ios", + 60 + ], + "final": [ + "ios", + 60 + ] + }, + "ios.1|SSH-2.0-Cisco-1.25|64": { + "after_auth": [ + "ios", + 80 + ], + "final": [ + "ios", + 80 + ] + }, + "ios.1|SSH-2.0-Cisco-1.25|65536": { + "after_auth": [ + "ios", + 80 + ], + "final": [ + "ios", + 80 + ] + }, + "ios.1|SSH-2.0-OpenSSH_9.6|64": { + "after_auth": [ + "ios", + 60 + ], + "final": [ + "ios", + 60 + ] + }, + "ios.1|SSH-2.0-OpenSSH_9.6|65536": { + "after_auth": [ + "ios", + 60 + ], + "final": [ + "ios", + 60 + ] + }, + "ios.2|-|64": { + "after_auth": [ + "ios", + 30 + ], + "final": [ + "ios", + 30 + ] + }, + "ios.2|-|65536": { + "after_auth": [ + "ios", + 30 + ], + "final": [ + "ios", + 30 + ] + }, + "ios.2|SSH-2.0-Cisco-1.25|64": { + "after_auth": [ + "ios", + 80 + ], + "final": [ + "ios", + 80 + ] + }, + "ios.2|SSH-2.0-Cisco-1.25|65536": { + "after_auth": [ + "ios", + 80 + ], + "final": [ + "ios", + 80 + ] + }, + "ios.2|SSH-2.0-OpenSSH_9.6|64": { + "after_auth": [ + "ios", + 30 + ], + "final": [ + "ios", + 30 + ] + }, + "ios.2|SSH-2.0-OpenSSH_9.6|65536": { + "after_auth": [ + "ios", + 30 + ], + "final": [ + "ios", + 30 + ] + }, + "ios.3|-|64": { + "after_auth": [ + "ios", + 50 + ], + "final": [ + "ios", + 50 + ] + }, + "ios.3|-|65536": { + "after_auth": [ + "ios", + 50 + ], + "final": [ + "ios", + 50 + ] + }, + "ios.3|SSH-2.0-Cisco-1.25|64": { + "after_auth": [ + "ios", + 80 + ], + "final": [ + "ios", + 80 + ] + }, + "ios.3|SSH-2.0-Cisco-1.25|65536": { + "after_auth": [ + "ios", + 80 + ], + "final": [ + "ios", + 80 + ] + }, + "ios.3|SSH-2.0-OpenSSH_9.6|64": { + "after_auth": [ + "ios", + 50 + ], + "final": [ + "ios", + 50 + ] + }, + "ios.3|SSH-2.0-OpenSSH_9.6|65536": { + "after_auth": [ + "ios", + 50 + ], + "final": [ + "ios", + 50 + ] + }, + "ios.4|-|64": { + "after_auth": [ + "ios", + 50 + ], + "final": [ + "ios", + 50 + ] + }, + "ios.4|-|65536": { + "after_auth": [ + "ios", + 50 + ], + "final": [ + "ios", + 50 + ] + }, + "ios.4|SSH-2.0-Cisco-1.25|64": { + "after_auth": [ + "ios", + 80 + ], + "final": [ + "ios", + 80 + ] + }, + "ios.4|SSH-2.0-Cisco-1.25|65536": { + "after_auth": [ + "ios", + 80 + ], + "final": [ + "ios", + 80 + ] + }, + "ios.4|SSH-2.0-OpenSSH_9.6|64": { + "after_auth": [ + "ios", + 50 + ], + "final": [ + "ios", + 50 + ] + }, + "ios.4|SSH-2.0-OpenSSH_9.6|65536": { + "after_auth": [ + "ios", + 50 + ], + "final": [ + "ios", + 50 + ] + }, + "ios.5|-|64": { + "after_auth": [ + "ios", + 60 + ], + "final": [ + "ios", + 60 + ] + }, + "ios.5|-|65536": { + "after_auth": [ + "ios", + 60 + ], + "final": [ + "ios", + 60 + ] + }, + "ios.5|SSH-2.0-Cisco-1.25|64": { + "after_auth": [ + "ios", + 80 + ], + "final": [ + "ios", + 80 + ] + }, + "ios.5|SSH-2.0-Cisco-1.25|65536": { + "after_auth": [ + "ios", + 80 + ], + "final": [ + "ios", + 80 + ] + }, + "ios.5|SSH-2.0-OpenSSH_9.6|64": { + "after_auth": [ + "ios", + 60 + ], + "final": [ + "ios", + 60 + ] + }, + "ios.5|SSH-2.0-OpenSSH_9.6|65536": { + "after_auth": [ + "ios", + 60 + ], + "final": [ + "ios", + 60 + ] + }, + "ios.6|-|64": { + "after_auth": [ + "ios", + 50 + ], + "final": [ + "ios", + 50 + ] + }, + "ios.6|-|65536": { + "after_auth": [ + "ios", + 50 + ], + "final": [ + "ios", + 50 + ] + }, + "ios.6|SSH-2.0-Cisco-1.25|64": { + "after_auth": [ + "ios", + 80 + ], + "final": [ + "ios", + 80 + ] + }, + "ios.6|SSH-2.0-Cisco-1.25|65536": { + "after_auth": [ + "ios", + 80 + ], + "final": [ + "ios", + 80 + ] + }, + "ios.6|SSH-2.0-OpenSSH_9.6|64": { + "after_auth": [ + "ios", + 50 + ], + "final": [ + "ios", + 50 + ] + }, + "ios.6|SSH-2.0-OpenSSH_9.6|65536": { + "after_auth": [ + "ios", + 50 + ], + "final": [ + "ios", + 50 + ] + }, + "ios_xr|-|64": { + "after_auth": [ + "ios_xr", + 95 + ], + "final": [ + "ios_xr", + 95 + ] + }, + "ios_xr|-|65536": { + "after_auth": [ + "ios_xr", + 95 + ], + "final": [ + "ios_xr", + 95 + ] + }, + "ios_xr|SSH-2.0-Cisco-1.25|64": { + "after_auth": [ + "ios_xr", + 95 + ], + "final": [ + "ios_xr", + 95 + ] + }, + "ios_xr|SSH-2.0-Cisco-1.25|65536": { + "after_auth": [ + "ios_xr", + 95 + ], + "final": [ + "ios_xr", + 95 + ] + }, + "ios_xr|SSH-2.0-OpenSSH_9.6|64": { + "after_auth": [ + "ios_xr", + 95 + ], + "final": [ + "ios_xr", + 95 + ] + }, + "ios_xr|SSH-2.0-OpenSSH_9.6|65536": { + "after_auth": [ + "ios_xr", + 95 + ], + "final": [ + "ios_xr", + 95 + ] + }, + "isam|-|64": { + "after_auth": [ + "isam", + 90 + ], + "final": [ + "isam", + 90 + ] + }, + "isam|-|65536": { + "after_auth": [ + "isam", + 90 + ], + "final": [ + "isam", + 90 + ] + }, + "isam|SSH-2.0-Cisco-1.25|64": { + "after_auth": [ + "isam", + 90 + ], + "final": [ + "isam", + 90 + ] + }, + "isam|SSH-2.0-Cisco-1.25|65536": { + "after_auth": [ + "isam", + 90 + ], + "final": [ + "isam", + 90 + ] + }, + "isam|SSH-2.0-OpenSSH_9.6|64": { + "after_auth": [ + "isam", + 90 + ], + "final": [ + "isam", + 90 + ] + }, + "isam|SSH-2.0-OpenSSH_9.6|65536": { + "after_auth": [ + "isam", + 90 + ], + "final": [ + "isam", + 90 + ] + }, + "junos.1|-|64": { + "after_auth": [ + "junos", + 80 + ], + "final": [ + "junos", + 80 + ] + }, + "junos.1|-|65536": { + "after_auth": [ + "junos", + 80 + ], + "final": [ + "junos", + 80 + ] + }, + "junos.1|SSH-2.0-Cisco-1.25|64": { + "after_auth": [ + "ios", + 80 + ], + "final": [ + "ios", + 80 + ] + }, + "junos.1|SSH-2.0-Cisco-1.25|65536": { + "after_auth": [ + "ios", + 80 + ], + "final": [ + "ios", + 80 + ] + }, + "junos.1|SSH-2.0-OpenSSH_9.6|64": { + "after_auth": [ + "junos", + 80 + ], + "final": [ + "junos", + 80 + ] + }, + "junos.1|SSH-2.0-OpenSSH_9.6|65536": { + "after_auth": [ + "junos", + 80 + ], + "final": [ + "junos", + 80 + ] + }, + "junos.2|-|64": { + "after_auth": [ + "junos", + 80 + ], + "final": [ + "junos", + 80 + ] + }, + "junos.2|-|65536": { + "after_auth": [ + "junos", + 80 + ], + "final": [ + "junos", + 80 + ] + }, + "junos.2|SSH-2.0-Cisco-1.25|64": { + "after_auth": [ + "ios", + 80 + ], + "final": [ + "ios", + 80 + ] + }, + "junos.2|SSH-2.0-Cisco-1.25|65536": { + "after_auth": [ + "ios", + 80 + ], + "final": [ + "ios", + 80 + ] + }, + "junos.2|SSH-2.0-OpenSSH_9.6|64": { + "after_auth": [ + "junos", + 80 + ], + "final": [ + "junos", + 80 + ] + }, + "junos.2|SSH-2.0-OpenSSH_9.6|65536": { + "after_auth": [ + "junos", + 80 + ], + "final": [ + "junos", + 80 + ] + }, + "junos.3|-|64": { + "after_auth": [ + "junos", + 80 + ], + "final": [ + "junos", + 80 + ] + }, + "junos.3|-|65536": { + "after_auth": [ + "junos", + 80 + ], + "final": [ + "junos", + 80 + ] + }, + "junos.3|SSH-2.0-Cisco-1.25|64": { + "after_auth": [ + "ios", + 80 + ], + "final": [ + "ios", + 80 + ] + }, + "junos.3|SSH-2.0-Cisco-1.25|65536": { + "after_auth": [ + "ios", + 80 + ], + "final": [ + "ios", + 80 + ] + }, + "junos.3|SSH-2.0-OpenSSH_9.6|64": { + "after_auth": [ + "junos", + 80 + ], + "final": [ + "junos", + 80 + ] + }, + "junos.3|SSH-2.0-OpenSSH_9.6|65536": { + "after_auth": [ + "junos", + 80 + ], + "final": [ + "junos", + 80 + ] + }, + "junos_erx|-|64": { + "after_auth": [ + "junos_erx", + 75 + ], + "final": [ + "junos_erx", + 75 + ] + }, + "junos_erx|-|65536": { + "after_auth": [ + "junos_erx", + 75 + ], + "final": [ + "junos_erx", + 75 + ] + }, + "junos_erx|SSH-2.0-Cisco-1.25|64": { + "after_auth": [ + "ios", + 80 + ], + "final": [ + "ios", + 80 + ] + }, + "junos_erx|SSH-2.0-Cisco-1.25|65536": { + "after_auth": [ + "ios", + 80 + ], + "final": [ + "ios", + 80 + ] + }, + "junos_erx|SSH-2.0-OpenSSH_9.6|64": { + "after_auth": [ + "junos_erx", + 75 + ], + "final": [ + "junos_erx", + 75 + ] + }, + "junos_erx|SSH-2.0-OpenSSH_9.6|65536": { + "after_auth": [ + "junos_erx", + 75 + ], + "final": [ + "junos_erx", + 75 + ] + }, + "mrv.1|-|64": { + "after_auth": [ + "mrv", + 75 + ], + "final": [ + "mrv", + 75 + ] + }, + "mrv.1|-|65536": { + "after_auth": [ + "mrv", + 75 + ], + "final": [ + "mrv", + 75 + ] + }, + "mrv.1|SSH-2.0-Cisco-1.25|64": { + "after_auth": [ + "ios", + 80 + ], + "final": [ + "ios", + 80 + ] + }, + "mrv.1|SSH-2.0-Cisco-1.25|65536": { + "after_auth": [ + "ios", + 80 + ], + "final": [ + "ios", + 80 + ] + }, + "mrv.1|SSH-2.0-OpenSSH_9.6|64": { + "after_auth": [ + "mrv", + 75 + ], + "final": [ + "mrv", + 75 + ] + }, + "mrv.1|SSH-2.0-OpenSSH_9.6|65536": { + "after_auth": [ + "mrv", + 75 + ], + "final": [ + "mrv", + 75 + ] + }, + "mrv.2|-|64": { + "after_auth": [ + "mrv", + 75 + ], + "final": [ + "mrv", + 75 + ] + }, + "mrv.2|-|65536": { + "after_auth": [ + "mrv", + 75 + ], + "final": [ + "mrv", + 75 + ] + }, + "mrv.2|SSH-2.0-Cisco-1.25|64": { + "after_auth": [ + "ios", + 80 + ], + "final": [ + "ios", + 80 + ] + }, + "mrv.2|SSH-2.0-Cisco-1.25|65536": { + "after_auth": [ + "ios", + 80 + ], + "final": [ + "ios", + 80 + ] + }, + "mrv.2|SSH-2.0-OpenSSH_9.6|64": { + "after_auth": [ + "mrv", + 75 + ], + "final": [ + "mrv", + 75 + ] + }, + "mrv.2|SSH-2.0-OpenSSH_9.6|65536": { + "after_auth": [ + "mrv", + 75 + ], + "final": [ + "mrv", + 75 + ] + }, + "nxos.1|-|64": { + "after_auth": [ + "nxos", + 95 + ], + "final": [ + "nxos", + 95 + ] + }, + "nxos.1|-|65536": { + "after_auth": [ + "nxos", + 95 + ], + "final": [ + "nxos", + 95 + ] + }, + "nxos.1|SSH-2.0-Cisco-1.25|64": { + "after_auth": [ + "nxos", + 95 + ], + "final": [ + "nxos", + 95 + ] + }, + "nxos.1|SSH-2.0-Cisco-1.25|65536": { + "after_auth": [ + "nxos", + 95 + ], + "final": [ + "nxos", + 95 + ] + }, + "nxos.1|SSH-2.0-OpenSSH_9.6|64": { + "after_auth": [ + "nxos", + 95 + ], + "final": [ + "nxos", + 95 + ] + }, + "nxos.1|SSH-2.0-OpenSSH_9.6|65536": { + "after_auth": [ + "nxos", + 95 + ], + "final": [ + "nxos", + 95 + ] + }, + "one_os.1|-|64": { + "after_auth": [ + "one_os", + 40 + ], + "final": [ + "one_os", + 40 + ] + }, + "one_os.1|-|65536": { + "after_auth": [ + "one_os", + 40 + ], + "final": [ + "one_os", + 40 + ] + }, + "one_os.1|SSH-2.0-Cisco-1.25|64": { + "after_auth": [ + "ios", + 80 + ], + "final": [ + "ios", + 80 + ] + }, + "one_os.1|SSH-2.0-Cisco-1.25|65536": { + "after_auth": [ + "ios", + 80 + ], + "final": [ + "ios", + 80 + ] + }, + "one_os.1|SSH-2.0-OpenSSH_9.6|64": { + "after_auth": [ + "one_os", + 40 + ], + "final": [ + "one_os", + 40 + ] + }, + "one_os.1|SSH-2.0-OpenSSH_9.6|65536": { + "after_auth": [ + "one_os", + 40 + ], + "final": [ + "one_os", + 40 + ] + }, + "one_os.2|-|64": { + "after_auth": [ + "one_os", + 40 + ], + "final": [ + "one_os", + 40 + ] + }, + "one_os.2|-|65536": { + "after_auth": [ + "one_os", + 40 + ], + "final": [ + "one_os", + 40 + ] + }, + "one_os.2|SSH-2.0-Cisco-1.25|64": { + "after_auth": [ + "ios", + 80 + ], + "final": [ + "ios", + 80 + ] + }, + "one_os.2|SSH-2.0-Cisco-1.25|65536": { + "after_auth": [ + "ios", + 80 + ], + "final": [ + "ios", + 80 + ] + }, + "one_os.2|SSH-2.0-OpenSSH_9.6|64": { + "after_auth": [ + "one_os", + 40 + ], + "final": [ + "one_os", + 40 + ] + }, + "one_os.2|SSH-2.0-OpenSSH_9.6|65536": { + "after_auth": [ + "one_os", + 40 + ], + "final": [ + "one_os", + 40 + ] + }, + "prompt-only:|SSH-2.0-OpenSSH_9.6|64": { + "after_auth": [ + "ericsson_ban", + 20 + ], + "final": [ + "ericsson_ban", + 20 + ] + }, + "prompt-only:|SSH-2.0-OpenSSH_9.6|65536": { + "after_auth": [ + "ericsson_ban", + 20 + ], + "final": [ + "ericsson_ban", + 20 + ] + }, + "prompt-only:A:sros-pe1#|SSH-2.0-OpenSSH_9.6|64": { + "after_auth": [ + "ericsson_ban", + 20 + ], + "final": [ + "ericsson_ban", + 20 + ] + }, + "prompt-only:A:sros-pe1#|SSH-2.0-OpenSSH_9.6|65536": { + "after_auth": [ + "ericsson_ban", + 20 + ], + "final": [ + "ericsson_ban", + 20 + ] + }, + "prompt-only:FGT60E #|SSH-2.0-OpenSSH_9.6|64": { + "after_auth": [ + "ericsson_ban", + 20 + ], + "final": [ + "ericsson_ban", + 20 + ] + }, + "prompt-only:FGT60E #|SSH-2.0-OpenSSH_9.6|65536": { + "after_auth": [ + "ericsson_ban", + 20 + ], + "final": [ + "ericsson_ban", + 20 + ] + }, + "prompt-only:RP/0/RSP0/CPU0:xr-pe1#|SSH-2.0-OpenSSH_9.6|64": { + "after_auth": [ + "ericsson_ban", + 20 + ], + "final": [ + "ios_xr", + 95 + ] + }, + "prompt-only:RP/0/RSP0/CPU0:xr-pe1#|SSH-2.0-OpenSSH_9.6|65536": { + "after_auth": [ + "ericsson_ban", + 20 + ], + "final": [ + "ios_xr", + 95 + ] + }, + "prompt-only:[admin@MikroTik] >|SSH-2.0-OpenSSH_9.6|64": { + "after_auth": [ + "ericsson_ban", + 20 + ], + "final": [ + "ericsson_ban", + 20 + ] + }, + "prompt-only:[admin@MikroTik] >|SSH-2.0-OpenSSH_9.6|65536": { + "after_auth": [ + "ericsson_ban", + 20 + ], + "final": [ + "ericsson_ban", + 20 + ] + }, + "prompt-only:admin@junos-mx> |SSH-2.0-OpenSSH_9.6|64": { + "after_auth": [ + "ericsson_ban", + 20 + ], + "final": [ + "ericsson_ban", + 20 + ] + }, + "prompt-only:admin@junos-mx> |SSH-2.0-OpenSSH_9.6|65536": { + "after_auth": [ + "ericsson_ban", + 20 + ], + "final": [ + "ericsson_ban", + 20 + ] + }, + "prompt-only:core-sw01(config)#|SSH-2.0-OpenSSH_9.6|64": { + "after_auth": [ + "ericsson_ban", + 20 + ], + "final": [ + "ericsson_ban", + 20 + ] + }, + "prompt-only:core-sw01(config)#|SSH-2.0-OpenSSH_9.6|65536": { + "after_auth": [ + "ericsson_ban", + 20 + ], + "final": [ + "ericsson_ban", + 20 + ] + }, + "prompt-only:root@host:~#|SSH-2.0-OpenSSH_9.6|64": { + "after_auth": [ + "ericsson_ban", + 20 + ], + "final": [ + "ericsson_ban", + 20 + ] + }, + "prompt-only:root@host:~#|SSH-2.0-OpenSSH_9.6|65536": { + "after_auth": [ + "ericsson_ban", + 20 + ], + "final": [ + "ericsson_ban", + 20 + ] + }, + "prompt-only:router#|SSH-2.0-OpenSSH_9.6|64": { + "after_auth": [ + "ericsson_ban", + 20 + ], + "final": [ + "ericsson_ban", + 20 + ] + }, + "prompt-only:router#|SSH-2.0-OpenSSH_9.6|65536": { + "after_auth": [ + "ericsson_ban", + 20 + ], + "final": [ + "ericsson_ban", + 20 + ] + }, + "prompt-only:switch>|SSH-2.0-OpenSSH_9.6|64": { + "after_auth": [ + "ericsson_ban", + 20 + ], + "final": [ + "ericsson_ban", + 20 + ] + }, + "prompt-only:switch>|SSH-2.0-OpenSSH_9.6|65536": { + "after_auth": [ + "ericsson_ban", + 20 + ], + "final": [ + "ericsson_ban", + 20 + ] + }, + "prompt-only:user@host:~$|SSH-2.0-OpenSSH_9.6|64": { + "after_auth": [ + "ericsson_ban", + 20 + ], + "final": [ + "ericsson_ban", + 20 + ] + }, + "prompt-only:user@host:~$|SSH-2.0-OpenSSH_9.6|65536": { + "after_auth": [ + "ericsson_ban", + 20 + ], + "final": [ + "ericsson_ban", + 20 + ] + }, + "rios.1|-|64": { + "after_auth": [ + "isam", + 20 + ], + "final": [ + "isam", + 20 + ] + }, + "rios.1|-|65536": { + "after_auth": [ + "isam", + 20 + ], + "final": [ + "isam", + 20 + ] + }, + "rios.1|SSH-2.0-Cisco-1.25|64": { + "after_auth": [ + "ios", + 80 + ], + "final": [ + "ios", + 80 + ] + }, + "rios.1|SSH-2.0-Cisco-1.25|65536": { + "after_auth": [ + "ios", + 80 + ], + "final": [ + "ios", + 80 + ] + }, + "rios.1|SSH-2.0-OpenSSH_9.6|64": { + "after_auth": [ + "isam", + 20 + ], + "final": [ + "isam", + 20 + ] + }, + "rios.1|SSH-2.0-OpenSSH_9.6|65536": { + "after_auth": [ + "isam", + 20 + ], + "final": [ + "isam", + 20 + ] + }, + "smart_edge_os.1|-|64": { + "after_auth": [ + "smart_edge_os", + 60 + ], + "final": [ + "smart_edge_os", + 60 + ] + }, + "smart_edge_os.1|-|65536": { + "after_auth": [ + "smart_edge_os", + 60 + ], + "final": [ + "smart_edge_os", + 60 + ] + }, + "smart_edge_os.1|SSH-2.0-Cisco-1.25|64": { + "after_auth": [ + "ios", + 80 + ], + "final": [ + "ios", + 80 + ] + }, + "smart_edge_os.1|SSH-2.0-Cisco-1.25|65536": { + "after_auth": [ + "ios", + 80 + ], + "final": [ + "ios", + 80 + ] + }, + "smart_edge_os.1|SSH-2.0-OpenSSH_9.6|64": { + "after_auth": [ + "smart_edge_os", + 60 + ], + "final": [ + "smart_edge_os", + 60 + ] + }, + "smart_edge_os.1|SSH-2.0-OpenSSH_9.6|65536": { + "after_auth": [ + "smart_edge_os", + 60 + ], + "final": [ + "smart_edge_os", + 60 + ] + }, + "smart_edge_os.2|-|64": { + "after_auth": [ + "smart_edge_os", + 60 + ], + "final": [ + "smart_edge_os", + 60 + ] + }, + "smart_edge_os.2|-|65536": { + "after_auth": [ + "smart_edge_os", + 60 + ], + "final": [ + "smart_edge_os", + 60 + ] + }, + "smart_edge_os.2|SSH-2.0-Cisco-1.25|64": { + "after_auth": [ + "ios", + 80 + ], + "final": [ + "ios", + 80 + ] + }, + "smart_edge_os.2|SSH-2.0-Cisco-1.25|65536": { + "after_auth": [ + "ios", + 80 + ], + "final": [ + "ios", + 80 + ] + }, + "smart_edge_os.2|SSH-2.0-OpenSSH_9.6|64": { + "after_auth": [ + "smart_edge_os", + 60 + ], + "final": [ + "smart_edge_os", + 60 + ] + }, + "smart_edge_os.2|SSH-2.0-OpenSSH_9.6|65536": { + "after_auth": [ + "smart_edge_os", + 60 + ], + "final": [ + "smart_edge_os", + 60 + ] + }, + "smart_edge_os|-|64": { + "after_auth": [ + "smart_edge_os", + 60 + ], + "final": [ + "smart_edge_os", + 60 + ] + }, + "smart_edge_os|-|65536": { + "after_auth": [ + "smart_edge_os", + 60 + ], + "final": [ + "smart_edge_os", + 60 + ] + }, + "smart_edge_os|SSH-2.0-Cisco-1.25|64": { + "after_auth": [ + "ios", + 80 + ], + "final": [ + "ios", + 80 + ] + }, + "smart_edge_os|SSH-2.0-Cisco-1.25|65536": { + "after_auth": [ + "ios", + 80 + ], + "final": [ + "ios", + 80 + ] + }, + "smart_edge_os|SSH-2.0-OpenSSH_9.6|64": { + "after_auth": [ + "smart_edge_os", + 60 + ], + "final": [ + "smart_edge_os", + 60 + ] + }, + "smart_edge_os|SSH-2.0-OpenSSH_9.6|65536": { + "after_auth": [ + "smart_edge_os", + 60 + ], + "final": [ + "smart_edge_os", + 60 + ] + }, + "sp_vxoa|-|64": { + "after_auth": [ + "sp_vxoa", + 90 + ], + "final": [ + "sp_vxoa", + 90 + ] + }, + "sp_vxoa|-|65536": { + "after_auth": [ + "sp_vxoa", + 90 + ], + "final": [ + "sp_vxoa", + 90 + ] + }, + "sp_vxoa|SSH-2.0-Cisco-1.25|64": { + "after_auth": [ + "sp_vxoa", + 90 + ], + "final": [ + "sp_vxoa", + 90 + ] + }, + "sp_vxoa|SSH-2.0-Cisco-1.25|65536": { + "after_auth": [ + "sp_vxoa", + 90 + ], + "final": [ + "sp_vxoa", + 90 + ] + }, + "sp_vxoa|SSH-2.0-OpenSSH_9.6|64": { + "after_auth": [ + "sp_vxoa", + 90 + ], + "final": [ + "sp_vxoa", + 90 + ] + }, + "sp_vxoa|SSH-2.0-OpenSSH_9.6|65536": { + "after_auth": [ + "sp_vxoa", + 90 + ], + "final": [ + "sp_vxoa", + 90 + ] + }, + "sros|-|64": { + "after_auth": [ + "sros", + 95 + ], + "final": [ + "sros", + 95 + ] + }, + "sros|-|65536": { + "after_auth": [ + "sros", + 95 + ], + "final": [ + "sros", + 95 + ] + }, + "sros|SSH-2.0-Cisco-1.25|64": { + "after_auth": [ + "sros", + 95 + ], + "final": [ + "sros", + 95 + ] + }, + "sros|SSH-2.0-Cisco-1.25|65536": { + "after_auth": [ + "sros", + 95 + ], + "final": [ + "sros", + 95 + ] + }, + "sros|SSH-2.0-OpenSSH_9.6|64": { + "after_auth": [ + "sros", + 95 + ], + "final": [ + "sros", + 95 + ] + }, + "sros|SSH-2.0-OpenSSH_9.6|65536": { + "after_auth": [ + "sros", + 95 + ], + "final": [ + "sros", + 95 + ] + }, + "unknown|-|64": { + "after_auth": [ + "unknown", + 0 + ], + "final": [ + "isam", + 20 + ] + }, + "unknown|-|65536": { + "after_auth": [ + "unknown", + 0 + ], + "final": [ + "isam", + 20 + ] + }, + "unknown|SSH-2.0-Cisco-1.25|64": { + "after_auth": [ + "ios", + 80 + ], + "final": [ + "ios", + 80 + ] + }, + "unknown|SSH-2.0-Cisco-1.25|65536": { + "after_auth": [ + "ios", + 80 + ], + "final": [ + "ios", + 80 + ] + }, + "unknown|SSH-2.0-OpenSSH_9.6|64": { + "after_auth": [ + "unknown", + 0 + ], + "final": [ + "isam", + 20 + ] + }, + "unknown|SSH-2.0-OpenSSH_9.6|65536": { + "after_auth": [ + "unknown", + 0 + ], + "final": [ + "isam", + 20 + ] + }, + "vrp.1|-|64": { + "after_auth": [ + "vrp", + 70 + ], + "final": [ + "vrp", + 70 + ] + }, + "vrp.1|-|65536": { + "after_auth": [ + "vrp", + 70 + ], + "final": [ + "vrp", + 70 + ] + }, + "vrp.1|SSH-2.0-Cisco-1.25|64": { + "after_auth": [ + "ios", + 80 + ], + "final": [ + "ios", + 80 + ] + }, + "vrp.1|SSH-2.0-Cisco-1.25|65536": { + "after_auth": [ + "ios", + 80 + ], + "final": [ + "ios", + 80 + ] + }, + "vrp.1|SSH-2.0-OpenSSH_9.6|64": { + "after_auth": [ + "vrp", + 70 + ], + "final": [ + "vrp", + 70 + ] + }, + "vrp.1|SSH-2.0-OpenSSH_9.6|65536": { + "after_auth": [ + "vrp", + 70 + ], + "final": [ + "vrp", + 70 + ] + }, + "vxworks|-|64": { + "after_auth": [ + "vxworks", + 90 + ], + "final": [ + "vxworks", + 90 + ] + }, + "vxworks|-|65536": { + "after_auth": [ + "vxworks", + 90 + ], + "final": [ + "vxworks", + 90 + ] + }, + "vxworks|SSH-2.0-Cisco-1.25|64": { + "after_auth": [ + "vxworks", + 90 + ], + "final": [ + "vxworks", + 90 + ] + }, + "vxworks|SSH-2.0-Cisco-1.25|65536": { + "after_auth": [ + "vxworks", + 90 + ], + "final": [ + "vxworks", + 90 + ] + }, + "vxworks|SSH-2.0-OpenSSH_9.6|64": { + "after_auth": [ + "vxworks", + 90 + ], + "final": [ + "vxworks", + 90 + ] + }, + "vxworks|SSH-2.0-OpenSSH_9.6|65536": { + "after_auth": [ + "vxworks", + 90 + ], + "final": [ + "vxworks", + 90 + ] + }, + "zte|-|64": { + "after_auth": [ + "zte", + 90 + ], + "final": [ + "zte", + 90 + ] + }, + "zte|-|65536": { + "after_auth": [ + "zte", + 90 + ], + "final": [ + "zte", + 90 + ] + }, + "zte|SSH-2.0-Cisco-1.25|64": { + "after_auth": [ + "zte", + 90 + ], + "final": [ + "zte", + 90 + ] + }, + "zte|SSH-2.0-Cisco-1.25|65536": { + "after_auth": [ + "zte", + 90 + ], + "final": [ + "zte", + 90 + ] + }, + "zte|SSH-2.0-OpenSSH_9.6|64": { + "after_auth": [ + "zte", + 90 + ], + "final": [ + "zte", + 90 + ] + }, + "zte|SSH-2.0-OpenSSH_9.6|65536": { + "after_auth": [ + "zte", + 90 + ], + "final": [ + "zte", + 90 + ] + }, + "zyxel|-|64": { + "after_auth": [ + "zyxel", + 90 + ], + "final": [ + "zyxel", + 90 + ] + }, + "zyxel|-|65536": { + "after_auth": [ + "zyxel", + 90 + ], + "final": [ + "zyxel", + 90 + ] + }, + "zyxel|SSH-2.0-Cisco-1.25|64": { + "after_auth": [ + "zyxel", + 90 + ], + "final": [ + "zyxel", + 90 + ] + }, + "zyxel|SSH-2.0-Cisco-1.25|65536": { + "after_auth": [ + "zyxel", + 90 + ], + "final": [ + "zyxel", + 90 + ] + }, + "zyxel|SSH-2.0-OpenSSH_9.6|64": { + "after_auth": [ + "zyxel", + 90 + ], + "final": [ + "zyxel", + 90 + ] + }, + "zyxel|SSH-2.0-OpenSSH_9.6|65536": { + "after_auth": [ + "zyxel", + 90 + ], + "final": [ + "zyxel", + 90 + ] + } +} diff --git a/tools/bench_bulk.py b/tools/bench_bulk.py new file mode 100755 index 00000000..319abd6f --- /dev/null +++ b/tools/bench_bulk.py @@ -0,0 +1,180 @@ +#!/usr/bin/env python3 +""" +Connection-layer benchmarks on loopback (no real devices needed). + +Measures ``execute()`` with a large response over Telnet and SSH, NETCONF 1.1 +chunk reassembly, and the SSH handshakes needed for one login when the IOS +driver is detected. Run from the repository root:: + + python tools/bench_bulk.py [--mib 4] [--runs 3] [--json results.json] +""" +import argparse +import json +import os +import socket +import statistics +import sys +import threading +import time + +sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), '..')) + +import paramiko + +from Exscript import Account +from Exscript.protocols import SSH2, Telnet +from Exscript.protocols.netconf import Netconf + +PROMPT = b'\r\nrouter# ' +LINE = b' description uplink to core router number 0000 via ae1.100\r\n' + + +class _SSHServerInterface(paramiko.ServerInterface): + + def __init__(self, allowed): + self.allowed = allowed + + def get_allowed_auths(self, username): + return self.allowed + + def check_auth_password(self, username, password): + return paramiko.AUTH_SUCCESSFUL + + def check_channel_request(self, kind, chanid): + return paramiko.OPEN_SUCCEEDED + + def check_channel_pty_request(self, *args): + return True + + def check_channel_shell_request(self, channel): + return True + + +class Server(threading.Thread): + """Loopback CLI: echoes each command and answers ``show big`` with the payload.""" + + def __init__(self, payload=b'', host_key=None, version=None, allowed='password'): + super().__init__(daemon=True) + self.payload = payload + self.host_key = host_key # None: plain TCP (Telnet without option negotiation) + self.version = version + self.allowed = allowed + self.connections = 0 + self.sock = socket.create_server(('127.0.0.1', 0)) + self.port = self.sock.getsockname()[1] + self.start() + + def run(self): + while True: + try: + conn, _ = self.sock.accept() + except OSError: + return + self.connections += 1 + threading.Thread(target=self.handle, args=(conn,), daemon=True).start() + + def handle(self, conn): + if self.host_key is None: + self.cli(conn.recv, conn.sendall) + return + transport = paramiko.Transport(conn) + if self.version: + transport.local_version = self.version + transport.add_server_key(self.host_key) + try: + transport.start_server(server=_SSHServerInterface(self.allowed)) + except (paramiko.SSHException, EOFError): + return + channel = transport.accept(10) + if channel is not None: + self.cli(channel.recv, channel.sendall) + + def cli(self, recv, sendall): + buf = b'' + try: + sendall(b'Welcome' + PROMPT) + while data := recv(65536): + buf += data + while b'\r' in buf: + line, _, buf = buf.partition(b'\r') + line = line.strip(b'\n\x00') + body = b'\r\n' + self.payload if line == b'show big' else b'' + sendall(line + body + PROMPT) + except OSError: + pass + + def close(self): + self.sock.close() + + +def bench_execute(conn, server, size): + """Returns (wall seconds, client thread CPU seconds) of one ``execute('show big')``.""" + conn.connect('127.0.0.1', server.port) + conn.login(Account('admin', 'password', needs_lock=False)) + wall, cpu = time.perf_counter(), time.thread_time() + conn.execute('show big') + wall, cpu = time.perf_counter() - wall, time.thread_time() - cpu + assert len(conn.response) >= size, 'incomplete response' + conn.close(force=True) + return wall, cpu + + +def bench_reassembly(size): + chunk = 'x' * 4096 + framed = ''.join('\n#%d\n%s' % (len(chunk), chunk) for _ in range(size // len(chunk))) + start = time.perf_counter() + assert Netconf.reassemble_chunks(framed) == chunk * (size // len(chunk)) + return time.perf_counter() - start + + +def bench_ios_login(host_key): + server = Server(host_key=host_key, version='SSH-2.0-Cisco-1.25', + allowed='publickey,keyboard-interactive,password') + conn = SSH2(verify_fingerprint=False) # default connect rate limit, like real usage + start = time.perf_counter() + conn.connect('127.0.0.1', server.port) + conn.login(Account('admin', 'password', needs_lock=False)) + elapsed = time.perf_counter() - start + driver = conn.get_driver().name + conn.close(force=True) + server.close() + return elapsed, server.connections, driver + + +def main(): + parser = argparse.ArgumentParser(description=__doc__.split('\n\n')[0]) + parser.add_argument('--mib', type=int, default=4, help='response size in MiB (default: 4)') + parser.add_argument('--runs', type=int, default=3, help='runs per measurement, median is reported') + parser.add_argument('--json', help='also write the results to this file') + args = parser.parse_args() + size = args.mib * 1024 * 1024 + payload = LINE * (size // len(LINE) + 1) + host_key = paramiko.RSAKey.generate(2048) + results = {'python': sys.version.split()[0], 'paramiko': paramiko.__version__, 'mib': args.mib} + + for name, make_conn, server_kwargs in ( + ('telnet_execute', lambda: Telnet(connect_rate_limit=0), {}), + ('ssh_execute', lambda: SSH2(connect_rate_limit=0, verify_fingerprint=False), {'host_key': host_key}), + ): + server = Server(payload, **server_kwargs) + runs = [bench_execute(make_conn(), server, size) for _ in range(args.runs)] + server.close() + results[name + '_wall_s'] = round(statistics.median(r[0] for r in runs), 3) + results[name + '_cpu_s'] = round(statistics.median(r[1] for r in runs), 3) + + results['netconf_reassembly_s'] = round( + statistics.median(bench_reassembly(size) for _ in range(args.runs)), 3) + elapsed, handshakes, driver = bench_ios_login(host_key) + results['ios_login_s'] = round(elapsed, 2) + results['ios_login_handshakes'] = handshakes + results['ios_login_driver'] = driver + + for key, value in results.items(): + print('%-28s %s' % (key, value)) + if args.json: + with open(args.json, 'w', encoding='utf-8') as fp: + json.dump(results, fp, indent=1) + + +if __name__ == '__main__': + main() From fef99ad8d2d857d411366bafc54299a7d611f43d Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Fri, 30 Oct 2026 17:47:31 +0100 Subject: [PATCH 003/250] Restore the public Lexer.forward() The 2.9 line renamed it to _forward(), which breaks external parsers built on Exscript.parselib that call lexer.forward(). --- Exscript/parselib/lexer.py | 4 ++-- tests/Exscript/LexerTest.py | 23 +++++++++++++++++++++++ 2 files changed, 25 insertions(+), 2 deletions(-) create mode 100644 tests/Exscript/LexerTest.py diff --git a/Exscript/parselib/lexer.py b/Exscript/parselib/lexer.py index 91a9c9f8..9e4da359 100644 --- a/Exscript/parselib/lexer.py +++ b/Exscript/parselib/lexer.py @@ -162,7 +162,7 @@ def syntax_warning(self, warning, sender=None): def runtime_error(self, error, sender=None): self._error(ExecuteError, error, sender) - def _forward(self, chars=1): + def forward(self, chars=1): self.last_char = self.current_char self.current_char += chars self.token_buffer = None @@ -172,7 +172,7 @@ def __next__(self): if self.token_buffer[0] == 'EOF': msg = "Unexpected EOF while iterating tokens." raise StopIteration(msg) - self._forward(len(self.token_buffer[1])) + self.forward(len(self.token_buffer[1])) def next_if(self, types, token=None): if type(types) is not type([]): diff --git a/tests/Exscript/LexerTest.py b/tests/Exscript/LexerTest.py new file mode 100644 index 00000000..9dc43103 --- /dev/null +++ b/tests/Exscript/LexerTest.py @@ -0,0 +1,23 @@ +import os +import sys +import unittest + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..')) + +from Exscript.parselib import Lexer + + +class LexerTest(unittest.TestCase): + + def testForward(self): + # Public API for parsers built on parselib. + lexer = Lexer(object) + lexer.forward(3) + lexer.forward() + self.assertEqual((lexer.last_char, lexer.current_char), (3, 4)) + + +def suite(): + return unittest.TestLoader().loadTestsFromTestCase(LexerTest) +if __name__ == '__main__': + unittest.TextTestRunner(verbosity=2).run(suite()) From 196760e90debf6b78e7b7707e0cafb02b492922e Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Fri, 30 Oct 2026 18:11:15 +0100 Subject: [PATCH 004/250] Move to pyproject.toml, hatchling and uv with a src layout The version comes from git tags (hatch-vcs), and pytest runs the unittest suite against the installed package. Python 3.12+ and paramiko 3.4+ (Terrapin fix) are required. --- .gitattributes | 2 + .gitignore | 4 + .hound.yml | 2 - .travis.yml | 14 - Exscript/version.py | 4 - MANIFEST.in | 1 - Makefile | 72 -- TODO | 5 - VERSION.in | 4 - pyproject.toml | 81 ++ requirements.txt | 5 - setup.cfg | 2 - setup.py | 63 -- {Exscript => src/Exscript}/__init__.py | 0 {Exscript => src/Exscript}/account.py | 0 .../Exscript}/contrib/__init__.py | 0 {Exscript => src/Exscript}/contrib/otel.py | 0 .../Exscript}/emulators/__init__.py | 0 .../Exscript}/emulators/command.py | 0 .../Exscript}/emulators/iosemu.py | 0 .../Exscript}/emulators/vdevice.py | 0 {Exscript => src/Exscript}/host.py | 0 .../Exscript}/interpreter/__init__.py | 0 .../Exscript}/interpreter/append.py | 0 .../Exscript}/interpreter/assign.py | 0 .../Exscript}/interpreter/code.py | 0 .../Exscript}/interpreter/enter.py | 0 .../Exscript}/interpreter/exception.py | 0 .../Exscript}/interpreter/execute.py | 0 .../Exscript}/interpreter/expression.py | 0 .../Exscript}/interpreter/extract.py | 0 .../Exscript}/interpreter/fail.py | 0 .../Exscript}/interpreter/functioncall.py | 0 .../Exscript}/interpreter/ifcondition.py | 0 .../Exscript}/interpreter/loop.py | 0 .../Exscript}/interpreter/number.py | 0 .../Exscript}/interpreter/parser.py | 0 .../Exscript}/interpreter/program.py | 0 .../Exscript}/interpreter/regex.py | 0 {Exscript => src/Exscript}/interpreter/rpc.py | 0 .../Exscript}/interpreter/scope.py | 0 .../Exscript}/interpreter/string.py | 0 .../Exscript}/interpreter/template.py | 0 .../Exscript}/interpreter/term.py | 0 .../Exscript}/interpreter/trying.py | 0 .../Exscript}/interpreter/variable.py | 0 {Exscript => src/Exscript}/key.py | 0 {Exscript => src/Exscript}/logger.py | 0 .../Exscript}/parselib/__init__.py | 0 .../Exscript}/parselib/exception.py | 0 {Exscript => src/Exscript}/parselib/lexer.py | 0 {Exscript => src/Exscript}/parselib/token.py | 0 .../Exscript}/protocols/__init__.py | 0 .../Exscript}/protocols/drivers/__init__.py | 0 .../Exscript}/protocols/drivers/ace.py | 0 .../Exscript}/protocols/drivers/adtran.py | 0 .../Exscript}/protocols/drivers/aironet.py | 0 .../Exscript}/protocols/drivers/aix.py | 0 .../protocols/drivers/arbor_peakflow.py | 0 .../Exscript}/protocols/drivers/aruba.py | 0 .../Exscript}/protocols/drivers/bigip.py | 0 .../Exscript}/protocols/drivers/brocade.py | 0 .../Exscript}/protocols/drivers/cienasaos.py | 0 .../Exscript}/protocols/drivers/driver.py | 0 .../Exscript}/protocols/drivers/enterasys.py | 0 .../protocols/drivers/enterasys_wc.py | 0 .../Exscript}/protocols/drivers/eos.py | 0 .../protocols/drivers/ericsson_ban.py | 0 .../Exscript}/protocols/drivers/fortios.py | 0 .../Exscript}/protocols/drivers/generic.py | 0 .../protocols/drivers/hp_pro_curve.py | 0 .../Exscript}/protocols/drivers/icotera.py | 0 .../Exscript}/protocols/drivers/ios.py | 0 .../Exscript}/protocols/drivers/ios_xr.py | 0 .../Exscript}/protocols/drivers/isam.py | 0 .../Exscript}/protocols/drivers/junos.py | 0 .../Exscript}/protocols/drivers/junos_erx.py | 0 .../Exscript}/protocols/drivers/mrv.py | 0 .../Exscript}/protocols/drivers/nxos.py | 0 .../Exscript}/protocols/drivers/one_os.py | 0 .../Exscript}/protocols/drivers/rios.py | 0 .../Exscript}/protocols/drivers/shell.py | 0 .../protocols/drivers/smart_edge_os.py | 0 .../Exscript}/protocols/drivers/sp_vxoa.py | 0 .../Exscript}/protocols/drivers/sros.py | 0 .../Exscript}/protocols/drivers/vrp.py | 0 .../Exscript}/protocols/drivers/vxworks.py | 0 .../Exscript}/protocols/drivers/zte.py | 0 .../Exscript}/protocols/drivers/zyxel.py | 0 {Exscript => src/Exscript}/protocols/dummy.py | 0 .../Exscript}/protocols/exception.py | 0 .../Exscript}/protocols/netconf.py | 0 .../Exscript}/protocols/osguesser.py | 0 .../Exscript}/protocols/protocol.py | 0 {Exscript => src/Exscript}/protocols/ssh2.py | 0 .../Exscript}/protocols/telnet.py | 0 .../Exscript}/protocols/telnetlib.py | 0 {Exscript => src/Exscript}/queue.py | 0 .../Exscript}/servers/__init__.py | 0 {Exscript => src/Exscript}/servers/httpd.py | 0 {Exscript => src/Exscript}/servers/server.py | 0 {Exscript => src/Exscript}/servers/sshd.py | 0 {Exscript => src/Exscript}/servers/telnetd.py | 0 {Exscript => src/Exscript}/stdlib/__init__.py | 0 .../Exscript}/stdlib/connection.py | 0 {Exscript => src/Exscript}/stdlib/crypt.py | 0 {Exscript => src/Exscript}/stdlib/file.py | 0 {Exscript => src/Exscript}/stdlib/ipv4.py | 0 {Exscript => src/Exscript}/stdlib/list.py | 0 {Exscript => src/Exscript}/stdlib/mysys.py | 0 {Exscript => src/Exscript}/stdlib/string.py | 0 {Exscript => src/Exscript}/stdlib/util.py | 0 {Exscript => src/Exscript}/util/__init__.py | 0 {Exscript => src/Exscript}/util/buffer.py | 0 {Exscript => src/Exscript}/util/cast.py | 0 .../Exscript}/util/collections.py | 0 {Exscript => src/Exscript}/util/crypt.py | 0 {Exscript => src/Exscript}/util/daemonize.py | 0 {Exscript => src/Exscript}/util/decorator.py | 0 {Exscript => src/Exscript}/util/event.py | 0 {Exscript => src/Exscript}/util/file.py | 0 {Exscript => src/Exscript}/util/impl.py | 0 {Exscript => src/Exscript}/util/interact.py | 0 {Exscript => src/Exscript}/util/ip.py | 0 {Exscript => src/Exscript}/util/ipv4.py | 0 {Exscript => src/Exscript}/util/ipv6.py | 0 {Exscript => src/Exscript}/util/log.py | 0 {Exscript => src/Exscript}/util/mail.py | 0 {Exscript => src/Exscript}/util/match.py | 0 {Exscript => src/Exscript}/util/pidutil.py | 0 {Exscript => src/Exscript}/util/report.py | 0 {Exscript => src/Exscript}/util/sigint.py | 0 .../Exscript}/util/sigintcatcher.py | 0 {Exscript => src/Exscript}/util/start.py | 0 {Exscript => src/Exscript}/util/syslog.py | 0 {Exscript => src/Exscript}/util/template.py | 0 {Exscript => src/Exscript}/util/tty.py | 0 {Exscript => src/Exscript}/util/url.py | 0 {Exscript => src/Exscript}/util/weakmethod.py | 0 {Exscript => src/Exscript}/util/xml.py | 0 src/Exscript/version.py | 9 + .../Exscript}/workqueue/__init__.py | 0 {Exscript => src/Exscript}/workqueue/job.py | 0 .../Exscript}/workqueue/mainloop.py | 0 .../Exscript}/workqueue/pipeline.py | 0 {Exscript => src/Exscript}/workqueue/task.py | 0 .../Exscript}/workqueue/workqueue.py | 0 tests/Exscript/AccountManagerTest.py | 5 - tests/Exscript/AccountPoolTest.py | 3 - tests/Exscript/AccountTest.py | 4 - tests/Exscript/FileLoggerTest.py | 5 +- tests/Exscript/HostTest.py | 4 - tests/Exscript/KnownIssuesTest.py | 1 - tests/Exscript/LexerTest.py | 3 - tests/Exscript/LogTest.py | 4 - tests/Exscript/LogfileTest.py | 3 - tests/Exscript/LoggerTest.py | 6 - tests/Exscript/PrivateKeyTest.py | 3 - tests/Exscript/QueueTest.py | 3 - tests/Exscript/TemplateTest.py | 3 - tests/Exscript/emulators/CommandSetTest.py | 4 - tests/Exscript/emulators/IOSEmulatorTest.py | 4 - tests/Exscript/emulators/VirtualDeviceTest.py | 4 - tests/Exscript/memtest.py | 124 --- tests/Exscript/memtest_host.py | 13 - tests/Exscript/protocols/DummyTest.py | 3 - .../Exscript/protocols/OsGuesserCorpusTest.py | 2 - tests/Exscript/protocols/OsGuesserTest.py | 2 - tests/Exscript/protocols/ProtocolTest.py | 2 - tests/Exscript/protocols/SSH2Test.py | 3 - tests/Exscript/protocols/TelnetTest.py | 6 - tests/Exscript/servers/SSHdTest.py | 4 - tests/Exscript/servers/ServerTest.py | 3 - tests/Exscript/servers/TelnetdTest.py | 4 - tests/Exscript/util/bufferTest.py | 4 - tests/Exscript/util/castTest.py | 4 - tests/Exscript/util/cryptTest.py | 4 - tests/Exscript/util/decoratorTest.py | 7 - tests/Exscript/util/eventTest.py | 4 - tests/Exscript/util/fileTest.py | 3 - tests/Exscript/util/interactTest.py | 14 +- tests/Exscript/util/ipTest.py | 4 - tests/Exscript/util/ipv4Test.py | 4 - tests/Exscript/util/ipv6Test.py | 4 - tests/Exscript/util/mailTest.py | 4 - tests/Exscript/util/matchTest.py | 4 - tests/Exscript/util/reportTest.py | 7 +- tests/Exscript/util/startTest.py | 4 - tests/Exscript/util/syslogTest.py | 5 +- tests/Exscript/util/ttyTest.py | 3 - tests/Exscript/util/urlTest.py | 4 - tests/Exscript/util/weakmethodTest.py | 4 - tests/Exscript/workqueue/JobTest.py | 6 - tests/Exscript/workqueue/MainLoopTest.py | 4 - tests/Exscript/workqueue/PipelineTest.py | 5 - tests/Exscript/workqueue/TaskTest.py | 4 - tests/Exscript/workqueue/WorkQueueTest.py | 5 - tests/conftest.py | 16 + tools/bench_bulk.py | 2 - tox.ini | 17 - uv.lock | 824 ++++++++++++++++++ version.sh | 94 -- 202 files changed, 945 insertions(+), 612 deletions(-) create mode 100644 .gitattributes delete mode 100644 .hound.yml delete mode 100644 .travis.yml delete mode 100644 Exscript/version.py delete mode 100644 MANIFEST.in delete mode 100644 Makefile delete mode 100644 TODO delete mode 100644 VERSION.in create mode 100644 pyproject.toml delete mode 100644 requirements.txt delete mode 100644 setup.cfg delete mode 100644 setup.py rename {Exscript => src/Exscript}/__init__.py (100%) rename {Exscript => src/Exscript}/account.py (100%) rename {Exscript => src/Exscript}/contrib/__init__.py (100%) rename {Exscript => src/Exscript}/contrib/otel.py (100%) rename {Exscript => src/Exscript}/emulators/__init__.py (100%) rename {Exscript => src/Exscript}/emulators/command.py (100%) rename {Exscript => src/Exscript}/emulators/iosemu.py (100%) rename {Exscript => src/Exscript}/emulators/vdevice.py (100%) rename {Exscript => src/Exscript}/host.py (100%) rename {Exscript => src/Exscript}/interpreter/__init__.py (100%) rename {Exscript => src/Exscript}/interpreter/append.py (100%) rename {Exscript => src/Exscript}/interpreter/assign.py (100%) rename {Exscript => src/Exscript}/interpreter/code.py (100%) rename {Exscript => src/Exscript}/interpreter/enter.py (100%) rename {Exscript => src/Exscript}/interpreter/exception.py (100%) rename {Exscript => src/Exscript}/interpreter/execute.py (100%) rename {Exscript => src/Exscript}/interpreter/expression.py (100%) rename {Exscript => src/Exscript}/interpreter/extract.py (100%) rename {Exscript => src/Exscript}/interpreter/fail.py (100%) rename {Exscript => src/Exscript}/interpreter/functioncall.py (100%) rename {Exscript => src/Exscript}/interpreter/ifcondition.py (100%) rename {Exscript => src/Exscript}/interpreter/loop.py (100%) rename {Exscript => src/Exscript}/interpreter/number.py (100%) rename {Exscript => src/Exscript}/interpreter/parser.py (100%) rename {Exscript => src/Exscript}/interpreter/program.py (100%) rename {Exscript => src/Exscript}/interpreter/regex.py (100%) rename {Exscript => src/Exscript}/interpreter/rpc.py (100%) rename {Exscript => src/Exscript}/interpreter/scope.py (100%) rename {Exscript => src/Exscript}/interpreter/string.py (100%) rename {Exscript => src/Exscript}/interpreter/template.py (100%) rename {Exscript => src/Exscript}/interpreter/term.py (100%) rename {Exscript => src/Exscript}/interpreter/trying.py (100%) rename {Exscript => src/Exscript}/interpreter/variable.py (100%) rename {Exscript => src/Exscript}/key.py (100%) rename {Exscript => src/Exscript}/logger.py (100%) rename {Exscript => src/Exscript}/parselib/__init__.py (100%) rename {Exscript => src/Exscript}/parselib/exception.py (100%) rename {Exscript => src/Exscript}/parselib/lexer.py (100%) rename {Exscript => src/Exscript}/parselib/token.py (100%) rename {Exscript => src/Exscript}/protocols/__init__.py (100%) rename {Exscript => src/Exscript}/protocols/drivers/__init__.py (100%) rename {Exscript => src/Exscript}/protocols/drivers/ace.py (100%) rename {Exscript => src/Exscript}/protocols/drivers/adtran.py (100%) rename {Exscript => src/Exscript}/protocols/drivers/aironet.py (100%) rename {Exscript => src/Exscript}/protocols/drivers/aix.py (100%) rename {Exscript => src/Exscript}/protocols/drivers/arbor_peakflow.py (100%) rename {Exscript => src/Exscript}/protocols/drivers/aruba.py (100%) rename {Exscript => src/Exscript}/protocols/drivers/bigip.py (100%) rename {Exscript => src/Exscript}/protocols/drivers/brocade.py (100%) rename {Exscript => src/Exscript}/protocols/drivers/cienasaos.py (100%) rename {Exscript => src/Exscript}/protocols/drivers/driver.py (100%) rename {Exscript => src/Exscript}/protocols/drivers/enterasys.py (100%) rename {Exscript => src/Exscript}/protocols/drivers/enterasys_wc.py (100%) rename {Exscript => src/Exscript}/protocols/drivers/eos.py (100%) rename {Exscript => src/Exscript}/protocols/drivers/ericsson_ban.py (100%) rename {Exscript => src/Exscript}/protocols/drivers/fortios.py (100%) rename {Exscript => src/Exscript}/protocols/drivers/generic.py (100%) rename {Exscript => src/Exscript}/protocols/drivers/hp_pro_curve.py (100%) rename {Exscript => src/Exscript}/protocols/drivers/icotera.py (100%) rename {Exscript => src/Exscript}/protocols/drivers/ios.py (100%) rename {Exscript => src/Exscript}/protocols/drivers/ios_xr.py (100%) rename {Exscript => src/Exscript}/protocols/drivers/isam.py (100%) rename {Exscript => src/Exscript}/protocols/drivers/junos.py (100%) rename {Exscript => src/Exscript}/protocols/drivers/junos_erx.py (100%) rename {Exscript => src/Exscript}/protocols/drivers/mrv.py (100%) rename {Exscript => src/Exscript}/protocols/drivers/nxos.py (100%) rename {Exscript => src/Exscript}/protocols/drivers/one_os.py (100%) rename {Exscript => src/Exscript}/protocols/drivers/rios.py (100%) rename {Exscript => src/Exscript}/protocols/drivers/shell.py (100%) rename {Exscript => src/Exscript}/protocols/drivers/smart_edge_os.py (100%) rename {Exscript => src/Exscript}/protocols/drivers/sp_vxoa.py (100%) rename {Exscript => src/Exscript}/protocols/drivers/sros.py (100%) rename {Exscript => src/Exscript}/protocols/drivers/vrp.py (100%) rename {Exscript => src/Exscript}/protocols/drivers/vxworks.py (100%) rename {Exscript => src/Exscript}/protocols/drivers/zte.py (100%) rename {Exscript => src/Exscript}/protocols/drivers/zyxel.py (100%) rename {Exscript => src/Exscript}/protocols/dummy.py (100%) rename {Exscript => src/Exscript}/protocols/exception.py (100%) rename {Exscript => src/Exscript}/protocols/netconf.py (100%) rename {Exscript => src/Exscript}/protocols/osguesser.py (100%) rename {Exscript => src/Exscript}/protocols/protocol.py (100%) rename {Exscript => src/Exscript}/protocols/ssh2.py (100%) rename {Exscript => src/Exscript}/protocols/telnet.py (100%) rename {Exscript => src/Exscript}/protocols/telnetlib.py (100%) rename {Exscript => src/Exscript}/queue.py (100%) rename {Exscript => src/Exscript}/servers/__init__.py (100%) rename {Exscript => src/Exscript}/servers/httpd.py (100%) rename {Exscript => src/Exscript}/servers/server.py (100%) rename {Exscript => src/Exscript}/servers/sshd.py (100%) rename {Exscript => src/Exscript}/servers/telnetd.py (100%) rename {Exscript => src/Exscript}/stdlib/__init__.py (100%) rename {Exscript => src/Exscript}/stdlib/connection.py (100%) rename {Exscript => src/Exscript}/stdlib/crypt.py (100%) rename {Exscript => src/Exscript}/stdlib/file.py (100%) rename {Exscript => src/Exscript}/stdlib/ipv4.py (100%) rename {Exscript => src/Exscript}/stdlib/list.py (100%) rename {Exscript => src/Exscript}/stdlib/mysys.py (100%) rename {Exscript => src/Exscript}/stdlib/string.py (100%) rename {Exscript => src/Exscript}/stdlib/util.py (100%) rename {Exscript => src/Exscript}/util/__init__.py (100%) rename {Exscript => src/Exscript}/util/buffer.py (100%) rename {Exscript => src/Exscript}/util/cast.py (100%) rename {Exscript => src/Exscript}/util/collections.py (100%) rename {Exscript => src/Exscript}/util/crypt.py (100%) rename {Exscript => src/Exscript}/util/daemonize.py (100%) rename {Exscript => src/Exscript}/util/decorator.py (100%) rename {Exscript => src/Exscript}/util/event.py (100%) rename {Exscript => src/Exscript}/util/file.py (100%) rename {Exscript => src/Exscript}/util/impl.py (100%) rename {Exscript => src/Exscript}/util/interact.py (100%) rename {Exscript => src/Exscript}/util/ip.py (100%) rename {Exscript => src/Exscript}/util/ipv4.py (100%) rename {Exscript => src/Exscript}/util/ipv6.py (100%) rename {Exscript => src/Exscript}/util/log.py (100%) rename {Exscript => src/Exscript}/util/mail.py (100%) rename {Exscript => src/Exscript}/util/match.py (100%) rename {Exscript => src/Exscript}/util/pidutil.py (100%) rename {Exscript => src/Exscript}/util/report.py (100%) rename {Exscript => src/Exscript}/util/sigint.py (100%) rename {Exscript => src/Exscript}/util/sigintcatcher.py (100%) rename {Exscript => src/Exscript}/util/start.py (100%) rename {Exscript => src/Exscript}/util/syslog.py (100%) rename {Exscript => src/Exscript}/util/template.py (100%) rename {Exscript => src/Exscript}/util/tty.py (100%) rename {Exscript => src/Exscript}/util/url.py (100%) rename {Exscript => src/Exscript}/util/weakmethod.py (100%) rename {Exscript => src/Exscript}/util/xml.py (100%) create mode 100644 src/Exscript/version.py rename {Exscript => src/Exscript}/workqueue/__init__.py (100%) rename {Exscript => src/Exscript}/workqueue/job.py (100%) rename {Exscript => src/Exscript}/workqueue/mainloop.py (100%) rename {Exscript => src/Exscript}/workqueue/pipeline.py (100%) rename {Exscript => src/Exscript}/workqueue/task.py (100%) rename {Exscript => src/Exscript}/workqueue/workqueue.py (100%) delete mode 100644 tests/Exscript/memtest.py delete mode 100644 tests/Exscript/memtest_host.py create mode 100644 tests/conftest.py delete mode 100644 tox.ini create mode 100644 uv.lock delete mode 100755 version.sh diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..d0d6371d --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +* text=auto eol=lf +*.png binary diff --git a/.gitignore b/.gitignore index 277334e6..7f99e872 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,7 @@ config.xml dist run_suite.log unit_test.cfg +.pytest_cache +.ruff_cache +htmlcov +coverage.xml diff --git a/.hound.yml b/.hound.yml deleted file mode 100644 index 7179ca4c..00000000 --- a/.hound.yml +++ /dev/null @@ -1,2 +0,0 @@ -python: - enabled: true diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 852164ed..00000000 --- a/.travis.yml +++ /dev/null @@ -1,14 +0,0 @@ -language: python -python: - - "3.11" - - "3.12" - - "3.13" - - "3.14" -# command to install dependencies -install: - - "pip install -r requirements.txt" - - "pip install coveralls" -# command to run tests -script: travis_wait coverage run --source=Exscript -m unittest discover -v tests/Exscript "*Test.py" -after_success: - coveralls diff --git a/Exscript/version.py b/Exscript/version.py deleted file mode 100644 index 674111e9..00000000 --- a/Exscript/version.py +++ /dev/null @@ -1,4 +0,0 @@ -""" -Warning: This file is automatically generated. -""" -__version__ = 'DEVELOPMENT' diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index bb3ec5f0..00000000 --- a/MANIFEST.in +++ /dev/null @@ -1 +0,0 @@ -include README.md diff --git a/Makefile b/Makefile deleted file mode 100644 index 0815d420..00000000 --- a/Makefile +++ /dev/null @@ -1,72 +0,0 @@ -NAME=exscript -VERSION=`python setup.py --version | sed s/^v//` -PREFIX=/usr/local/ -BIN_DIR=$(PREFIX)/bin -SITE_DIR=$(PREFIX)`python -c "import sys; from distutils.sysconfig import get_python_lib; print(get_python_lib()[len(sys.prefix):])"` - -################################################################### -# Standard targets. -################################################################### -.PHONY : clean -clean: - find . -name "*.pyc" -o -name "*.pyo" | xargs -n1 rm -f - rm -Rf build *.egg-info - cd doc; make clean - -.PHONY : dist-clean -dist-clean: clean - rm -Rf dist - -.PHONY : doc -doc: - cd doc; make - -install: - mkdir -p $(SITE_DIR) - ./version.sh - export PYTHONPATH=$(SITE_DIR):$(PYTHONPATH); \ - python setup.py install --prefix $(PREFIX) \ - --install-scripts $(BIN_DIR) \ - --install-lib $(SITE_DIR) - ./version.sh --reset - -uninstall: - # Sorry, Python's distutils support no such action yet. - -.PHONY : tests -tests: - cd tests/Exscript/; ./run_suite.py 1 - -################################################################### -# Package builders. -################################################################### -targz: clean - ./version.sh - python setup.py sdist --formats gztar - ./version.sh --reset - -tarbz: clean - ./version.sh - python setup.py sdist --formats bztar - ./version.sh --reset - -wheel: clean - ./version.sh - python setup.py bdist_wheel --universal - ./version.sh --reset - -deb: clean - ./version.sh - debuild -S -sa - cd ..; sudo pbuilder build $(NAME)_$(VERSION)-0ubuntu1.dsc; cd - - ./version.sh --reset - -dist: targz tarbz wheel - -################################################################### -# Publishers. -################################################################### -dist-publish: - ./version.sh - twine upload dist/* - ./version.sh --reset diff --git a/TODO b/TODO deleted file mode 100644 index 4c0256c8..00000000 --- a/TODO +++ /dev/null @@ -1,5 +0,0 @@ -* Add tests for Transport class debug levels. -* Add API docs for Exscript.parselib. -* Test the exscript executable. -* Test Queue constructor kwargs. -* Create more stdlib tests like tests/templates/stdlib.crypt/. diff --git a/VERSION.in b/VERSION.in deleted file mode 100644 index 90ff5817..00000000 --- a/VERSION.in +++ /dev/null @@ -1,4 +0,0 @@ -""" -Warning: This file is automatically generated. -""" -__version__ = '@VERSION@' diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..3210dab5 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,81 @@ +[build-system] +requires = ["hatchling>=1.27", "hatch-vcs>=0.5"] +build-backend = "hatchling.build" + +[project] +name = "Exscript" +dynamic = ["version"] +description = "Automating Telnet, SSH and NETCONF" +readme = "README.md" +requires-python = ">=3.12" +license = "MIT" +license-files = ["COPYING"] +authors = [{ name = "Samuel Abels", email = "knipknap@gmail.com" }] +keywords = [ + "exscript", + "telnet", + "ssh", + "netconf", + "network", + "networking", + "automate", + "automation", + "library", +] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "Operating System :: OS Independent", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", + "Programming Language :: Python :: 3.15", + "Topic :: Security", + "Topic :: Software Development :: Libraries", + "Topic :: Software Development :: Libraries :: Python Modules", +] +dependencies = [ + "lxml", + "ncclient>=0.7,<1", + "paramiko>=3.4,<6", + "pycryptodomex", +] + +[project.optional-dependencies] +otel = [ + "opentelemetry-api>=1.12", + "opentelemetry-sdk>=1.12", + "opentelemetry-instrumentation>=0.42b0", +] + +[project.urls] +Homepage = "https://github.com/knipknap/exscript" +Documentation = "https://exscript.readthedocs.io" +Issues = "https://github.com/knipknap/exscript/issues" +Changelog = "https://github.com/knipknap/exscript/blob/master/CHANGES.md" + +[dependency-groups] +test = [ + "pytest>=9.2", + "pytest-cov>=7.1", + "opentelemetry-sdk>=1.12", + "opentelemetry-instrumentation>=0.42b0", +] +dev = [{ include-group = "test" }] + +[tool.hatch.version] +source = "vcs" + +[tool.hatch.build.targets.wheel] +packages = ["src/Exscript"] + +[tool.hatch.build.targets.wheel.shared-scripts] +"scripts/exscript" = "exscript" +"scripts/otp" = "otp" + +[tool.pytest.ini_options] +testpaths = ["tests"] +python_files = ["*Test.py"] +addopts = ["-ra", "--strict-config", "--strict-markers"] diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index d656c981..00000000 --- a/requirements.txt +++ /dev/null @@ -1,5 +0,0 @@ -configparser -lxml -ncclient>=0.7,<1 -paramiko>=2.2,<5 -pycryptodomex diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index b88034e4..00000000 --- a/setup.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[metadata] -description-file = README.md diff --git a/setup.py b/setup.py deleted file mode 100644 index 6d23f756..00000000 --- a/setup.py +++ /dev/null @@ -1,63 +0,0 @@ -import sys - -from setuptools import find_packages, setup - -sys.path.insert(0, 'Exscript') -from version import __version__ - -# Import the project description from the README. -descr = ''' -Exscript is a Python module and a template processor for automating network -connections over protocols such as Telnet or SSH/Netconf. We attempt to create -the best possible set of tools for working with Telnet and SSH/Netconf. -'''.strip() - -# Run the setup. -setup(name = 'Exscript', - version = __version__, - description = 'Automating Telnet and SSH/Netconf', - long_description = descr, - author = 'Samuel Abels', - author_email = 'knipknap@gmail.com', - license = 'MIT', - package_dir = {'Exscript': 'Exscript'}, - package_data = {}, - packages = find_packages(), - scripts = ['scripts/exscript', 'scripts/otp'], - install_requires = [ - 'configparser', - 'lxml', - 'ncclient>=0.7,<1', - 'paramiko>=2.2,<5', - 'pycryptodomex', - ], - extras_require = { - 'otel': [ - 'opentelemetry-api>=1.12', - 'opentelemetry-sdk>=1.12', - 'opentelemetry-instrumentation>=0.42b0', - ], - }, - keywords = ' '.join(['exscript', - 'telnet', - 'ssh', - 'netconf', - 'network', - 'networking', - 'automate', - 'automation', - 'library']), - url = 'https://github.com/knipknap/exscript/', - classifiers = [ - 'Development Status :: 5 - Production/Stable', - 'Intended Audience :: Developers', - 'License :: OSI Approved :: MIT License', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.11', - 'Programming Language :: Python :: 3.12', - 'Programming Language :: Python :: 3.13', - 'Programming Language :: Python :: 3.14', - 'Topic :: Security', - 'Topic :: Software Development :: Libraries', - 'Topic :: Software Development :: Libraries :: Python Modules' - ]) diff --git a/Exscript/__init__.py b/src/Exscript/__init__.py similarity index 100% rename from Exscript/__init__.py rename to src/Exscript/__init__.py diff --git a/Exscript/account.py b/src/Exscript/account.py similarity index 100% rename from Exscript/account.py rename to src/Exscript/account.py diff --git a/Exscript/contrib/__init__.py b/src/Exscript/contrib/__init__.py similarity index 100% rename from Exscript/contrib/__init__.py rename to src/Exscript/contrib/__init__.py diff --git a/Exscript/contrib/otel.py b/src/Exscript/contrib/otel.py similarity index 100% rename from Exscript/contrib/otel.py rename to src/Exscript/contrib/otel.py diff --git a/Exscript/emulators/__init__.py b/src/Exscript/emulators/__init__.py similarity index 100% rename from Exscript/emulators/__init__.py rename to src/Exscript/emulators/__init__.py diff --git a/Exscript/emulators/command.py b/src/Exscript/emulators/command.py similarity index 100% rename from Exscript/emulators/command.py rename to src/Exscript/emulators/command.py diff --git a/Exscript/emulators/iosemu.py b/src/Exscript/emulators/iosemu.py similarity index 100% rename from Exscript/emulators/iosemu.py rename to src/Exscript/emulators/iosemu.py diff --git a/Exscript/emulators/vdevice.py b/src/Exscript/emulators/vdevice.py similarity index 100% rename from Exscript/emulators/vdevice.py rename to src/Exscript/emulators/vdevice.py diff --git a/Exscript/host.py b/src/Exscript/host.py similarity index 100% rename from Exscript/host.py rename to src/Exscript/host.py diff --git a/Exscript/interpreter/__init__.py b/src/Exscript/interpreter/__init__.py similarity index 100% rename from Exscript/interpreter/__init__.py rename to src/Exscript/interpreter/__init__.py diff --git a/Exscript/interpreter/append.py b/src/Exscript/interpreter/append.py similarity index 100% rename from Exscript/interpreter/append.py rename to src/Exscript/interpreter/append.py diff --git a/Exscript/interpreter/assign.py b/src/Exscript/interpreter/assign.py similarity index 100% rename from Exscript/interpreter/assign.py rename to src/Exscript/interpreter/assign.py diff --git a/Exscript/interpreter/code.py b/src/Exscript/interpreter/code.py similarity index 100% rename from Exscript/interpreter/code.py rename to src/Exscript/interpreter/code.py diff --git a/Exscript/interpreter/enter.py b/src/Exscript/interpreter/enter.py similarity index 100% rename from Exscript/interpreter/enter.py rename to src/Exscript/interpreter/enter.py diff --git a/Exscript/interpreter/exception.py b/src/Exscript/interpreter/exception.py similarity index 100% rename from Exscript/interpreter/exception.py rename to src/Exscript/interpreter/exception.py diff --git a/Exscript/interpreter/execute.py b/src/Exscript/interpreter/execute.py similarity index 100% rename from Exscript/interpreter/execute.py rename to src/Exscript/interpreter/execute.py diff --git a/Exscript/interpreter/expression.py b/src/Exscript/interpreter/expression.py similarity index 100% rename from Exscript/interpreter/expression.py rename to src/Exscript/interpreter/expression.py diff --git a/Exscript/interpreter/extract.py b/src/Exscript/interpreter/extract.py similarity index 100% rename from Exscript/interpreter/extract.py rename to src/Exscript/interpreter/extract.py diff --git a/Exscript/interpreter/fail.py b/src/Exscript/interpreter/fail.py similarity index 100% rename from Exscript/interpreter/fail.py rename to src/Exscript/interpreter/fail.py diff --git a/Exscript/interpreter/functioncall.py b/src/Exscript/interpreter/functioncall.py similarity index 100% rename from Exscript/interpreter/functioncall.py rename to src/Exscript/interpreter/functioncall.py diff --git a/Exscript/interpreter/ifcondition.py b/src/Exscript/interpreter/ifcondition.py similarity index 100% rename from Exscript/interpreter/ifcondition.py rename to src/Exscript/interpreter/ifcondition.py diff --git a/Exscript/interpreter/loop.py b/src/Exscript/interpreter/loop.py similarity index 100% rename from Exscript/interpreter/loop.py rename to src/Exscript/interpreter/loop.py diff --git a/Exscript/interpreter/number.py b/src/Exscript/interpreter/number.py similarity index 100% rename from Exscript/interpreter/number.py rename to src/Exscript/interpreter/number.py diff --git a/Exscript/interpreter/parser.py b/src/Exscript/interpreter/parser.py similarity index 100% rename from Exscript/interpreter/parser.py rename to src/Exscript/interpreter/parser.py diff --git a/Exscript/interpreter/program.py b/src/Exscript/interpreter/program.py similarity index 100% rename from Exscript/interpreter/program.py rename to src/Exscript/interpreter/program.py diff --git a/Exscript/interpreter/regex.py b/src/Exscript/interpreter/regex.py similarity index 100% rename from Exscript/interpreter/regex.py rename to src/Exscript/interpreter/regex.py diff --git a/Exscript/interpreter/rpc.py b/src/Exscript/interpreter/rpc.py similarity index 100% rename from Exscript/interpreter/rpc.py rename to src/Exscript/interpreter/rpc.py diff --git a/Exscript/interpreter/scope.py b/src/Exscript/interpreter/scope.py similarity index 100% rename from Exscript/interpreter/scope.py rename to src/Exscript/interpreter/scope.py diff --git a/Exscript/interpreter/string.py b/src/Exscript/interpreter/string.py similarity index 100% rename from Exscript/interpreter/string.py rename to src/Exscript/interpreter/string.py diff --git a/Exscript/interpreter/template.py b/src/Exscript/interpreter/template.py similarity index 100% rename from Exscript/interpreter/template.py rename to src/Exscript/interpreter/template.py diff --git a/Exscript/interpreter/term.py b/src/Exscript/interpreter/term.py similarity index 100% rename from Exscript/interpreter/term.py rename to src/Exscript/interpreter/term.py diff --git a/Exscript/interpreter/trying.py b/src/Exscript/interpreter/trying.py similarity index 100% rename from Exscript/interpreter/trying.py rename to src/Exscript/interpreter/trying.py diff --git a/Exscript/interpreter/variable.py b/src/Exscript/interpreter/variable.py similarity index 100% rename from Exscript/interpreter/variable.py rename to src/Exscript/interpreter/variable.py diff --git a/Exscript/key.py b/src/Exscript/key.py similarity index 100% rename from Exscript/key.py rename to src/Exscript/key.py diff --git a/Exscript/logger.py b/src/Exscript/logger.py similarity index 100% rename from Exscript/logger.py rename to src/Exscript/logger.py diff --git a/Exscript/parselib/__init__.py b/src/Exscript/parselib/__init__.py similarity index 100% rename from Exscript/parselib/__init__.py rename to src/Exscript/parselib/__init__.py diff --git a/Exscript/parselib/exception.py b/src/Exscript/parselib/exception.py similarity index 100% rename from Exscript/parselib/exception.py rename to src/Exscript/parselib/exception.py diff --git a/Exscript/parselib/lexer.py b/src/Exscript/parselib/lexer.py similarity index 100% rename from Exscript/parselib/lexer.py rename to src/Exscript/parselib/lexer.py diff --git a/Exscript/parselib/token.py b/src/Exscript/parselib/token.py similarity index 100% rename from Exscript/parselib/token.py rename to src/Exscript/parselib/token.py diff --git a/Exscript/protocols/__init__.py b/src/Exscript/protocols/__init__.py similarity index 100% rename from Exscript/protocols/__init__.py rename to src/Exscript/protocols/__init__.py diff --git a/Exscript/protocols/drivers/__init__.py b/src/Exscript/protocols/drivers/__init__.py similarity index 100% rename from Exscript/protocols/drivers/__init__.py rename to src/Exscript/protocols/drivers/__init__.py diff --git a/Exscript/protocols/drivers/ace.py b/src/Exscript/protocols/drivers/ace.py similarity index 100% rename from Exscript/protocols/drivers/ace.py rename to src/Exscript/protocols/drivers/ace.py diff --git a/Exscript/protocols/drivers/adtran.py b/src/Exscript/protocols/drivers/adtran.py similarity index 100% rename from Exscript/protocols/drivers/adtran.py rename to src/Exscript/protocols/drivers/adtran.py diff --git a/Exscript/protocols/drivers/aironet.py b/src/Exscript/protocols/drivers/aironet.py similarity index 100% rename from Exscript/protocols/drivers/aironet.py rename to src/Exscript/protocols/drivers/aironet.py diff --git a/Exscript/protocols/drivers/aix.py b/src/Exscript/protocols/drivers/aix.py similarity index 100% rename from Exscript/protocols/drivers/aix.py rename to src/Exscript/protocols/drivers/aix.py diff --git a/Exscript/protocols/drivers/arbor_peakflow.py b/src/Exscript/protocols/drivers/arbor_peakflow.py similarity index 100% rename from Exscript/protocols/drivers/arbor_peakflow.py rename to src/Exscript/protocols/drivers/arbor_peakflow.py diff --git a/Exscript/protocols/drivers/aruba.py b/src/Exscript/protocols/drivers/aruba.py similarity index 100% rename from Exscript/protocols/drivers/aruba.py rename to src/Exscript/protocols/drivers/aruba.py diff --git a/Exscript/protocols/drivers/bigip.py b/src/Exscript/protocols/drivers/bigip.py similarity index 100% rename from Exscript/protocols/drivers/bigip.py rename to src/Exscript/protocols/drivers/bigip.py diff --git a/Exscript/protocols/drivers/brocade.py b/src/Exscript/protocols/drivers/brocade.py similarity index 100% rename from Exscript/protocols/drivers/brocade.py rename to src/Exscript/protocols/drivers/brocade.py diff --git a/Exscript/protocols/drivers/cienasaos.py b/src/Exscript/protocols/drivers/cienasaos.py similarity index 100% rename from Exscript/protocols/drivers/cienasaos.py rename to src/Exscript/protocols/drivers/cienasaos.py diff --git a/Exscript/protocols/drivers/driver.py b/src/Exscript/protocols/drivers/driver.py similarity index 100% rename from Exscript/protocols/drivers/driver.py rename to src/Exscript/protocols/drivers/driver.py diff --git a/Exscript/protocols/drivers/enterasys.py b/src/Exscript/protocols/drivers/enterasys.py similarity index 100% rename from Exscript/protocols/drivers/enterasys.py rename to src/Exscript/protocols/drivers/enterasys.py diff --git a/Exscript/protocols/drivers/enterasys_wc.py b/src/Exscript/protocols/drivers/enterasys_wc.py similarity index 100% rename from Exscript/protocols/drivers/enterasys_wc.py rename to src/Exscript/protocols/drivers/enterasys_wc.py diff --git a/Exscript/protocols/drivers/eos.py b/src/Exscript/protocols/drivers/eos.py similarity index 100% rename from Exscript/protocols/drivers/eos.py rename to src/Exscript/protocols/drivers/eos.py diff --git a/Exscript/protocols/drivers/ericsson_ban.py b/src/Exscript/protocols/drivers/ericsson_ban.py similarity index 100% rename from Exscript/protocols/drivers/ericsson_ban.py rename to src/Exscript/protocols/drivers/ericsson_ban.py diff --git a/Exscript/protocols/drivers/fortios.py b/src/Exscript/protocols/drivers/fortios.py similarity index 100% rename from Exscript/protocols/drivers/fortios.py rename to src/Exscript/protocols/drivers/fortios.py diff --git a/Exscript/protocols/drivers/generic.py b/src/Exscript/protocols/drivers/generic.py similarity index 100% rename from Exscript/protocols/drivers/generic.py rename to src/Exscript/protocols/drivers/generic.py diff --git a/Exscript/protocols/drivers/hp_pro_curve.py b/src/Exscript/protocols/drivers/hp_pro_curve.py similarity index 100% rename from Exscript/protocols/drivers/hp_pro_curve.py rename to src/Exscript/protocols/drivers/hp_pro_curve.py diff --git a/Exscript/protocols/drivers/icotera.py b/src/Exscript/protocols/drivers/icotera.py similarity index 100% rename from Exscript/protocols/drivers/icotera.py rename to src/Exscript/protocols/drivers/icotera.py diff --git a/Exscript/protocols/drivers/ios.py b/src/Exscript/protocols/drivers/ios.py similarity index 100% rename from Exscript/protocols/drivers/ios.py rename to src/Exscript/protocols/drivers/ios.py diff --git a/Exscript/protocols/drivers/ios_xr.py b/src/Exscript/protocols/drivers/ios_xr.py similarity index 100% rename from Exscript/protocols/drivers/ios_xr.py rename to src/Exscript/protocols/drivers/ios_xr.py diff --git a/Exscript/protocols/drivers/isam.py b/src/Exscript/protocols/drivers/isam.py similarity index 100% rename from Exscript/protocols/drivers/isam.py rename to src/Exscript/protocols/drivers/isam.py diff --git a/Exscript/protocols/drivers/junos.py b/src/Exscript/protocols/drivers/junos.py similarity index 100% rename from Exscript/protocols/drivers/junos.py rename to src/Exscript/protocols/drivers/junos.py diff --git a/Exscript/protocols/drivers/junos_erx.py b/src/Exscript/protocols/drivers/junos_erx.py similarity index 100% rename from Exscript/protocols/drivers/junos_erx.py rename to src/Exscript/protocols/drivers/junos_erx.py diff --git a/Exscript/protocols/drivers/mrv.py b/src/Exscript/protocols/drivers/mrv.py similarity index 100% rename from Exscript/protocols/drivers/mrv.py rename to src/Exscript/protocols/drivers/mrv.py diff --git a/Exscript/protocols/drivers/nxos.py b/src/Exscript/protocols/drivers/nxos.py similarity index 100% rename from Exscript/protocols/drivers/nxos.py rename to src/Exscript/protocols/drivers/nxos.py diff --git a/Exscript/protocols/drivers/one_os.py b/src/Exscript/protocols/drivers/one_os.py similarity index 100% rename from Exscript/protocols/drivers/one_os.py rename to src/Exscript/protocols/drivers/one_os.py diff --git a/Exscript/protocols/drivers/rios.py b/src/Exscript/protocols/drivers/rios.py similarity index 100% rename from Exscript/protocols/drivers/rios.py rename to src/Exscript/protocols/drivers/rios.py diff --git a/Exscript/protocols/drivers/shell.py b/src/Exscript/protocols/drivers/shell.py similarity index 100% rename from Exscript/protocols/drivers/shell.py rename to src/Exscript/protocols/drivers/shell.py diff --git a/Exscript/protocols/drivers/smart_edge_os.py b/src/Exscript/protocols/drivers/smart_edge_os.py similarity index 100% rename from Exscript/protocols/drivers/smart_edge_os.py rename to src/Exscript/protocols/drivers/smart_edge_os.py diff --git a/Exscript/protocols/drivers/sp_vxoa.py b/src/Exscript/protocols/drivers/sp_vxoa.py similarity index 100% rename from Exscript/protocols/drivers/sp_vxoa.py rename to src/Exscript/protocols/drivers/sp_vxoa.py diff --git a/Exscript/protocols/drivers/sros.py b/src/Exscript/protocols/drivers/sros.py similarity index 100% rename from Exscript/protocols/drivers/sros.py rename to src/Exscript/protocols/drivers/sros.py diff --git a/Exscript/protocols/drivers/vrp.py b/src/Exscript/protocols/drivers/vrp.py similarity index 100% rename from Exscript/protocols/drivers/vrp.py rename to src/Exscript/protocols/drivers/vrp.py diff --git a/Exscript/protocols/drivers/vxworks.py b/src/Exscript/protocols/drivers/vxworks.py similarity index 100% rename from Exscript/protocols/drivers/vxworks.py rename to src/Exscript/protocols/drivers/vxworks.py diff --git a/Exscript/protocols/drivers/zte.py b/src/Exscript/protocols/drivers/zte.py similarity index 100% rename from Exscript/protocols/drivers/zte.py rename to src/Exscript/protocols/drivers/zte.py diff --git a/Exscript/protocols/drivers/zyxel.py b/src/Exscript/protocols/drivers/zyxel.py similarity index 100% rename from Exscript/protocols/drivers/zyxel.py rename to src/Exscript/protocols/drivers/zyxel.py diff --git a/Exscript/protocols/dummy.py b/src/Exscript/protocols/dummy.py similarity index 100% rename from Exscript/protocols/dummy.py rename to src/Exscript/protocols/dummy.py diff --git a/Exscript/protocols/exception.py b/src/Exscript/protocols/exception.py similarity index 100% rename from Exscript/protocols/exception.py rename to src/Exscript/protocols/exception.py diff --git a/Exscript/protocols/netconf.py b/src/Exscript/protocols/netconf.py similarity index 100% rename from Exscript/protocols/netconf.py rename to src/Exscript/protocols/netconf.py diff --git a/Exscript/protocols/osguesser.py b/src/Exscript/protocols/osguesser.py similarity index 100% rename from Exscript/protocols/osguesser.py rename to src/Exscript/protocols/osguesser.py diff --git a/Exscript/protocols/protocol.py b/src/Exscript/protocols/protocol.py similarity index 100% rename from Exscript/protocols/protocol.py rename to src/Exscript/protocols/protocol.py diff --git a/Exscript/protocols/ssh2.py b/src/Exscript/protocols/ssh2.py similarity index 100% rename from Exscript/protocols/ssh2.py rename to src/Exscript/protocols/ssh2.py diff --git a/Exscript/protocols/telnet.py b/src/Exscript/protocols/telnet.py similarity index 100% rename from Exscript/protocols/telnet.py rename to src/Exscript/protocols/telnet.py diff --git a/Exscript/protocols/telnetlib.py b/src/Exscript/protocols/telnetlib.py similarity index 100% rename from Exscript/protocols/telnetlib.py rename to src/Exscript/protocols/telnetlib.py diff --git a/Exscript/queue.py b/src/Exscript/queue.py similarity index 100% rename from Exscript/queue.py rename to src/Exscript/queue.py diff --git a/Exscript/servers/__init__.py b/src/Exscript/servers/__init__.py similarity index 100% rename from Exscript/servers/__init__.py rename to src/Exscript/servers/__init__.py diff --git a/Exscript/servers/httpd.py b/src/Exscript/servers/httpd.py similarity index 100% rename from Exscript/servers/httpd.py rename to src/Exscript/servers/httpd.py diff --git a/Exscript/servers/server.py b/src/Exscript/servers/server.py similarity index 100% rename from Exscript/servers/server.py rename to src/Exscript/servers/server.py diff --git a/Exscript/servers/sshd.py b/src/Exscript/servers/sshd.py similarity index 100% rename from Exscript/servers/sshd.py rename to src/Exscript/servers/sshd.py diff --git a/Exscript/servers/telnetd.py b/src/Exscript/servers/telnetd.py similarity index 100% rename from Exscript/servers/telnetd.py rename to src/Exscript/servers/telnetd.py diff --git a/Exscript/stdlib/__init__.py b/src/Exscript/stdlib/__init__.py similarity index 100% rename from Exscript/stdlib/__init__.py rename to src/Exscript/stdlib/__init__.py diff --git a/Exscript/stdlib/connection.py b/src/Exscript/stdlib/connection.py similarity index 100% rename from Exscript/stdlib/connection.py rename to src/Exscript/stdlib/connection.py diff --git a/Exscript/stdlib/crypt.py b/src/Exscript/stdlib/crypt.py similarity index 100% rename from Exscript/stdlib/crypt.py rename to src/Exscript/stdlib/crypt.py diff --git a/Exscript/stdlib/file.py b/src/Exscript/stdlib/file.py similarity index 100% rename from Exscript/stdlib/file.py rename to src/Exscript/stdlib/file.py diff --git a/Exscript/stdlib/ipv4.py b/src/Exscript/stdlib/ipv4.py similarity index 100% rename from Exscript/stdlib/ipv4.py rename to src/Exscript/stdlib/ipv4.py diff --git a/Exscript/stdlib/list.py b/src/Exscript/stdlib/list.py similarity index 100% rename from Exscript/stdlib/list.py rename to src/Exscript/stdlib/list.py diff --git a/Exscript/stdlib/mysys.py b/src/Exscript/stdlib/mysys.py similarity index 100% rename from Exscript/stdlib/mysys.py rename to src/Exscript/stdlib/mysys.py diff --git a/Exscript/stdlib/string.py b/src/Exscript/stdlib/string.py similarity index 100% rename from Exscript/stdlib/string.py rename to src/Exscript/stdlib/string.py diff --git a/Exscript/stdlib/util.py b/src/Exscript/stdlib/util.py similarity index 100% rename from Exscript/stdlib/util.py rename to src/Exscript/stdlib/util.py diff --git a/Exscript/util/__init__.py b/src/Exscript/util/__init__.py similarity index 100% rename from Exscript/util/__init__.py rename to src/Exscript/util/__init__.py diff --git a/Exscript/util/buffer.py b/src/Exscript/util/buffer.py similarity index 100% rename from Exscript/util/buffer.py rename to src/Exscript/util/buffer.py diff --git a/Exscript/util/cast.py b/src/Exscript/util/cast.py similarity index 100% rename from Exscript/util/cast.py rename to src/Exscript/util/cast.py diff --git a/Exscript/util/collections.py b/src/Exscript/util/collections.py similarity index 100% rename from Exscript/util/collections.py rename to src/Exscript/util/collections.py diff --git a/Exscript/util/crypt.py b/src/Exscript/util/crypt.py similarity index 100% rename from Exscript/util/crypt.py rename to src/Exscript/util/crypt.py diff --git a/Exscript/util/daemonize.py b/src/Exscript/util/daemonize.py similarity index 100% rename from Exscript/util/daemonize.py rename to src/Exscript/util/daemonize.py diff --git a/Exscript/util/decorator.py b/src/Exscript/util/decorator.py similarity index 100% rename from Exscript/util/decorator.py rename to src/Exscript/util/decorator.py diff --git a/Exscript/util/event.py b/src/Exscript/util/event.py similarity index 100% rename from Exscript/util/event.py rename to src/Exscript/util/event.py diff --git a/Exscript/util/file.py b/src/Exscript/util/file.py similarity index 100% rename from Exscript/util/file.py rename to src/Exscript/util/file.py diff --git a/Exscript/util/impl.py b/src/Exscript/util/impl.py similarity index 100% rename from Exscript/util/impl.py rename to src/Exscript/util/impl.py diff --git a/Exscript/util/interact.py b/src/Exscript/util/interact.py similarity index 100% rename from Exscript/util/interact.py rename to src/Exscript/util/interact.py diff --git a/Exscript/util/ip.py b/src/Exscript/util/ip.py similarity index 100% rename from Exscript/util/ip.py rename to src/Exscript/util/ip.py diff --git a/Exscript/util/ipv4.py b/src/Exscript/util/ipv4.py similarity index 100% rename from Exscript/util/ipv4.py rename to src/Exscript/util/ipv4.py diff --git a/Exscript/util/ipv6.py b/src/Exscript/util/ipv6.py similarity index 100% rename from Exscript/util/ipv6.py rename to src/Exscript/util/ipv6.py diff --git a/Exscript/util/log.py b/src/Exscript/util/log.py similarity index 100% rename from Exscript/util/log.py rename to src/Exscript/util/log.py diff --git a/Exscript/util/mail.py b/src/Exscript/util/mail.py similarity index 100% rename from Exscript/util/mail.py rename to src/Exscript/util/mail.py diff --git a/Exscript/util/match.py b/src/Exscript/util/match.py similarity index 100% rename from Exscript/util/match.py rename to src/Exscript/util/match.py diff --git a/Exscript/util/pidutil.py b/src/Exscript/util/pidutil.py similarity index 100% rename from Exscript/util/pidutil.py rename to src/Exscript/util/pidutil.py diff --git a/Exscript/util/report.py b/src/Exscript/util/report.py similarity index 100% rename from Exscript/util/report.py rename to src/Exscript/util/report.py diff --git a/Exscript/util/sigint.py b/src/Exscript/util/sigint.py similarity index 100% rename from Exscript/util/sigint.py rename to src/Exscript/util/sigint.py diff --git a/Exscript/util/sigintcatcher.py b/src/Exscript/util/sigintcatcher.py similarity index 100% rename from Exscript/util/sigintcatcher.py rename to src/Exscript/util/sigintcatcher.py diff --git a/Exscript/util/start.py b/src/Exscript/util/start.py similarity index 100% rename from Exscript/util/start.py rename to src/Exscript/util/start.py diff --git a/Exscript/util/syslog.py b/src/Exscript/util/syslog.py similarity index 100% rename from Exscript/util/syslog.py rename to src/Exscript/util/syslog.py diff --git a/Exscript/util/template.py b/src/Exscript/util/template.py similarity index 100% rename from Exscript/util/template.py rename to src/Exscript/util/template.py diff --git a/Exscript/util/tty.py b/src/Exscript/util/tty.py similarity index 100% rename from Exscript/util/tty.py rename to src/Exscript/util/tty.py diff --git a/Exscript/util/url.py b/src/Exscript/util/url.py similarity index 100% rename from Exscript/util/url.py rename to src/Exscript/util/url.py diff --git a/Exscript/util/weakmethod.py b/src/Exscript/util/weakmethod.py similarity index 100% rename from Exscript/util/weakmethod.py rename to src/Exscript/util/weakmethod.py diff --git a/Exscript/util/xml.py b/src/Exscript/util/xml.py similarity index 100% rename from Exscript/util/xml.py rename to src/Exscript/util/xml.py diff --git a/src/Exscript/version.py b/src/Exscript/version.py new file mode 100644 index 00000000..f96ad9e0 --- /dev/null +++ b/src/Exscript/version.py @@ -0,0 +1,9 @@ +""" +The version of the installed Exscript distribution. +""" +from importlib.metadata import PackageNotFoundError, version + +try: + __version__ = version('Exscript') +except PackageNotFoundError: # used from a source tree without installation + __version__ = 'DEVELOPMENT' diff --git a/Exscript/workqueue/__init__.py b/src/Exscript/workqueue/__init__.py similarity index 100% rename from Exscript/workqueue/__init__.py rename to src/Exscript/workqueue/__init__.py diff --git a/Exscript/workqueue/job.py b/src/Exscript/workqueue/job.py similarity index 100% rename from Exscript/workqueue/job.py rename to src/Exscript/workqueue/job.py diff --git a/Exscript/workqueue/mainloop.py b/src/Exscript/workqueue/mainloop.py similarity index 100% rename from Exscript/workqueue/mainloop.py rename to src/Exscript/workqueue/mainloop.py diff --git a/Exscript/workqueue/pipeline.py b/src/Exscript/workqueue/pipeline.py similarity index 100% rename from Exscript/workqueue/pipeline.py rename to src/Exscript/workqueue/pipeline.py diff --git a/Exscript/workqueue/task.py b/src/Exscript/workqueue/task.py similarity index 100% rename from Exscript/workqueue/task.py rename to src/Exscript/workqueue/task.py diff --git a/Exscript/workqueue/workqueue.py b/src/Exscript/workqueue/workqueue.py similarity index 100% rename from Exscript/workqueue/workqueue.py rename to src/Exscript/workqueue/workqueue.py diff --git a/tests/Exscript/AccountManagerTest.py b/tests/Exscript/AccountManagerTest.py index 30cb9315..71c9be06 100644 --- a/tests/Exscript/AccountManagerTest.py +++ b/tests/Exscript/AccountManagerTest.py @@ -1,9 +1,4 @@ -import sys import unittest -import re -import os.path -import warnings -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..')) from Exscript.account import Account, AccountPool, AccountManager diff --git a/tests/Exscript/AccountPoolTest.py b/tests/Exscript/AccountPoolTest.py index a4cc80bb..beee1b5b 100644 --- a/tests/Exscript/AccountPoolTest.py +++ b/tests/Exscript/AccountPoolTest.py @@ -1,9 +1,6 @@ import os.path -import re -import sys import unittest -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..')) from Exscript import Account from Exscript.account import AccountPool diff --git a/tests/Exscript/AccountTest.py b/tests/Exscript/AccountTest.py index 346f0c04..132430e2 100644 --- a/tests/Exscript/AccountTest.py +++ b/tests/Exscript/AccountTest.py @@ -1,8 +1,4 @@ -import sys import unittest -import re -import os.path -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..')) from Exscript import Account, PrivateKey diff --git a/tests/Exscript/FileLoggerTest.py b/tests/Exscript/FileLoggerTest.py index b2ba1e6c..ec91da55 100644 --- a/tests/Exscript/FileLoggerTest.py +++ b/tests/Exscript/FileLoggerTest.py @@ -1,12 +1,9 @@ -import sys import unittest -import re import os.path -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..')) from tempfile import mkdtemp from shutil import rmtree -from Exscript import Host, FileLogger +from Exscript import FileLogger from LoggerTest import LoggerTest, FakeJob diff --git a/tests/Exscript/HostTest.py b/tests/Exscript/HostTest.py index 626ac6e5..4c5a4578 100644 --- a/tests/Exscript/HostTest.py +++ b/tests/Exscript/HostTest.py @@ -1,9 +1,5 @@ -import os.path -import re -import sys import unittest -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..')) from util.urlTest import urls diff --git a/tests/Exscript/KnownIssuesTest.py b/tests/Exscript/KnownIssuesTest.py index a8ea3661..37cc2a9e 100644 --- a/tests/Exscript/KnownIssuesTest.py +++ b/tests/Exscript/KnownIssuesTest.py @@ -16,7 +16,6 @@ import uuid from unittest import mock -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..')) import paramiko diff --git a/tests/Exscript/LexerTest.py b/tests/Exscript/LexerTest.py index 9dc43103..1784a1ba 100644 --- a/tests/Exscript/LexerTest.py +++ b/tests/Exscript/LexerTest.py @@ -1,8 +1,5 @@ -import os -import sys import unittest -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..')) from Exscript.parselib import Lexer diff --git a/tests/Exscript/LogTest.py b/tests/Exscript/LogTest.py index bda761cb..0c0bf565 100644 --- a/tests/Exscript/LogTest.py +++ b/tests/Exscript/LogTest.py @@ -1,13 +1,9 @@ -import os.path -import re import sys import unittest -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..')) from util.reportTest import FakeError -from Exscript import Host from Exscript.logger import Log diff --git a/tests/Exscript/LogfileTest.py b/tests/Exscript/LogfileTest.py index 33609f54..10e38389 100644 --- a/tests/Exscript/LogfileTest.py +++ b/tests/Exscript/LogfileTest.py @@ -1,9 +1,6 @@ import os.path -import re -import sys import unittest -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..')) from shutil import rmtree from tempfile import mkdtemp diff --git a/tests/Exscript/LoggerTest.py b/tests/Exscript/LoggerTest.py index cff696cc..332f13a2 100644 --- a/tests/Exscript/LoggerTest.py +++ b/tests/Exscript/LoggerTest.py @@ -1,14 +1,8 @@ -import os -import re import sys import unittest -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..')) -import gc from itertools import islice -from shutil import rmtree -from tempfile import mkdtemp from LogTest import FakeError from util.reportTest import FakeJob diff --git a/tests/Exscript/PrivateKeyTest.py b/tests/Exscript/PrivateKeyTest.py index 892c8f89..ce554bb0 100644 --- a/tests/Exscript/PrivateKeyTest.py +++ b/tests/Exscript/PrivateKeyTest.py @@ -1,8 +1,5 @@ -import os.path -import sys import unittest -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..')) from Exscript import PrivateKey diff --git a/tests/Exscript/QueueTest.py b/tests/Exscript/QueueTest.py index 0842bcac..e5a318f2 100644 --- a/tests/Exscript/QueueTest.py +++ b/tests/Exscript/QueueTest.py @@ -1,11 +1,8 @@ import logging import os.path -import re -import sys import unittest import warnings -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..')) warnings.simplefilter('ignore', DeprecationWarning) diff --git a/tests/Exscript/TemplateTest.py b/tests/Exscript/TemplateTest.py index cce7f019..c9fd9584 100644 --- a/tests/Exscript/TemplateTest.py +++ b/tests/Exscript/TemplateTest.py @@ -1,10 +1,7 @@ import os -import re -import sys import unittest dirname = os.path.dirname(__file__) -sys.path.insert(0, os.path.join(dirname, '..', '..')) from Exscript import Account, Host, Logger, Queue, protocols from Exscript.emulators import IOSEmulator diff --git a/tests/Exscript/emulators/CommandSetTest.py b/tests/Exscript/emulators/CommandSetTest.py index 68464eef..735c9fa3 100644 --- a/tests/Exscript/emulators/CommandSetTest.py +++ b/tests/Exscript/emulators/CommandSetTest.py @@ -1,8 +1,4 @@ -import sys import unittest -import re -import os.path -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..', '..')) from Exscript.emulators import CommandSet diff --git a/tests/Exscript/emulators/IOSEmulatorTest.py b/tests/Exscript/emulators/IOSEmulatorTest.py index f5fe5857..b43cc05c 100644 --- a/tests/Exscript/emulators/IOSEmulatorTest.py +++ b/tests/Exscript/emulators/IOSEmulatorTest.py @@ -1,9 +1,5 @@ -import os.path -import re -import sys import unittest -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..', '..')) from Exscript.emulators import IOSEmulator from Exscript.emulators.iosemu import iosbanner diff --git a/tests/Exscript/emulators/VirtualDeviceTest.py b/tests/Exscript/emulators/VirtualDeviceTest.py index 5152a2bc..0c3ae139 100644 --- a/tests/Exscript/emulators/VirtualDeviceTest.py +++ b/tests/Exscript/emulators/VirtualDeviceTest.py @@ -1,8 +1,4 @@ -import sys import unittest -import re -import os.path -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..', '..')) from Exscript.emulators import VirtualDevice diff --git a/tests/Exscript/memtest.py b/tests/Exscript/memtest.py deleted file mode 100644 index dabdd673..00000000 --- a/tests/Exscript/memtest.py +++ /dev/null @@ -1,124 +0,0 @@ -# This script is not meant to provide a fully automated test, it's -# merely a hack/starting point for investigating memory consumption -# manually. The behavior also depends heavily on the version of meliae. -import gc -import re - -from Exscript import Account, Host, Queue -from Exscript.protocols import connect -from Exscript.util.decorator import bind - -objnames = ('count_calls',) -follow_modules = False - - -def count_calls(conn, thedata, **kwargs): - thedata['n_calls'] += 1 - - -def foobar(): - pass - -queue = Queue() -data = {'n_calls': 0} -func = bind(count_calls, data) -task = queue.run(['t1', 't2', 't3', 't4', 't5', 't6'], func) -task.wait() -queue.shutdown() -queue.destroy() - -del func - -# Test memory consumption. -from meliae import scanner - -gc.collect() -scanner.dump_all_objects("test.dump") -from meliae import loader - -om = loader.load('test.dump') -om.remove_expensive_references() -om.collapse_instance_dicts() -om.compute_referrers() -om.compute_total_size() -# print(om.summarize()) - -from pprint import pprint as pp - - -def larger(x, y): - return om[y].total_size - om[x].total_size - - -def larger(x, y): - return int(y.total_size - x.total_size) - -objs = sorted(iter(list(om.objs.values())), larger) - - -def is_builtin(o): - return o.type_str in ('builtin_function_or_method',) - - -def contains_builtin(reflist): - for ref in reflist: - if is_builtin(om[ref]): - return True - return False - - -def is_basic_type(o): - return o.type_str in ('int', 'str', 'bool', 'NoneType') - - -def print_obj(lbl, o, indent=0): - print((" " * indent) + lbl, o.address, - o.type_str, o.name, o.total_size, o.referrers) - - -def print_ref(o, indent=0, done=None): - if o.type_str == '': - return - if not is_basic_type(o): - print_obj('Ref:', o, indent) - # for ref in o.referrers: - # print_obj_recursive(om[ref], indent + 1, done) - - -def print_obj_recursive(o, indent=0, done=None): - if o.type_str == 'frame': - print_obj('Frame:', o, indent) - return - if o.type_str == 'module' and not follow_modules: - print_obj('Module:', o, indent) - return - - if done is None: - done = set() - elif o.address in done: - print_obj('Did that:', o, indent) - return - done.add(o.address) - - builtin = contains_builtin(o.ref_list) - if builtin: - print_obj('Builtin:', o, indent) - else: - print_obj('Obj:', o, indent) - - for ref in o.referrers: - child = om[ref] - print_obj_recursive(child, indent + 1, done) - - if not builtin: - for ref in o.ref_list: - print_ref(om[ref], indent + 1, done) - -for obj in objs: - if obj.type_str in objnames: - print_obj_recursive(obj) - if obj.name and obj.name in objnames: - print_obj_recursive(obj) - -# for addr in sorted(set(thedict.ref_list), larger): -# print(om[addr]) diff --git a/tests/Exscript/memtest_host.py b/tests/Exscript/memtest_host.py deleted file mode 100644 index a4a8518a..00000000 --- a/tests/Exscript/memtest_host.py +++ /dev/null @@ -1,13 +0,0 @@ -# This script is not meant to provide a fully automated test, it's -# merely a hack/starting point for investigating memory consumption -# manually. The behavior also depends heavily on the version of meliae. -from meliae import loader, scanner - -from Exscript import Account, Host - -hostlist = [Host(str(i)) for i in range(1, 10000)] -# accountlist = [Account(str(i)) for i in range(1, 10000)] - -scanner.dump_all_objects('test.dump') -om = loader.load('test.dump') -print(om.summarize()) diff --git a/tests/Exscript/protocols/DummyTest.py b/tests/Exscript/protocols/DummyTest.py index 1211ae30..cdf547ee 100644 --- a/tests/Exscript/protocols/DummyTest.py +++ b/tests/Exscript/protocols/DummyTest.py @@ -1,9 +1,6 @@ -import os.path import re -import sys import unittest -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..', '..')) from Exscript.emulators import VirtualDevice from Exscript.protocols import Dummy diff --git a/tests/Exscript/protocols/OsGuesserCorpusTest.py b/tests/Exscript/protocols/OsGuesserCorpusTest.py index da9051a9..911b059b 100644 --- a/tests/Exscript/protocols/OsGuesserCorpusTest.py +++ b/tests/Exscript/protocols/OsGuesserCorpusTest.py @@ -13,10 +13,8 @@ """ import json import os -import sys import unittest -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..', '..')) from Exscript.protocols.osguesser import OsGuesser diff --git a/tests/Exscript/protocols/OsGuesserTest.py b/tests/Exscript/protocols/OsGuesserTest.py index 0610672b..dcec0381 100644 --- a/tests/Exscript/protocols/OsGuesserTest.py +++ b/tests/Exscript/protocols/OsGuesserTest.py @@ -1,9 +1,7 @@ import os import re -import sys import unittest -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..', '..')) from Exscript.protocols import drivers from Exscript.protocols.osguesser import OsGuesser diff --git a/tests/Exscript/protocols/ProtocolTest.py b/tests/Exscript/protocols/ProtocolTest.py index fe7ed323..9e47ab19 100644 --- a/tests/Exscript/protocols/ProtocolTest.py +++ b/tests/Exscript/protocols/ProtocolTest.py @@ -1,11 +1,9 @@ import os import re -import sys import tempfile import unittest import unittest.mock -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..', '..')) import time from functools import partial diff --git a/tests/Exscript/protocols/SSH2Test.py b/tests/Exscript/protocols/SSH2Test.py index 0316690c..472b14e1 100644 --- a/tests/Exscript/protocols/SSH2Test.py +++ b/tests/Exscript/protocols/SSH2Test.py @@ -1,9 +1,6 @@ import os.path -import re -import sys import unittest -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..', '..')) from Exscript import PrivateKey from Exscript.protocols import SSH2 diff --git a/tests/Exscript/protocols/TelnetTest.py b/tests/Exscript/protocols/TelnetTest.py index d0687c9f..781257d8 100644 --- a/tests/Exscript/protocols/TelnetTest.py +++ b/tests/Exscript/protocols/TelnetTest.py @@ -1,12 +1,6 @@ -import os -import re -import sys -import time import unittest -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..')) -from Exscript import Account from Exscript.protocols import Telnet from Exscript.servers import Telnetd diff --git a/tests/Exscript/servers/SSHdTest.py b/tests/Exscript/servers/SSHdTest.py index e5e4125a..73658ed4 100644 --- a/tests/Exscript/servers/SSHdTest.py +++ b/tests/Exscript/servers/SSHdTest.py @@ -1,9 +1,5 @@ -import os -import re -import sys import unittest -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..')) from Exscript.protocols import SSH2 from Exscript.servers import SSHd diff --git a/tests/Exscript/servers/ServerTest.py b/tests/Exscript/servers/ServerTest.py index 7c473fbf..b24eb152 100644 --- a/tests/Exscript/servers/ServerTest.py +++ b/tests/Exscript/servers/ServerTest.py @@ -1,9 +1,6 @@ -import os import re -import sys import unittest -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..')) import time diff --git a/tests/Exscript/servers/TelnetdTest.py b/tests/Exscript/servers/TelnetdTest.py index c26f26d9..27e2d80e 100644 --- a/tests/Exscript/servers/TelnetdTest.py +++ b/tests/Exscript/servers/TelnetdTest.py @@ -1,9 +1,5 @@ -import os -import re -import sys import unittest -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..')) from Exscript.protocols import Telnet from Exscript.servers import Telnetd diff --git a/tests/Exscript/util/bufferTest.py b/tests/Exscript/util/bufferTest.py index 9fb2c3d6..2e3aa96d 100644 --- a/tests/Exscript/util/bufferTest.py +++ b/tests/Exscript/util/bufferTest.py @@ -1,9 +1,5 @@ -import os -import re -import sys import unittest -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..', '..')) from functools import partial from tempfile import TemporaryFile diff --git a/tests/Exscript/util/castTest.py b/tests/Exscript/util/castTest.py index fb61843e..ca2a82eb 100644 --- a/tests/Exscript/util/castTest.py +++ b/tests/Exscript/util/castTest.py @@ -1,13 +1,9 @@ -import sys import unittest import re -import os.path -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..', '..')) import Exscript.util.cast import re from Exscript import Host -from Exscript.logger import Log, Logfile class castTest(unittest.TestCase): diff --git a/tests/Exscript/util/cryptTest.py b/tests/Exscript/util/cryptTest.py index 37ac2239..d0dfeddd 100644 --- a/tests/Exscript/util/cryptTest.py +++ b/tests/Exscript/util/cryptTest.py @@ -1,8 +1,4 @@ -import sys import unittest -import re -import os.path -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..', '..')) import Exscript.util.crypt diff --git a/tests/Exscript/util/decoratorTest.py b/tests/Exscript/util/decoratorTest.py index 16b4bc12..e80c0439 100644 --- a/tests/Exscript/util/decoratorTest.py +++ b/tests/Exscript/util/decoratorTest.py @@ -1,18 +1,11 @@ -import os.path -import re -import sys import unittest -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..', '..')) from functools import partial from multiprocessing import Value import Exscript.util.decorator -from Exscript import Account, Host -from Exscript.protocols import Protocol from Exscript.protocols.exception import LoginFailure -from Exscript.util.impl import get_label from util.reportTest import FakeJob diff --git a/tests/Exscript/util/eventTest.py b/tests/Exscript/util/eventTest.py index 6ab4e1cd..caef5291 100644 --- a/tests/Exscript/util/eventTest.py +++ b/tests/Exscript/util/eventTest.py @@ -1,8 +1,4 @@ -import sys import unittest -import re -import os -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..', '..')) from Exscript.util.event import Event diff --git a/tests/Exscript/util/fileTest.py b/tests/Exscript/util/fileTest.py index 7d2de588..5a5828da 100644 --- a/tests/Exscript/util/fileTest.py +++ b/tests/Exscript/util/fileTest.py @@ -1,8 +1,5 @@ -import sys import unittest -import re import os.path -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..', '..')) import base64 import Exscript.util.file diff --git a/tests/Exscript/util/interactTest.py b/tests/Exscript/util/interactTest.py index 8ebf901a..bf583ac2 100644 --- a/tests/Exscript/util/interactTest.py +++ b/tests/Exscript/util/interactTest.py @@ -1,8 +1,4 @@ -import sys import unittest -import re -import os.path -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..', '..')) from tempfile import NamedTemporaryFile import Exscript.util.interact @@ -50,23 +46,23 @@ class interactTest(unittest.TestCase): CORRELATE = Exscript.util.interact def testPrompt(self): - from Exscript.util.interact import prompt + pass # Can't really be tested, as it is interactive. def testGetFilename(self): - from Exscript.util.interact import get_filename + pass # Can't really be tested, as it is interactive. def testGetUser(self): - from Exscript.util.interact import get_user + pass # Can't really be tested, as it is interactive. def testGetLogin(self): - from Exscript.util.interact import get_login + pass # Can't really be tested, as it is interactive. def testReadLogin(self): - from Exscript.util.interact import read_login + pass # Can't really be tested, as it is interactive. diff --git a/tests/Exscript/util/ipTest.py b/tests/Exscript/util/ipTest.py index 225721d7..5e9c7985 100644 --- a/tests/Exscript/util/ipTest.py +++ b/tests/Exscript/util/ipTest.py @@ -1,8 +1,4 @@ -import sys import unittest -import re -import os.path -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..', '..')) import Exscript.util.ip diff --git a/tests/Exscript/util/ipv4Test.py b/tests/Exscript/util/ipv4Test.py index f531f76f..8c92aa9e 100644 --- a/tests/Exscript/util/ipv4Test.py +++ b/tests/Exscript/util/ipv4Test.py @@ -1,9 +1,5 @@ -import os.path -import re -import sys import unittest -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..', '..')) import Exscript.util.ipv4 diff --git a/tests/Exscript/util/ipv6Test.py b/tests/Exscript/util/ipv6Test.py index ecc8db24..2c68f253 100644 --- a/tests/Exscript/util/ipv6Test.py +++ b/tests/Exscript/util/ipv6Test.py @@ -1,8 +1,4 @@ -import sys import unittest -import re -import os.path -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..', '..')) import Exscript.util.ipv6 diff --git a/tests/Exscript/util/mailTest.py b/tests/Exscript/util/mailTest.py index c3972824..890483f5 100644 --- a/tests/Exscript/util/mailTest.py +++ b/tests/Exscript/util/mailTest.py @@ -1,9 +1,5 @@ -import sys import unittest -import re -import os.path import tempfile -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..', '..')) from getpass import getuser import Exscript.util.mail diff --git a/tests/Exscript/util/matchTest.py b/tests/Exscript/util/matchTest.py index ba74901a..a9a17fee 100644 --- a/tests/Exscript/util/matchTest.py +++ b/tests/Exscript/util/matchTest.py @@ -1,8 +1,4 @@ -import sys import unittest -import re -import os.path -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..', '..')) import Exscript.util.match diff --git a/tests/Exscript/util/reportTest.py b/tests/Exscript/util/reportTest.py index 4e19b130..53df752c 100644 --- a/tests/Exscript/util/reportTest.py +++ b/tests/Exscript/util/reportTest.py @@ -1,15 +1,12 @@ import os -import re import sys import unittest -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..', '..')) import traceback import Exscript.util.report from Exscript import Host, Logger -from Exscript.util.event import Event from Exscript.workqueue import WorkQueue @@ -95,7 +92,7 @@ def testFormat(self): --------------- fake2: Traceback (most recent call last): - File "%s.py", line 54, in createAbortedLog + File "%s.py", line %d, in createAbortedLog raise FakeError() FakeError @@ -103,7 +100,7 @@ def testFormat(self): Successful actions: ------------------- fake1 -fake3'''.strip() % file +fake3'''.strip() % (file, self.createAbortedLog.__code__.co_firstlineno + 3) expected = expected.replace('FakeError\n', 'util.reportTest.FakeError\n') self.assertEqual(format(self.logger), expected) diff --git a/tests/Exscript/util/startTest.py b/tests/Exscript/util/startTest.py index b184fcd0..327bed04 100644 --- a/tests/Exscript/util/startTest.py +++ b/tests/Exscript/util/startTest.py @@ -1,8 +1,4 @@ -import sys import unittest -import re -import os.path -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..', '..')) import Exscript import Exscript.util.start diff --git a/tests/Exscript/util/syslogTest.py b/tests/Exscript/util/syslogTest.py index ec72a55a..77e5e2b9 100644 --- a/tests/Exscript/util/syslogTest.py +++ b/tests/Exscript/util/syslogTest.py @@ -1,7 +1,4 @@ -import sys import unittest -import os.path -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..', '..')) import Exscript.util.syslog @@ -10,7 +7,7 @@ class castTest(unittest.TestCase): CORRELATE = Exscript.util.syslog def testNetlog(self): - from Exscript.util.syslog import netlog + pass # NOTE: No idea how to test netlog. # How to open a local syslog server? Mocking? diff --git a/tests/Exscript/util/ttyTest.py b/tests/Exscript/util/ttyTest.py index 33abeb0b..a0e0fe10 100644 --- a/tests/Exscript/util/ttyTest.py +++ b/tests/Exscript/util/ttyTest.py @@ -1,12 +1,9 @@ import os -import re import sys import unittest -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..', '..')) import tempfile -from subprocess import PIPE, Popen import Exscript.util.tty diff --git a/tests/Exscript/util/urlTest.py b/tests/Exscript/util/urlTest.py index b9594625..aaeb85ae 100644 --- a/tests/Exscript/util/urlTest.py +++ b/tests/Exscript/util/urlTest.py @@ -1,9 +1,5 @@ -import os.path -import re -import sys import unittest -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..', '..')) from Exscript.util.url import Url diff --git a/tests/Exscript/util/weakmethodTest.py b/tests/Exscript/util/weakmethodTest.py index f2ef6f95..9b454116 100644 --- a/tests/Exscript/util/weakmethodTest.py +++ b/tests/Exscript/util/weakmethodTest.py @@ -1,9 +1,5 @@ -import os -import re -import sys import unittest -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..', '..')) from Exscript.util.weakmethod import DeadMethodCalled, WeakMethod, ref diff --git a/tests/Exscript/workqueue/JobTest.py b/tests/Exscript/workqueue/JobTest.py index 5e948928..4f5ca020 100644 --- a/tests/Exscript/workqueue/JobTest.py +++ b/tests/Exscript/workqueue/JobTest.py @@ -1,14 +1,8 @@ -import os.path -import re -import sys -import threading import unittest -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..', '..')) from multiprocessing import Pipe from pickle import dumps, loads -from tempfile import NamedTemporaryFile from Exscript.workqueue.job import Job, Process, Thread diff --git a/tests/Exscript/workqueue/MainLoopTest.py b/tests/Exscript/workqueue/MainLoopTest.py index 2c5effe4..35077462 100644 --- a/tests/Exscript/workqueue/MainLoopTest.py +++ b/tests/Exscript/workqueue/MainLoopTest.py @@ -1,10 +1,6 @@ -import os.path -import re -import sys import threading import unittest -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..', '..')) from Exscript.workqueue.job import Process from Exscript.workqueue.mainloop import MainLoop diff --git a/tests/Exscript/workqueue/PipelineTest.py b/tests/Exscript/workqueue/PipelineTest.py index 472c5ea0..8149c319 100644 --- a/tests/Exscript/workqueue/PipelineTest.py +++ b/tests/Exscript/workqueue/PipelineTest.py @@ -1,11 +1,6 @@ -import os.path -import re -import sys -import threading import time import unittest -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..', '..')) import time from multiprocessing import Value diff --git a/tests/Exscript/workqueue/TaskTest.py b/tests/Exscript/workqueue/TaskTest.py index aca5614e..0f13f288 100644 --- a/tests/Exscript/workqueue/TaskTest.py +++ b/tests/Exscript/workqueue/TaskTest.py @@ -1,9 +1,5 @@ -import sys import unittest -import re -import os.path import warnings -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..')) warnings.simplefilter('ignore', DeprecationWarning) diff --git a/tests/Exscript/workqueue/WorkQueueTest.py b/tests/Exscript/workqueue/WorkQueueTest.py index 4c15da43..5c1b3630 100644 --- a/tests/Exscript/workqueue/WorkQueueTest.py +++ b/tests/Exscript/workqueue/WorkQueueTest.py @@ -1,11 +1,6 @@ -import os.path -import re -import sys -import threading import time import unittest -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..', '..')) import random import time diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 00000000..87140bf5 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,16 @@ +"""pytest configuration for the unittest-based suite.""" +import inspect +import unittest + + +def pytest_pycollect_makeitem(collector, name, obj): + """Collects TestCase classes only in the module that defines them. + + Several test modules import a base TestCase (e.g. ProtocolTest) to derive + from it; without this hook pytest would run the base tests once more for + every importing module. + """ + if (inspect.isclass(obj) and issubclass(obj, unittest.TestCase) + and obj.__module__ != collector.module.__name__): + return [] + return None diff --git a/tools/bench_bulk.py b/tools/bench_bulk.py index 319abd6f..2f0fd0df 100755 --- a/tools/bench_bulk.py +++ b/tools/bench_bulk.py @@ -10,14 +10,12 @@ """ import argparse import json -import os import socket import statistics import sys import threading import time -sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), '..')) import paramiko diff --git a/tox.ini b/tox.ini deleted file mode 100644 index e4c1d7e4..00000000 --- a/tox.ini +++ /dev/null @@ -1,17 +0,0 @@ -# Tox (http://tox.testrun.org/) is a tool for running tests -# in multiple virtualenvs. This configuration file will run the -# test suite on all supported python versions. To use it, "pip install tox" -# and then run "tox" from this directory. - -[tox] -envlist = py311, py312, py313, py314 - -[testenv] -changedir = tests/Exscript -commands = {envpython} ./run_suite.py 1 -deps = - configparser - lxml - ncclient>=0.7,<1 - paramiko>=2.2,<5 - pycryptodomex diff --git a/uv.lock b/uv.lock new file mode 100644 index 00000000..a835f7e4 --- /dev/null +++ b/uv.lock @@ -0,0 +1,824 @@ +version = 1 +revision = 5 +requires-python = ">=3.12" + +[[package]] +name = "bcrypt" +version = "5.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d4/36/3329e2518d70ad8e2e5817d5a4cac6bba05a47767ec416c7d020a965f408/bcrypt-5.0.0.tar.gz", hash = "sha256:f748f7c2d6fd375cc93d3fba7ef4a9e3a092421b8dbf34d8d4dc06be9492dfdd", size = 25386, upload-time = "2025-09-25T19:50:47.829Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/13/85/3e65e01985fddf25b64ca67275bb5bdb4040bd1a53b66d355c6c37c8a680/bcrypt-5.0.0-cp313-cp313t-macosx_10_12_universal2.whl", hash = "sha256:f3c08197f3039bec79cee59a606d62b96b16669cff3949f21e74796b6e3cd2be", size = 481806, upload-time = "2025-09-25T19:49:05.102Z" }, + { url = "https://files.pythonhosted.org/packages/44/dc/01eb79f12b177017a726cbf78330eb0eb442fae0e7b3dfd84ea2849552f3/bcrypt-5.0.0-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:200af71bc25f22006f4069060c88ed36f8aa4ff7f53e67ff04d2ab3f1e79a5b2", size = 268626, upload-time = "2025-09-25T19:49:06.723Z" }, + { url = "https://files.pythonhosted.org/packages/8c/cf/e82388ad5959c40d6afd94fb4743cc077129d45b952d46bdc3180310e2df/bcrypt-5.0.0-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:baade0a5657654c2984468efb7d6c110db87ea63ef5a4b54732e7e337253e44f", size = 271853, upload-time = "2025-09-25T19:49:08.028Z" }, + { url = "https://files.pythonhosted.org/packages/ec/86/7134b9dae7cf0efa85671651341f6afa695857fae172615e960fb6a466fa/bcrypt-5.0.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:c58b56cdfb03202b3bcc9fd8daee8e8e9b6d7e3163aa97c631dfcfcc24d36c86", size = 269793, upload-time = "2025-09-25T19:49:09.727Z" }, + { url = "https://files.pythonhosted.org/packages/cc/82/6296688ac1b9e503d034e7d0614d56e80c5d1a08402ff856a4549cb59207/bcrypt-5.0.0-cp313-cp313t-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:4bfd2a34de661f34d0bda43c3e4e79df586e4716ef401fe31ea39d69d581ef23", size = 289930, upload-time = "2025-09-25T19:49:11.204Z" }, + { url = "https://files.pythonhosted.org/packages/d1/18/884a44aa47f2a3b88dd09bc05a1e40b57878ecd111d17e5bba6f09f8bb77/bcrypt-5.0.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:ed2e1365e31fc73f1825fa830f1c8f8917ca1b3ca6185773b349c20fd606cec2", size = 272194, upload-time = "2025-09-25T19:49:12.524Z" }, + { url = "https://files.pythonhosted.org/packages/0e/8f/371a3ab33c6982070b674f1788e05b656cfbf5685894acbfef0c65483a59/bcrypt-5.0.0-cp313-cp313t-manylinux_2_34_aarch64.whl", hash = "sha256:83e787d7a84dbbfba6f250dd7a5efd689e935f03dd83b0f919d39349e1f23f83", size = 269381, upload-time = "2025-09-25T19:49:14.308Z" }, + { url = "https://files.pythonhosted.org/packages/b1/34/7e4e6abb7a8778db6422e88b1f06eb07c47682313997ee8a8f9352e5a6f1/bcrypt-5.0.0-cp313-cp313t-manylinux_2_34_x86_64.whl", hash = "sha256:137c5156524328a24b9fac1cb5db0ba618bc97d11970b39184c1d87dc4bf1746", size = 271750, upload-time = "2025-09-25T19:49:15.584Z" }, + { url = "https://files.pythonhosted.org/packages/c0/1b/54f416be2499bd72123c70d98d36c6cd61a4e33d9b89562c22481c81bb30/bcrypt-5.0.0-cp313-cp313t-musllinux_1_1_aarch64.whl", hash = "sha256:38cac74101777a6a7d3b3e3cfefa57089b5ada650dce2baf0cbdd9d65db22a9e", size = 303757, upload-time = "2025-09-25T19:49:17.244Z" }, + { url = "https://files.pythonhosted.org/packages/13/62/062c24c7bcf9d2826a1a843d0d605c65a755bc98002923d01fd61270705a/bcrypt-5.0.0-cp313-cp313t-musllinux_1_1_x86_64.whl", hash = "sha256:d8d65b564ec849643d9f7ea05c6d9f0cd7ca23bdd4ac0c2dbef1104ab504543d", size = 306740, upload-time = "2025-09-25T19:49:18.693Z" }, + { url = "https://files.pythonhosted.org/packages/d5/c8/1fdbfc8c0f20875b6b4020f3c7dc447b8de60aa0be5faaf009d24242aec9/bcrypt-5.0.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:741449132f64b3524e95cd30e5cd3343006ce146088f074f31ab26b94e6c75ba", size = 334197, upload-time = "2025-09-25T19:49:20.523Z" }, + { url = "https://files.pythonhosted.org/packages/a6/c1/8b84545382d75bef226fbc6588af0f7b7d095f7cd6a670b42a86243183cd/bcrypt-5.0.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:212139484ab3207b1f0c00633d3be92fef3c5f0af17cad155679d03ff2ee1e41", size = 352974, upload-time = "2025-09-25T19:49:22.254Z" }, + { url = "https://files.pythonhosted.org/packages/10/a6/ffb49d4254ed085e62e3e5dd05982b4393e32fe1e49bb1130186617c29cd/bcrypt-5.0.0-cp313-cp313t-win32.whl", hash = "sha256:9d52ed507c2488eddd6a95bccee4e808d3234fa78dd370e24bac65a21212b861", size = 148498, upload-time = "2025-09-25T19:49:24.134Z" }, + { url = "https://files.pythonhosted.org/packages/48/a9/259559edc85258b6d5fc5471a62a3299a6aa37a6611a169756bf4689323c/bcrypt-5.0.0-cp313-cp313t-win_amd64.whl", hash = "sha256:f6984a24db30548fd39a44360532898c33528b74aedf81c26cf29c51ee47057e", size = 145853, upload-time = "2025-09-25T19:49:25.702Z" }, + { url = "https://files.pythonhosted.org/packages/2d/df/9714173403c7e8b245acf8e4be8876aac64a209d1b392af457c79e60492e/bcrypt-5.0.0-cp313-cp313t-win_arm64.whl", hash = "sha256:9fffdb387abe6aa775af36ef16f55e318dcda4194ddbf82007a6f21da29de8f5", size = 139626, upload-time = "2025-09-25T19:49:26.928Z" }, + { url = "https://files.pythonhosted.org/packages/f8/14/c18006f91816606a4abe294ccc5d1e6f0e42304df5a33710e9e8e95416e1/bcrypt-5.0.0-cp314-cp314t-macosx_10_12_universal2.whl", hash = "sha256:4870a52610537037adb382444fefd3706d96d663ac44cbb2f37e3919dca3d7ef", size = 481862, upload-time = "2025-09-25T19:49:28.365Z" }, + { url = "https://files.pythonhosted.org/packages/67/49/dd074d831f00e589537e07a0725cf0e220d1f0d5d8e85ad5bbff251c45aa/bcrypt-5.0.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:48f753100931605686f74e27a7b49238122aa761a9aefe9373265b8b7aa43ea4", size = 268544, upload-time = "2025-09-25T19:49:30.39Z" }, + { url = "https://files.pythonhosted.org/packages/f5/91/50ccba088b8c474545b034a1424d05195d9fcbaaf802ab8bfe2be5a4e0d7/bcrypt-5.0.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f70aadb7a809305226daedf75d90379c397b094755a710d7014b8b117df1ebbf", size = 271787, upload-time = "2025-09-25T19:49:32.144Z" }, + { url = "https://files.pythonhosted.org/packages/aa/e7/d7dba133e02abcda3b52087a7eea8c0d4f64d3e593b4fffc10c31b7061f3/bcrypt-5.0.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:744d3c6b164caa658adcb72cb8cc9ad9b4b75c7db507ab4bc2480474a51989da", size = 269753, upload-time = "2025-09-25T19:49:33.885Z" }, + { url = "https://files.pythonhosted.org/packages/33/fc/5b145673c4b8d01018307b5c2c1fc87a6f5a436f0ad56607aee389de8ee3/bcrypt-5.0.0-cp314-cp314t-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a28bc05039bdf3289d757f49d616ab3efe8cf40d8e8001ccdd621cd4f98f4fc9", size = 289587, upload-time = "2025-09-25T19:49:35.144Z" }, + { url = "https://files.pythonhosted.org/packages/27/d7/1ff22703ec6d4f90e62f1a5654b8867ef96bafb8e8102c2288333e1a6ca6/bcrypt-5.0.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:7f277a4b3390ab4bebe597800a90da0edae882c6196d3038a73adf446c4f969f", size = 272178, upload-time = "2025-09-25T19:49:36.793Z" }, + { url = "https://files.pythonhosted.org/packages/c8/88/815b6d558a1e4d40ece04a2f84865b0fef233513bd85fd0e40c294272d62/bcrypt-5.0.0-cp314-cp314t-manylinux_2_34_aarch64.whl", hash = "sha256:79cfa161eda8d2ddf29acad370356b47f02387153b11d46042e93a0a95127493", size = 269295, upload-time = "2025-09-25T19:49:38.164Z" }, + { url = "https://files.pythonhosted.org/packages/51/8c/e0db387c79ab4931fc89827d37608c31cc57b6edc08ccd2386139028dc0d/bcrypt-5.0.0-cp314-cp314t-manylinux_2_34_x86_64.whl", hash = "sha256:a5393eae5722bcef046a990b84dff02b954904c36a194f6cfc817d7dca6c6f0b", size = 271700, upload-time = "2025-09-25T19:49:39.917Z" }, + { url = "https://files.pythonhosted.org/packages/06/83/1570edddd150f572dbe9fc00f6203a89fc7d4226821f67328a85c330f239/bcrypt-5.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:7f4c94dec1b5ab5d522750cb059bb9409ea8872d4494fd152b53cca99f1ddd8c", size = 334034, upload-time = "2025-09-25T19:49:41.227Z" }, + { url = "https://files.pythonhosted.org/packages/c9/f2/ea64e51a65e56ae7a8a4ec236c2bfbdd4b23008abd50ac33fbb2d1d15424/bcrypt-5.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:0cae4cb350934dfd74c020525eeae0a5f79257e8a201c0c176f4b84fdbf2a4b4", size = 352766, upload-time = "2025-09-25T19:49:43.08Z" }, + { url = "https://files.pythonhosted.org/packages/d7/d4/1a388d21ee66876f27d1a1f41287897d0c0f1712ef97d395d708ba93004c/bcrypt-5.0.0-cp314-cp314t-win32.whl", hash = "sha256:b17366316c654e1ad0306a6858e189fc835eca39f7eb2cafd6aaca8ce0c40a2e", size = 152449, upload-time = "2025-09-25T19:49:44.971Z" }, + { url = "https://files.pythonhosted.org/packages/3f/61/3291c2243ae0229e5bca5d19f4032cecad5dfb05a2557169d3a69dc0ba91/bcrypt-5.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:92864f54fb48b4c718fc92a32825d0e42265a627f956bc0361fe869f1adc3e7d", size = 149310, upload-time = "2025-09-25T19:49:46.162Z" }, + { url = "https://files.pythonhosted.org/packages/3e/89/4b01c52ae0c1a681d4021e5dd3e45b111a8fb47254a274fa9a378d8d834b/bcrypt-5.0.0-cp314-cp314t-win_arm64.whl", hash = "sha256:dd19cf5184a90c873009244586396a6a884d591a5323f0e8a5922560718d4993", size = 143761, upload-time = "2025-09-25T19:49:47.345Z" }, + { url = "https://files.pythonhosted.org/packages/84/29/6237f151fbfe295fe3e074ecc6d44228faa1e842a81f6d34a02937ee1736/bcrypt-5.0.0-cp38-abi3-macosx_10_12_universal2.whl", hash = "sha256:fc746432b951e92b58317af8e0ca746efe93e66555f1b40888865ef5bf56446b", size = 494553, upload-time = "2025-09-25T19:49:49.006Z" }, + { url = "https://files.pythonhosted.org/packages/45/b6/4c1205dde5e464ea3bd88e8742e19f899c16fa8916fb8510a851fae985b5/bcrypt-5.0.0-cp38-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:c2388ca94ffee269b6038d48747f4ce8df0ffbea43f31abfa18ac72f0218effb", size = 275009, upload-time = "2025-09-25T19:49:50.581Z" }, + { url = "https://files.pythonhosted.org/packages/3b/71/427945e6ead72ccffe77894b2655b695ccf14ae1866cd977e185d606dd2f/bcrypt-5.0.0-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:560ddb6ec730386e7b3b26b8b4c88197aaed924430e7b74666a586ac997249ef", size = 278029, upload-time = "2025-09-25T19:49:52.533Z" }, + { url = "https://files.pythonhosted.org/packages/17/72/c344825e3b83c5389a369c8a8e58ffe1480b8a699f46c127c34580c4666b/bcrypt-5.0.0-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:d79e5c65dcc9af213594d6f7f1fa2c98ad3fc10431e7aa53c176b441943efbdd", size = 275907, upload-time = "2025-09-25T19:49:54.709Z" }, + { url = "https://files.pythonhosted.org/packages/0b/7e/d4e47d2df1641a36d1212e5c0514f5291e1a956a7749f1e595c07a972038/bcrypt-5.0.0-cp38-abi3-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:2b732e7d388fa22d48920baa267ba5d97cca38070b69c0e2d37087b381c681fd", size = 296500, upload-time = "2025-09-25T19:49:56.013Z" }, + { url = "https://files.pythonhosted.org/packages/0f/c3/0ae57a68be2039287ec28bc463b82e4b8dc23f9d12c0be331f4782e19108/bcrypt-5.0.0-cp38-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:0c8e093ea2532601a6f686edbc2c6b2ec24131ff5c52f7610dd64fa4553b5464", size = 278412, upload-time = "2025-09-25T19:49:57.356Z" }, + { url = "https://files.pythonhosted.org/packages/45/2b/77424511adb11e6a99e3a00dcc7745034bee89036ad7d7e255a7e47be7d8/bcrypt-5.0.0-cp38-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:5b1589f4839a0899c146e8892efe320c0fa096568abd9b95593efac50a87cb75", size = 275486, upload-time = "2025-09-25T19:49:59.116Z" }, + { url = "https://files.pythonhosted.org/packages/43/0a/405c753f6158e0f3f14b00b462d8bca31296f7ecfc8fc8bc7919c0c7d73a/bcrypt-5.0.0-cp38-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:89042e61b5e808b67daf24a434d89bab164d4de1746b37a8d173b6b14f3db9ff", size = 277940, upload-time = "2025-09-25T19:50:00.869Z" }, + { url = "https://files.pythonhosted.org/packages/62/83/b3efc285d4aadc1fa83db385ec64dcfa1707e890eb42f03b127d66ac1b7b/bcrypt-5.0.0-cp38-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:e3cf5b2560c7b5a142286f69bde914494b6d8f901aaa71e453078388a50881c4", size = 310776, upload-time = "2025-09-25T19:50:02.393Z" }, + { url = "https://files.pythonhosted.org/packages/95/7d/47ee337dacecde6d234890fe929936cb03ebc4c3a7460854bbd9c97780b8/bcrypt-5.0.0-cp38-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:f632fd56fc4e61564f78b46a2269153122db34988e78b6be8b32d28507b7eaeb", size = 312922, upload-time = "2025-09-25T19:50:04.232Z" }, + { url = "https://files.pythonhosted.org/packages/d6/3a/43d494dfb728f55f4e1cf8fd435d50c16a2d75493225b54c8d06122523c6/bcrypt-5.0.0-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:801cad5ccb6b87d1b430f183269b94c24f248dddbbc5c1f78b6ed231743e001c", size = 341367, upload-time = "2025-09-25T19:50:05.559Z" }, + { url = "https://files.pythonhosted.org/packages/55/ab/a0727a4547e383e2e22a630e0f908113db37904f58719dc48d4622139b5c/bcrypt-5.0.0-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:3cf67a804fc66fc217e6914a5635000259fbbbb12e78a99488e4d5ba445a71eb", size = 359187, upload-time = "2025-09-25T19:50:06.916Z" }, + { url = "https://files.pythonhosted.org/packages/1b/bb/461f352fdca663524b4643d8b09e8435b4990f17fbf4fea6bc2a90aa0cc7/bcrypt-5.0.0-cp38-abi3-win32.whl", hash = "sha256:3abeb543874b2c0524ff40c57a4e14e5d3a66ff33fb423529c88f180fd756538", size = 153752, upload-time = "2025-09-25T19:50:08.515Z" }, + { url = "https://files.pythonhosted.org/packages/41/aa/4190e60921927b7056820291f56fc57d00d04757c8b316b2d3c0d1d6da2c/bcrypt-5.0.0-cp38-abi3-win_amd64.whl", hash = "sha256:35a77ec55b541e5e583eb3436ffbbf53b0ffa1fa16ca6782279daf95d146dcd9", size = 150881, upload-time = "2025-09-25T19:50:09.742Z" }, + { url = "https://files.pythonhosted.org/packages/54/12/cd77221719d0b39ac0b55dbd39358db1cd1246e0282e104366ebbfb8266a/bcrypt-5.0.0-cp38-abi3-win_arm64.whl", hash = "sha256:cde08734f12c6a4e28dc6755cd11d3bdfea608d93d958fffbe95a7026ebe4980", size = 144931, upload-time = "2025-09-25T19:50:11.016Z" }, + { url = "https://files.pythonhosted.org/packages/5d/ba/2af136406e1c3839aea9ecadc2f6be2bcd1eff255bd451dd39bcf302c47a/bcrypt-5.0.0-cp39-abi3-macosx_10_12_universal2.whl", hash = "sha256:0c418ca99fd47e9c59a301744d63328f17798b5947b0f791e9af3c1c499c2d0a", size = 495313, upload-time = "2025-09-25T19:50:12.309Z" }, + { url = "https://files.pythonhosted.org/packages/ac/ee/2f4985dbad090ace5ad1f7dd8ff94477fe089b5fab2040bd784a3d5f187b/bcrypt-5.0.0-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ddb4e1500f6efdd402218ffe34d040a1196c072e07929b9820f363a1fd1f4191", size = 275290, upload-time = "2025-09-25T19:50:13.673Z" }, + { url = "https://files.pythonhosted.org/packages/e4/6e/b77ade812672d15cf50842e167eead80ac3514f3beacac8902915417f8b7/bcrypt-5.0.0-cp39-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7aeef54b60ceddb6f30ee3db090351ecf0d40ec6e2abf41430997407a46d2254", size = 278253, upload-time = "2025-09-25T19:50:15.089Z" }, + { url = "https://files.pythonhosted.org/packages/36/c4/ed00ed32f1040f7990dac7115f82273e3c03da1e1a1587a778d8cea496d8/bcrypt-5.0.0-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:f0ce778135f60799d89c9693b9b398819d15f1921ba15fe719acb3178215a7db", size = 276084, upload-time = "2025-09-25T19:50:16.699Z" }, + { url = "https://files.pythonhosted.org/packages/e7/c4/fa6e16145e145e87f1fa351bbd54b429354fd72145cd3d4e0c5157cf4c70/bcrypt-5.0.0-cp39-abi3-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a71f70ee269671460b37a449f5ff26982a6f2ba493b3eabdd687b4bf35f875ac", size = 297185, upload-time = "2025-09-25T19:50:18.525Z" }, + { url = "https://files.pythonhosted.org/packages/24/b4/11f8a31d8b67cca3371e046db49baa7c0594d71eb40ac8121e2fc0888db0/bcrypt-5.0.0-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:f8429e1c410b4073944f03bd778a9e066e7fad723564a52ff91841d278dfc822", size = 278656, upload-time = "2025-09-25T19:50:19.809Z" }, + { url = "https://files.pythonhosted.org/packages/ac/31/79f11865f8078e192847d2cb526e3fa27c200933c982c5b2869720fa5fce/bcrypt-5.0.0-cp39-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:edfcdcedd0d0f05850c52ba3127b1fce70b9f89e0fe5ff16517df7e81fa3cbb8", size = 275662, upload-time = "2025-09-25T19:50:21.567Z" }, + { url = "https://files.pythonhosted.org/packages/d4/8d/5e43d9584b3b3591a6f9b68f755a4da879a59712981ef5ad2a0ac1379f7a/bcrypt-5.0.0-cp39-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:611f0a17aa4a25a69362dcc299fda5c8a3d4f160e2abb3831041feb77393a14a", size = 278240, upload-time = "2025-09-25T19:50:23.305Z" }, + { url = "https://files.pythonhosted.org/packages/89/48/44590e3fc158620f680a978aafe8f87a4c4320da81ed11552f0323aa9a57/bcrypt-5.0.0-cp39-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:db99dca3b1fdc3db87d7c57eac0c82281242d1eabf19dcb8a6b10eb29a2e72d1", size = 311152, upload-time = "2025-09-25T19:50:24.597Z" }, + { url = "https://files.pythonhosted.org/packages/5f/85/e4fbfc46f14f47b0d20493669a625da5827d07e8a88ee460af6cd9768b44/bcrypt-5.0.0-cp39-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:5feebf85a9cefda32966d8171f5db7e3ba964b77fdfe31919622256f80f9cf42", size = 313284, upload-time = "2025-09-25T19:50:26.268Z" }, + { url = "https://files.pythonhosted.org/packages/25/ae/479f81d3f4594456a01ea2f05b132a519eff9ab5768a70430fa1132384b1/bcrypt-5.0.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:3ca8a166b1140436e058298a34d88032ab62f15aae1c598580333dc21d27ef10", size = 341643, upload-time = "2025-09-25T19:50:28.02Z" }, + { url = "https://files.pythonhosted.org/packages/df/d2/36a086dee1473b14276cd6ea7f61aef3b2648710b5d7f1c9e032c29b859f/bcrypt-5.0.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:61afc381250c3182d9078551e3ac3a41da14154fbff647ddf52a769f588c4172", size = 359698, upload-time = "2025-09-25T19:50:31.347Z" }, + { url = "https://files.pythonhosted.org/packages/c0/f6/688d2cd64bfd0b14d805ddb8a565e11ca1fb0fd6817175d58b10052b6d88/bcrypt-5.0.0-cp39-abi3-win32.whl", hash = "sha256:64d7ce196203e468c457c37ec22390f1a61c85c6f0b8160fd752940ccfb3a683", size = 153725, upload-time = "2025-09-25T19:50:34.384Z" }, + { url = "https://files.pythonhosted.org/packages/9f/b9/9d9a641194a730bda138b3dfe53f584d61c58cd5230e37566e83ec2ffa0d/bcrypt-5.0.0-cp39-abi3-win_amd64.whl", hash = "sha256:64ee8434b0da054d830fa8e89e1c8bf30061d539044a39524ff7dec90481e5c2", size = 150912, upload-time = "2025-09-25T19:50:35.69Z" }, + { url = "https://files.pythonhosted.org/packages/27/44/d2ef5e87509158ad2187f4dd0852df80695bb1ee0cfe0a684727b01a69e0/bcrypt-5.0.0-cp39-abi3-win_arm64.whl", hash = "sha256:f2347d3534e76bf50bca5500989d6c1d05ed64b440408057a37673282c654927", size = 144953, upload-time = "2025-09-25T19:50:37.32Z" }, +] + +[[package]] +name = "cffi" +version = "2.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pycparser", marker = "implementation_name != 'PyPy'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9e/ef/008a1939e372c06329a3fce4279c02f328488f3526744906eeec3da7ad5f/cffi-2.1.1.tar.gz", hash = "sha256:dd31f52ea1086513bb9df30f8fcee9b8918323ae067a3d5b78bc826a000712be", size = 530807, upload-time = "2026-08-03T21:21:18.939Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/10/69/43965eccfdead3b9220015fd1320e117be8c6ed01a62ffab76eeb752f5d5/cffi-2.1.1-cp312-cp312-macosx_10_15_x86_64.whl", hash = "sha256:c8c69575568085ba0b1b10c0249d779a214aea6f6522e949a0fc9fb0fcb449d0", size = 184821, upload-time = "2026-08-03T21:19:44.887Z" }, + { url = "https://files.pythonhosted.org/packages/54/7d/16e5a096677b5e313ca80cd5e5170efa3ea44624a82bb111925522da64b1/cffi-2.1.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f81b3b8f3d4e343550fa4baa0e479bba9f2d29ce9c2e9b51d1ce1718d7442fcf", size = 184719, upload-time = "2026-08-03T21:19:46.129Z" }, + { url = "https://files.pythonhosted.org/packages/56/e6/8941622732edec876dd17d0453dce07317ae96db34f2ec1436c9d3785986/cffi-2.1.1-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:811bd1e21d32de12efca32393a0ab3f5133b54fce9bd44b8bd77ab07da14bf6a", size = 214799, upload-time = "2026-08-03T21:19:47.218Z" }, + { url = "https://files.pythonhosted.org/packages/44/de/f98430906df1545ffde0d543dd124a7a439bc2cd32b36b9c53f805df7333/cffi-2.1.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:68e62fe11f30d5ca8289242866f0a5291402d8529ca2178ab8afc5c9694ae890", size = 222389, upload-time = "2026-08-03T21:19:48.331Z" }, + { url = "https://files.pythonhosted.org/packages/6a/5b/717f1526b9957b34456313c31645c5b82b8fb5c3fe9e4752999be7128bfc/cffi-2.1.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:4a7c934f7360e8cd64fe9efadcbd10c7c6364f531e432b9a4bf5ccbc9e0e8b50", size = 210249, upload-time = "2026-08-03T21:19:49.543Z" }, + { url = "https://files.pythonhosted.org/packages/64/b3/f8aa4f3e34986c7e4ec45072d1b1b9dd295b6b18007b45518d79726dd725/cffi-2.1.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:3143d81e29e1e20a9ce10901ec369012947876596f75a222235965f2b7ae832e", size = 208775, upload-time = "2026-08-03T21:19:50.918Z" }, + { url = "https://files.pythonhosted.org/packages/b1/db/dceb9dd5b231e1da801793f8acc9f3c52a7e1afe40bb1aae37e02b0faad5/cffi-2.1.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c1453022f490d2459a11819d83ad1d586e9ff65a12ac3e705ffebd46d3685dcf", size = 221822, upload-time = "2026-08-03T21:19:52.054Z" }, + { url = "https://files.pythonhosted.org/packages/a0/d2/6cd24ae3be000a634109c247d1475d62e5616d0dc78c82770942ec384248/cffi-2.1.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:208f941bb9d18e768138677f0a6d2ce01f590df56043dda1df1535ac57c88517", size = 225232, upload-time = "2026-08-03T21:19:53.109Z" }, + { url = "https://files.pythonhosted.org/packages/cb/52/3fa190537004dd7f0ab860a6dc7c0175b8667f68d1e618a46f5498d30250/cffi-2.1.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:210019b6c7cf07f081b4c54635c8cf744377001350e29cc0f81c4377b4797735", size = 223597, upload-time = "2026-08-03T21:19:54.515Z" }, + { url = "https://files.pythonhosted.org/packages/80/fb/0bb75b7039588c074b37ae99f40d9bfddf990ecb2fbc346ebccd2e56b9be/cffi-2.1.1-cp312-cp312-win32.whl", hash = "sha256:046bfc24911b37851ee1b51aab8bffe713d89c68c6a057b09484ce9fd5f69b4e", size = 175292, upload-time = "2026-08-03T21:19:55.566Z" }, + { url = "https://files.pythonhosted.org/packages/d9/79/615cc094e2fb508cade7de88d3b4f6c4ec2bab695c97bce9153dc65aadf5/cffi-2.1.1-cp312-cp312-win_amd64.whl", hash = "sha256:f53e442b08449d42821fa4a4fba000095af9f62742a500f978a9f557ec44339a", size = 185919, upload-time = "2026-08-03T21:19:56.89Z" }, + { url = "https://files.pythonhosted.org/packages/70/c6/d0ea84713fe46b243a436a18fcd47d639732747e21635c8a27191b06dc30/cffi-2.1.1-cp312-cp312-win_arm64.whl", hash = "sha256:7bde5e4cc5c10140859842b9d383af292b22639a4dffb725314baf45968cef80", size = 180093, upload-time = "2026-08-03T21:19:58.155Z" }, + { url = "https://files.pythonhosted.org/packages/9d/f4/035513d4117049066b4779dc3b7c0c0fdad175fa13731c9f4003f1cd1478/cffi-2.1.1-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:b5bdfd1c873d4e093aabc0ca84c4ca6dbc4f752afb5c86f146d9742580c9da2e", size = 194248, upload-time = "2026-08-03T21:19:59.399Z" }, + { url = "https://files.pythonhosted.org/packages/76/af/2aeb4dbb5fc41a04161ae9ff1518de7cec08e164f44a8ce6a4cf7fd2cd1d/cffi-2.1.1-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:31348097ff5bbe827ccc41795d4dd099d9f0625e7def00ee653c137a490c2a6c", size = 196908, upload-time = "2026-08-03T21:20:00.746Z" }, + { url = "https://files.pythonhosted.org/packages/a7/46/2e5fdde8555706dd98139a910ca11be02809f3f605ce956f655d0214e100/cffi-2.1.1-cp313-cp313-macosx_10_15_x86_64.whl", hash = "sha256:9d2055050ea716bd38b7f7f1579c275386646b4894c155a3e2f3cd62ed41b7c6", size = 184805, upload-time = "2026-08-03T21:20:02.02Z" }, + { url = "https://files.pythonhosted.org/packages/55/41/4c7042f317b9217502988f0873af87e16ad606dc20f84e546e3e6ce9764c/cffi-2.1.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:19ee6127ee34de7d83ce3d371ebc5ed91addbdcc39f9ab15ce4eb35a4e534971", size = 184764, upload-time = "2026-08-03T21:20:03.141Z" }, + { url = "https://files.pythonhosted.org/packages/43/1f/1c3d90d91811c8f86ced9ed637956c54bfe5b79ca98fe976d7f8c8979f6b/cffi-2.1.1-cp313-cp313-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:6a8dddef476fab96d066d578fc88526767b836ab5ab21754e1d5bf3879c31c7c", size = 214722, upload-time = "2026-08-03T21:20:04.377Z" }, + { url = "https://files.pythonhosted.org/packages/37/6f/3b5ce4c3b2192d250f04908f2bfd91ef34552ec8f7716a5d4abdb8d67bb2/cffi-2.1.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f16c709686a78c727bbbf059f92b0bf41c6fc60deec706d2dc19f529175a6125", size = 222369, upload-time = "2026-08-03T21:20:05.544Z" }, + { url = "https://files.pythonhosted.org/packages/02/10/4b3c75dde3d9663c9e02ba05c2668b954f671d4bbe346413ca8c696b295a/cffi-2.1.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:fcd22650c908d7b7da162bbfaab594a1227a15d1643a98c68b122ac642fa2264", size = 210175, upload-time = "2026-08-03T21:20:06.75Z" }, + { url = "https://files.pythonhosted.org/packages/df/62/14f74b9543e605d17701dc797b815958b8bb70b7624ce1b832ddad48ed6c/cffi-2.1.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:aa9511c62d14da7aacc9b4bf51f3f697a621e83b2d6919008243c3aad168eea3", size = 208670, upload-time = "2026-08-03T21:20:08.04Z" }, + { url = "https://files.pythonhosted.org/packages/95/95/86342356ff5953b3fb06f7ef7c5bee212d45e770abc7218d451b9148313c/cffi-2.1.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:a931079504ecc49efed7744c476a5c343a92fabf66dec2db95edb1b2fdc770e2", size = 221824, upload-time = "2026-08-03T21:20:09.274Z" }, + { url = "https://files.pythonhosted.org/packages/eb/ff/7b3429ff53aafe931ed8a5fc69f481bbef7ba6de87ddcbb63d08f483f613/cffi-2.1.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a2d7755bef5a12ed488f4ef1f1b69ee9191d7396083b755a5d2295f6edb4768b", size = 225148, upload-time = "2026-08-03T21:20:10.7Z" }, + { url = "https://files.pythonhosted.org/packages/34/34/a95870b9221e09cf4f2ce3178b1a210abdfe63a1bd357da940418d7b8d15/cffi-2.1.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:e0bcb7e0f677f543555d2adff3bf19c05f66cdb4796e5ff602442ab2fe3c4ef7", size = 223564, upload-time = "2026-08-03T21:20:12.165Z" }, + { url = "https://files.pythonhosted.org/packages/70/ea/839b50531021a647fb5e929f72cf97bc1ff702b5472166164b5b6e76b851/cffi-2.1.1-cp313-cp313-win32.whl", hash = "sha256:334644fbac4eff73d985a17a91226df55d0f394160c4cfb880e084c8f7161cac", size = 175263, upload-time = "2026-08-03T21:20:13.559Z" }, + { url = "https://files.pythonhosted.org/packages/60/a6/8b149b2c3f2e11aaa1618ef64500b45f50f22c57a977a4dff1aff1f91042/cffi-2.1.1-cp313-cp313-win_amd64.whl", hash = "sha256:1aa5645c30469b09530c4ebca77ebf8f17618293c58f8549cb1a543a50236e7d", size = 185688, upload-time = "2026-08-03T21:20:14.69Z" }, + { url = "https://files.pythonhosted.org/packages/01/9a/11f687cb39d6a3504060d5242f04f48c735afb4d3d533958a20594890cb2/cffi-2.1.1-cp313-cp313-win_arm64.whl", hash = "sha256:63bbfd5ded17c4840ac07cd8f1c21ba9d9708141f840b324f422f41b207e3973", size = 180078, upload-time = "2026-08-03T21:20:15.917Z" }, + { url = "https://files.pythonhosted.org/packages/d3/7b/d6bbf82b8b96e7391438898c42f5bd96dd02030fd5b64937d248220003e2/cffi-2.1.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:7dbb61fe3a7699468030f71bbe5f8a0e326a151daa91beb11a6fc1f980c55e1c", size = 194064, upload-time = "2026-08-03T21:20:17.148Z" }, + { url = "https://files.pythonhosted.org/packages/94/e6/bcc91b283be94735e268487a054004f0aa19947b6348fa367db53230abc8/cffi-2.1.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:f24fb43132a4c6b4cb4eb029492919b2db645be6808d738f244fd146c03c32cb", size = 196720, upload-time = "2026-08-03T21:20:18.268Z" }, + { url = "https://files.pythonhosted.org/packages/d9/99/c4b0c17cacdc9c3b8f280026286a9826d6a208c0f047591a3c3ce99b91fd/cffi-2.1.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:d28630f5854ab07ab1fd4aba756de52326c82e6be15d414b12793f1975048b54", size = 184964, upload-time = "2026-08-03T21:20:19.708Z" }, + { url = "https://files.pythonhosted.org/packages/b3/a9/9db617d05d7367c1ad0ab00b3aa6e6f9281edd689b4ee9ea0e5a84e89c97/cffi-2.1.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:661c298b4821edebead0c91edd2b00374d67ad7c5a1f7a91d4442633b79d6a72", size = 184962, upload-time = "2026-08-03T21:20:20.833Z" }, + { url = "https://files.pythonhosted.org/packages/67/b8/b42132ca113dc567d37684437b46ca1dafc885902b02a110a02d5b511857/cffi-2.1.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:58acb8ab8e295e6c5ea12f888cbb13cf21511ef2a3303a23f4325c29d17fe5c1", size = 222328, upload-time = "2026-08-03T21:20:22.118Z" }, + { url = "https://files.pythonhosted.org/packages/80/10/c5c0cbf0a657aecf59ef511409734230bf556f05a0d6c9eed7aa5c0a0166/cffi-2.1.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:456a61fa52d579ebf9df2e9552ead5129855dbaff6c1e5a9b1bc408809bdc062", size = 209985, upload-time = "2026-08-03T21:20:23.401Z" }, + { url = "https://files.pythonhosted.org/packages/d5/6c/bfa0b87b03b9238148beca990292843c9396ba069b54496596594173de7b/cffi-2.1.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:a4f00aa42f75d6e4595e8866e748cc1705adc0cddfeb2ca86d0d03993d63ba03", size = 208530, upload-time = "2026-08-03T21:20:24.628Z" }, + { url = "https://files.pythonhosted.org/packages/e9/02/4e7d553a7ac4b4238b38b3c1b80d486e9d4436f8d2acbf87a0997fe3f402/cffi-2.1.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:b0431303acaea1089ad4b3e9ce4e6518193def1118d4073ca848635ee4ea2e96", size = 221525, upload-time = "2026-08-03T21:20:25.758Z" }, + { url = "https://files.pythonhosted.org/packages/82/1d/a4aaf9babd75acb4d5f223bff71533bee748dd770a382619a798960ee9ba/cffi-2.1.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:64faea20f4e2613363a1a9b9c7dd73058f3ecd00133a511e72ad7c511658f527", size = 225053, upload-time = "2026-08-03T21:20:26.985Z" }, + { url = "https://files.pythonhosted.org/packages/81/10/5dc0e7bdd18e22107054288283380fc97a06ae3f1656a106908d666a3c88/cffi-2.1.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5c58fe613dc5e5336357eff555824a314d8e43282600435c8d1cb6a7a2fedd13", size = 223213, upload-time = "2026-08-03T21:20:28.277Z" }, + { url = "https://files.pythonhosted.org/packages/0b/e9/d0061c364cde06ee43168a0d076ac1da512cbc380d44767b844ba34fe2b6/cffi-2.1.1-cp314-cp314-win32.whl", hash = "sha256:1a18a57b58cfb21fc28d72e876acf10eaed67a1ed96226f92af4df681d571c4c", size = 177682, upload-time = "2026-08-03T21:20:44.288Z" }, + { url = "https://files.pythonhosted.org/packages/a7/06/1c3e01e3ba14c39f6d10bfbac52753b7e22259e38088e5cfe1d704918690/cffi-2.1.1-cp314-cp314-win_amd64.whl", hash = "sha256:3222ba5d678f80a030e6afbcc33dc1ae5cb45facabb61cee2c7016b8432fde48", size = 187949, upload-time = "2026-08-03T21:20:45.623Z" }, + { url = "https://files.pythonhosted.org/packages/87/5b/da4e39efe18eeb89cf580ea9cfc66b6a7c3eadb808fc0cc1d3a295cb5a5d/cffi-2.1.1-cp314-cp314-win_arm64.whl", hash = "sha256:ab36d55f9ed2d067327667c2fea18dda018eb628dd6347aa01dda6cf1f5d3836", size = 182947, upload-time = "2026-08-03T21:20:46.955Z" }, + { url = "https://files.pythonhosted.org/packages/23/59/40338bf421c5accea1d45158170c87006ef1cd371b05c077e76476949728/cffi-2.1.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:7750c6449dff7864bb9bb27ddfb0267756189201a3afc911d82b3caacd70dfc3", size = 188504, upload-time = "2026-08-03T21:20:29.495Z" }, + { url = "https://files.pythonhosted.org/packages/7d/47/5ecf1023850036e674c77ec4de86182d309ae344e39e7cba984b7df5d647/cffi-2.1.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:0beceaabe56af686895136a2de78db54ecd8e4046b236b8fd6d6cb61389e9bf2", size = 188259, upload-time = "2026-08-03T21:20:31.291Z" }, + { url = "https://files.pythonhosted.org/packages/2a/9c/92934c3bea9f785b23eba304538c0b4d37a2a96d2431eb3a1bc87a11aa19/cffi-2.1.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:49cbc70e6542d4ccccb936558d1064a8012541e78f821f955cff24e357776c94", size = 223864, upload-time = "2026-08-03T21:20:32.571Z" }, + { url = "https://files.pythonhosted.org/packages/4d/45/ba4c93527bc38616a8bd36488acb69a2212d60486794f0c1f318949bbb76/cffi-2.1.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:e2d65b31f36619cda3999b78b2aa9632e76b78448e7a56fc4240824200e7c4fc", size = 211538, upload-time = "2026-08-03T21:20:33.808Z" }, + { url = "https://files.pythonhosted.org/packages/80/e9/b6ef565e452acb932fb0cb5443f44a78efbd1233e566f02b5a83855e9115/cffi-2.1.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:28907ab9bfb6aa13184cfc17c6b8e1023c5ab6fd7076d8c20a35e59fe04f8f29", size = 210688, upload-time = "2026-08-03T21:20:34.974Z" }, + { url = "https://files.pythonhosted.org/packages/9a/95/eff5f0cee78d2eabc7eebffec40d3fc1876b5f3c95582e018bb4b99601f2/cffi-2.1.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:51b31d1c98274844cfd7838ce00bfc27c7423a4dc00fc0772fc3331c2cc90676", size = 223803, upload-time = "2026-08-03T21:20:36.564Z" }, + { url = "https://files.pythonhosted.org/packages/fa/01/579d39fb8bef00a335a23d83757b44feb24cd6345a2c451b64cb67b9c362/cffi-2.1.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:5e7cecbaadb83884793e05828cee59b210b24583b9c7425d0ba6a754fe22eb4e", size = 226763, upload-time = "2026-08-03T21:20:37.816Z" }, + { url = "https://files.pythonhosted.org/packages/8d/b0/0b44f47c60b01b57b6e2bbd92343f13a85a1d93bc46ccf6e47e244acd99c/cffi-2.1.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:25792eac27877609e7bb06d42ff88278a6624fff2ba9bbb523c09616b117e80f", size = 225688, upload-time = "2026-08-03T21:20:38.959Z" }, + { url = "https://files.pythonhosted.org/packages/eb/d2/3b7176cb570a1d3e27faf67b72f591af508036e0d8b2be2ef9af9e8c84bb/cffi-2.1.1-cp314-cp314t-win32.whl", hash = "sha256:8ef53b2de9bcb9197d31854256575d59dbac0cba72ac627bb291ef5eceb74be4", size = 182868, upload-time = "2026-08-03T21:20:40.388Z" }, + { url = "https://files.pythonhosted.org/packages/56/78/31f00c1bcd97c9bbf55f1bfdf5bc809a5de8887473e90bb9960dca825e80/cffi-2.1.1-cp314-cp314t-win_amd64.whl", hash = "sha256:616f097f2fe415bc92a247f02e11f634e1f9e9a83d327e3c915c15089c87869e", size = 194104, upload-time = "2026-08-03T21:20:41.725Z" }, + { url = "https://files.pythonhosted.org/packages/7b/1b/58496f2ed0a35de575250c02a43ab3cc2c04d494a88fed31c1cabc0fd176/cffi-2.1.1-cp314-cp314t-win_arm64.whl", hash = "sha256:ad2c86c495b899d862ea0f4b42891b8713a3bd45dd4105c7fd51c2a72f39f3a5", size = 186402, upload-time = "2026-08-03T21:20:43.042Z" }, + { url = "https://files.pythonhosted.org/packages/c1/8f/9ebe220eab48a093d1a5a5e339ab0dc7316eef3bb04d63c42f0251b61f50/cffi-2.1.1-cp315-cp315-ios_13_0_arm64_iphoneos.whl", hash = "sha256:dddad92b554513a31f272570678ba307fb9f618f05e3d4a5eacafff9eae03e1d", size = 194043, upload-time = "2026-08-03T21:20:48.179Z" }, + { url = "https://files.pythonhosted.org/packages/ff/69/844bad3ece306c4782c2ecb93597035b6690d48704b803914c199da1e8b3/cffi-2.1.1-cp315-cp315-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:da0e573f9f97159390c89d9f1a9e41908b66d408cc5b58d08cf3847d844c531b", size = 196737, upload-time = "2026-08-03T21:20:49.457Z" }, + { url = "https://files.pythonhosted.org/packages/1b/8a/af668013284634733f02d683458a0728739c7d6ddb5e14cb0c20832266fe/cffi-2.1.1-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:fb92203a88b3d3053034db775110081c49d28be6551923805e039924093761e4", size = 184933, upload-time = "2026-08-03T21:20:50.639Z" }, + { url = "https://files.pythonhosted.org/packages/0c/75/2f5207ff6d1a613133b23a5203cc0c2a628313b5eb3974d7956ae3c57950/cffi-2.1.1-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:2ae64be792b8966f2c69538199728b290e34726562896df1e5dc8ffd8d8188e8", size = 185002, upload-time = "2026-08-03T21:20:52.173Z" }, + { url = "https://files.pythonhosted.org/packages/e2/31/9e1313b0a6e30e91b3b3d3fff51ae99c857c07738e3afcce1f7334e1b7ab/cffi-2.1.1-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:507a24c282e0f42f8ed737cf048572cbf580468da5555764a8331735e9c736b6", size = 222271, upload-time = "2026-08-03T21:20:53.462Z" }, + { url = "https://files.pythonhosted.org/packages/50/e3/f6234a833e6e08c7007003074723c406559eecf9b48dfc97471e5a8eb7a0/cffi-2.1.1-cp315-cp315-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:246fa40ce8645a614ff682e0b70f37134e460eaf93a775e0cbe3cca585a67a80", size = 209919, upload-time = "2026-08-03T21:20:54.783Z" }, + { url = "https://files.pythonhosted.org/packages/0d/fc/5f74e293fced6edb51af3a46c4ccf6c23c9943774ecb375ddbd522c76add/cffi-2.1.1-cp315-cp315-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:471cee653ae88de62096552e6d24ccb4a5adb8c8c9f10b5054d0122c15bf2779", size = 208529, upload-time = "2026-08-03T21:20:56.066Z" }, + { url = "https://files.pythonhosted.org/packages/44/16/29e6d01b388bef055ecd6ca8244b3f4d336bd09e92d5d892187b9601084e/cffi-2.1.1-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:aeae0e330c9f6acd681f647d46cefd30c29f93e3392882e792e82080c9691399", size = 221630, upload-time = "2026-08-03T21:20:57.336Z" }, + { url = "https://files.pythonhosted.org/packages/a4/18/fa7f1f6857d5eb88a4ca99ffcbfb7c387a287ccc154c64a73e86314745d7/cffi-2.1.1-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:42a494cee34437f05546455144f2b5d9ac09b1face62bcfce597d2e521066688", size = 225134, upload-time = "2026-08-03T21:20:58.675Z" }, + { url = "https://files.pythonhosted.org/packages/e0/9f/e8e3dfa04a1b4c241f8c91faacad872b4d4efd051d49764ad4e2fd4b9fea/cffi-2.1.1-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:cc572dace3f60ef98d7b12ff411d20f5362feb31a0439eab0085bbfd349982d7", size = 223197, upload-time = "2026-08-03T21:20:59.968Z" }, + { url = "https://files.pythonhosted.org/packages/f8/7e/8debeb04f1ab9fe2a6963964cd6f1aaf7192627b83926586a6a4e089c9fa/cffi-2.1.1-cp315-cp315-win32.whl", hash = "sha256:4f42141fc14250de6dde5ee7ea4432be017252d91f19c5ad043c084cea629cac", size = 177683, upload-time = "2026-08-03T21:21:14.901Z" }, + { url = "https://files.pythonhosted.org/packages/e0/31/5158704cc474ab65c1647932e88be78dc0873f47130e253be38bcaf13d01/cffi-2.1.1-cp315-cp315-win_amd64.whl", hash = "sha256:e6e8cff14d6fb0be70a09c0bdc58096f501952d04624ebf867e0e56da2df8960", size = 187897, upload-time = "2026-08-03T21:21:16.108Z" }, + { url = "https://files.pythonhosted.org/packages/cc/4b/b3a2da8570c704ffc0f9762cdc3ec0f02c8573798e0b5cf7f11c82bbb70f/cffi-2.1.1-cp315-cp315-win_arm64.whl", hash = "sha256:27350daa11d4f10c540e6e89dada4c54feb7256ad03e9a4dc075ebad7ba360d1", size = 182935, upload-time = "2026-08-03T21:21:17.271Z" }, + { url = "https://files.pythonhosted.org/packages/d0/ef/5443574510a1207e6f6bc38ba6e1f1de36cb48fef07b2728bb896a21f430/cffi-2.1.1-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:c26608d2222fb1e94487e4a387d85f13eb55d5ed725cb25a0c589ac4ee60e7bc", size = 188464, upload-time = "2026-08-03T21:21:01.163Z" }, + { url = "https://files.pythonhosted.org/packages/7e/ae/a56fa8c4686ad50e148fcbc8d3ae0d03915ff5c30d795058988c24118cef/cffi-2.1.1-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:4be96343e422f2dfcd12ab5c9f5aebe03f82f737c6bffeca6830b3875cb44aab", size = 188262, upload-time = "2026-08-03T21:21:02.382Z" }, + { url = "https://files.pythonhosted.org/packages/53/b2/6187f46f2912276a3ae284076109cc5c8680482f11f766ccf26db4a86427/cffi-2.1.1-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:937c0052c05a31ca1daf18de3158eed4dbfcb9cc107adbea227728d647be701e", size = 223779, upload-time = "2026-08-03T21:21:03.553Z" }, + { url = "https://files.pythonhosted.org/packages/8a/f6/c3ad28bd19f77047a03084424fbd4cbe997303267c14423737324be0385d/cffi-2.1.1-cp315-cp315t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:df423d40ee8654634421812bc3b196da3f9bd7d32929da813f8394c4348a5358", size = 211520, upload-time = "2026-08-03T21:21:04.863Z" }, + { url = "https://files.pythonhosted.org/packages/a0/cd/ccac9013a5bd9fd764de118674ab9c805b5ca10c19270d90ee273f8b2240/cffi-2.1.1-cp315-cp315t-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:a730a083190634c65cca36ba5f489531576ebd79bcd5c8e172130f6453127231", size = 210673, upload-time = "2026-08-03T21:21:06.223Z" }, + { url = "https://files.pythonhosted.org/packages/52/86/2976131c639aead931c5bee5aba67e4b09fbeb8018b6f282f70803f923a7/cffi-2.1.1-cp315-cp315t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:363e05fa78e15116c3c32c210ee36884fd6b9afa6d440e47112c3bd511d64cb6", size = 223835, upload-time = "2026-08-03T21:21:07.539Z" }, + { url = "https://files.pythonhosted.org/packages/ac/0c/33a7aeab2f9c76918c52e084beb39c570db3588133412929e8ec06fab90b/cffi-2.1.1-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:770de9db11e84213beec501cfcaa013b019820ca881e03344dea5844f7876d94", size = 226705, upload-time = "2026-08-03T21:21:08.774Z" }, + { url = "https://files.pythonhosted.org/packages/e3/26/2cde30fdde421130bfc18f70395731a6e6b2053c6a1978a5258ff04e72fa/cffi-2.1.1-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:7da0c5eff80f0197f3b3d1232ec5a682a9325f4ae9016a78f5f5ca35f9ced1f5", size = 225539, upload-time = "2026-08-03T21:21:09.911Z" }, + { url = "https://files.pythonhosted.org/packages/6d/cd/a361394c94b2129d604bb846f624a8e88255a3ee33129c434a00d715e64f/cffi-2.1.1-cp315-cp315t-win32.whl", hash = "sha256:06c72bb76605a4b0cd0aad6930b69d4baf7dd5d806cfc409b824191099700e66", size = 182707, upload-time = "2026-08-03T21:21:11.226Z" }, + { url = "https://files.pythonhosted.org/packages/9b/b5/ba2b299993c26577d529b6ae29841f9e15b9fcf004d65f423f4fcf94ade9/cffi-2.1.1-cp315-cp315t-win_amd64.whl", hash = "sha256:d9c275eaacd24aa73f94ffd6de08fc3f932424d8b6c376f4bed7cde376fe7bc3", size = 193772, upload-time = "2026-08-03T21:21:12.39Z" }, + { url = "https://files.pythonhosted.org/packages/aa/29/35e016098c814cd93de9cd320c66b5bfba14dc6ecedd3cb518fa7c408c69/cffi-2.1.1-cp315-cp315t-win_arm64.whl", hash = "sha256:d18e5ac0f2f03f4f518d3e23db0f0cad7faa1da8620e9c09461d443bbf6e6692", size = 186360, upload-time = "2026-08-03T21:21:13.636Z" }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "coverage" +version = "7.17.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d2/79/e78bd8f75a9b8e32233ce92d4607f0a21ce88a9156cace2884e7175a82f5/coverage-7.17.0.tar.gz", hash = "sha256:cc010a0a7fbdd9bb8bdfd0d3f04a5151275860213cbd7e2b876b29778489c712", size = 991021, upload-time = "2026-10-22T16:06:50.009Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0a/25/989fd3b495c4274ac0cf54cb6664326415e60183a7f8a73dd9bbb0b11e34/coverage-7.17.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:0cfdc938ca3319d873d1f1252e60620a39aee4ebf9dbd7cc58238109ebda9c51", size = 225938, upload-time = "2026-10-22T16:04:18.113Z" }, + { url = "https://files.pythonhosted.org/packages/db/cb/14726426bf3241657facbf0f7df7babd8f7d3fcf8e4609b7929f9dceecbc/coverage-7.17.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d523da3c2e4bca49417768bcdb1a443c5b359dc44b8b93a3102e118fce912c40", size = 226166, upload-time = "2026-10-22T16:04:19.524Z" }, + { url = "https://files.pythonhosted.org/packages/8e/97/f95ed32cd3cfbe261a3598e694cb770cefc7b2d13ff013418d6f70a44dde/coverage-7.17.0-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:adbf3450c0f3f2b2854e282a8758f2b3eddf787eeff24ec9c9bc2cbc38328cc1", size = 258462, upload-time = "2026-10-22T16:04:21.079Z" }, + { url = "https://files.pythonhosted.org/packages/91/ac/eb4c31f741509a9ef48cf6f28385e98e9de0785599282252eee252ff8af7/coverage-7.17.0-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:85932227e9dd38e62cd29c17fefdfecfb919eead6af1e3259522d03cf85de983", size = 261789, upload-time = "2026-10-22T16:04:23.037Z" }, + { url = "https://files.pythonhosted.org/packages/e8/3e/d9cd006d1a3cc5deab07beb29b86cb4a10dd84d66b330820363fb8d310d6/coverage-7.17.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:30f5477aac9c9e4a0a641c55f8f8fddd7f3bd91176c912c50f40ffced96b4034", size = 262322, upload-time = "2026-10-22T16:04:24.94Z" }, + { url = "https://files.pythonhosted.org/packages/c0/3a/4c872c6ae0762b09cdb8e856891bb3f584848cb60b6dda2bacb85fe2f3f0/coverage-7.17.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0b8bb651a4406fb6a983a905309fec1783443c65e1771b41e92740a0f4a1176b", size = 264600, upload-time = "2026-10-22T16:04:26.444Z" }, + { url = "https://files.pythonhosted.org/packages/df/98/1d41a2373fa98b423bcd70adb91d75e38eafb417955426dbb920b10016f3/coverage-7.17.0-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0edeb6594f080c8de1263cccb239cb316ef77aeb959e2e0f38f2953fcc72a49e", size = 258678, upload-time = "2026-10-22T16:04:27.832Z" }, + { url = "https://files.pythonhosted.org/packages/92/6a/fe04b5e63e76506daa63d90bdfb00a53767fe5df18546b437d6226918d6b/coverage-7.17.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:da94ec0484e84fce5c7bf8d0f54b97767a90d992653a129b35634699d516e0a5", size = 260210, upload-time = "2026-10-22T16:04:29.275Z" }, + { url = "https://files.pythonhosted.org/packages/4f/c3/5c5a78dc19dd4799be9c6ec022316fb25ababaeacfaf0f52f24c56a57b43/coverage-7.17.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:a25cc5e8c23a590815c6a7aa43624234d25cb23b21b484adb8a0daed59860db5", size = 258375, upload-time = "2026-10-22T16:04:30.647Z" }, + { url = "https://files.pythonhosted.org/packages/ed/e7/4c1d8eaff4d756fefd15d58c4953a78ab656782fd0e32c5a3465dba61aa0/coverage-7.17.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:db645c81cf4d6bcb392ea1894aad65d55d1588a8472c834b25ca575da3029a56", size = 262422, upload-time = "2026-10-22T16:04:32.053Z" }, + { url = "https://files.pythonhosted.org/packages/a6/2a/0b17c3e2e7b6ff1a5d92eb58915c0d1026668928f961d14dac73cf1b6096/coverage-7.17.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:6a80a11b90c4a03df1502304cf24ff953255b6b5d076ef79066ecf88baaff31f", size = 257875, upload-time = "2026-10-22T16:04:33.824Z" }, + { url = "https://files.pythonhosted.org/packages/d3/2d/eb0cbde0535342df0d17a667163030a3afb67e5eb9e3ad94af9540041584/coverage-7.17.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e4eaf11d6b0f2147f46cfbf06fe7edcca6f095d0120da00ddc2458995ee869fc", size = 260361, upload-time = "2026-10-22T16:04:35.661Z" }, + { url = "https://files.pythonhosted.org/packages/2e/47/1be2a6db7eb655d0a1d899217c3f2469f9121cbbeddac35c1c8173b1482e/coverage-7.17.0-cp312-cp312-win32.whl", hash = "sha256:912a29f338798541d795e2e25903c6fe7a99dee70a40fe4706a9a124649b523e", size = 227747, upload-time = "2026-10-22T16:04:37.102Z" }, + { url = "https://files.pythonhosted.org/packages/11/c5/201c894c57de81875d8b8cc527ba222aa350e8cfb183eb3f6df45d06ad8e/coverage-7.17.0-cp312-cp312-win_amd64.whl", hash = "sha256:88a1a9fb0075b83389dd9b403bb3fce976031b3c2aa0e05e3e57143cf639de69", size = 228325, upload-time = "2026-10-22T16:04:38.989Z" }, + { url = "https://files.pythonhosted.org/packages/0b/5d/196095339b19758cf83f507fb598bc42820ae124745feb3084662729e17a/coverage-7.17.0-cp312-cp312-win_arm64.whl", hash = "sha256:2c680e838ff9e68d491158625ca2c1ddb17d78d791bba96a31f8b839b4990438", size = 228131, upload-time = "2026-10-22T16:04:40.654Z" }, + { url = "https://files.pythonhosted.org/packages/54/e6/1f01afbc9f4fa8ca5e1fa6fdf537a5c58b5765d1b4167ee284cddb9acc72/coverage-7.17.0-cp313-cp313-android_24_arm64_v8a.whl", hash = "sha256:84538c040f4e6214f663459d89372ce4e6b2bbdd9d397ef0dca7b3ecd56b8eb2", size = 227080, upload-time = "2026-10-22T16:04:42.325Z" }, + { url = "https://files.pythonhosted.org/packages/92/7a/bdb7e6260c4a211389aeac9b89dc31662ebe122153ad0ecfd156897912df/coverage-7.17.0-cp313-cp313-android_24_x86_64.whl", hash = "sha256:6abfb6a4cf1cb1b67bc8fccdde177e2d2b781d0d34449e7078156547025140e5", size = 227152, upload-time = "2026-10-22T16:04:43.737Z" }, + { url = "https://files.pythonhosted.org/packages/37/4e/276e7d5b3cfb124871bdb544c27b004a8b4e270e6bf8e7414149c54b5e5b/coverage-7.17.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:ef65e96f0fc1a7c8b7644c7011d47a8228555e7888ac4d6246f715a7f3d7c398", size = 225397, upload-time = "2026-10-22T16:04:45.292Z" }, + { url = "https://files.pythonhosted.org/packages/b3/7d/dac073db12f3c131598a6693253aded581e4d7d622a89864f11bfb0ca2f5/coverage-7.17.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:55939ea4c097fda2edabf80fac5e49bef15597b950a9f29fc56292a895245feb", size = 225705, upload-time = "2026-10-22T16:04:46.603Z" }, + { url = "https://files.pythonhosted.org/packages/00/45/877b0a9248e921080ae91d115c13192d17e308f5aee86bcc7f7bc2a54ed1/coverage-7.17.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:bfb6f868af552c0ceb2d19220b8cbcc2be096ca26163ac30e682a13041433429", size = 225828, upload-time = "2026-10-22T16:04:48.065Z" }, + { url = "https://files.pythonhosted.org/packages/4f/f5/2c48d47abec0e4590b302920334be5ee3854f96bbd22524feabb8d92b220/coverage-7.17.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:8d823d66afd59566030539c49c7a6acaf3083996682be2dda760c63555842094", size = 226088, upload-time = "2026-10-22T16:04:49.513Z" }, + { url = "https://files.pythonhosted.org/packages/a8/a0/ec0442f3f372c7fd742f49b3022edd392f79e37a0048e1fa333f56d01ec3/coverage-7.17.0-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:50678474e714c684b82c108e63fbb8f1bcf1266d8293a49980f1d08526f093b5", size = 257257, upload-time = "2026-10-22T16:04:50.982Z" }, + { url = "https://files.pythonhosted.org/packages/92/e2/eb0927ba93809f76eee69fa1e47999825fc7dbbabc3e0e1a2744f2677acb/coverage-7.17.0-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:045c1eeb19fb4eae949850a761102fffb3a5f638b97f977912f95843e54f6669", size = 260165, upload-time = "2026-10-22T16:04:52.409Z" }, + { url = "https://files.pythonhosted.org/packages/9a/a2/8d4c6055efb32099b15c9cc33d007b8aabebc4832f372a3274de5ba0afc5/coverage-7.17.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dfb12891a773cbc2132349c77c690a045a882fe3f98b4bda50dfabb01893f9bd", size = 261098, upload-time = "2026-10-22T16:04:53.763Z" }, + { url = "https://files.pythonhosted.org/packages/2c/1e/93412288688a7b03f152a734c8ef086e395108a0a6a746985d3967e220f2/coverage-7.17.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:3067608b4429629f640c1b23d2f69b05d2fd45e00ebb6ee9e8fad220a6c11722", size = 263164, upload-time = "2026-10-22T16:04:55.386Z" }, + { url = "https://files.pythonhosted.org/packages/3c/20/7eb8d805c4f1a212245cef20181dfbf39535357985073d49730ee9cf08d4/coverage-7.17.0-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a1762e85cbbe3b5ff9c6dcbcc1b89d5c72320094292b7b9572bf882c85e428b7", size = 257287, upload-time = "2026-10-22T16:04:56.976Z" }, + { url = "https://files.pythonhosted.org/packages/1a/c1/1df29630b4ed17e135c76390afcc6bd273c3dc5ba2a21a76813a600fe98c/coverage-7.17.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:62bb77987d3f72a0aaa77c567ee7f9806bc2e748c8f46fafbe1fdb90dd148398", size = 259256, upload-time = "2026-10-22T16:04:58.545Z" }, + { url = "https://files.pythonhosted.org/packages/a1/79/2365d1008d28aafe468a450a24c6c63363de794b5140223d67685cd0e205/coverage-7.17.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:6cc31f27240d4e080437e3b8229d92f1d809c9a4f233607385373b73974e183e", size = 257178, upload-time = "2026-10-22T16:05:00.103Z" }, + { url = "https://files.pythonhosted.org/packages/c0/9f/f002952d295d772688dffce0d991e3822795b1e3aad158073712a5a5e86e/coverage-7.17.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:61cde5222f1933d476647c5a25ed1a8678b7703098cb5a15f4a0aa39967d8c28", size = 261275, upload-time = "2026-10-22T16:05:01.462Z" }, + { url = "https://files.pythonhosted.org/packages/40/fe/f9e15fbb426be97cd679ee5d359151e046f103bcd76b5092c318c26ed596/coverage-7.17.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:11f5f7fb2d97a0fe865f5e7bad4eb0619b957637f676bc088d13844adf26ba95", size = 257029, upload-time = "2026-10-22T16:05:02.8Z" }, + { url = "https://files.pythonhosted.org/packages/c2/4a/3c398f38694387334ff4c9f71afc8e15b4fa7db7bc0bea4d5b6032eb0df0/coverage-7.17.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6e09612b1b82c9df1d2b6c85dee88d5aad3e73d77a8ec0b16ac6a7322cbdf5d9", size = 259109, upload-time = "2026-10-22T16:05:04.241Z" }, + { url = "https://files.pythonhosted.org/packages/bd/a9/908a3cf1a2335aec1c788682a06c2f742a62a4f987033f080de1c5fd1997/coverage-7.17.0-cp313-cp313-win32.whl", hash = "sha256:bd3e9eb493631ff1ca34bf5efac25b5c66724f837bf6182c4fc771c55946423d", size = 227691, upload-time = "2026-10-22T16:05:05.636Z" }, + { url = "https://files.pythonhosted.org/packages/13/f7/8cdd7bfd8a63b3e6b133fc395ecfc7b1cb6f561926e7e2ad050de85b83e7/coverage-7.17.0-cp313-cp313-win_amd64.whl", hash = "sha256:f1fc9a7cea899c51d8a53e66c2f082a3d9e461ece32418d3ef0a0b90c32bfd3d", size = 228418, upload-time = "2026-10-22T16:05:07.057Z" }, + { url = "https://files.pythonhosted.org/packages/09/cf/0b3fc57f735cd3831ca13d0be5312a5477d5934066683c5d34018cab7541/coverage-7.17.0-cp313-cp313-win_arm64.whl", hash = "sha256:e65a1e61a68493fbfbe44d8bec71b3231d81b9917f62d3dafe0e105072147e83", size = 228142, upload-time = "2026-10-22T16:05:08.435Z" }, + { url = "https://files.pythonhosted.org/packages/4d/e1/722462fe79912ebebc101497bd16c42413cde2dbae771017f1ec0acb84c2/coverage-7.17.0-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:dc5936491e306f2f6f095bd5fba593de11eb0375cee787ca1f245c3a333a523a", size = 227197, upload-time = "2026-10-22T16:05:09.846Z" }, + { url = "https://files.pythonhosted.org/packages/b7/74/826b9c09f2ebfea803bccb391d935688a25309bff8e819fc446bc629c108/coverage-7.17.0-cp314-cp314-android_24_x86_64.whl", hash = "sha256:968c5fb6e0c83cab752d79a70209648ce4d90cae815c60fa229b59e264601062", size = 227314, upload-time = "2026-10-22T16:05:11.422Z" }, + { url = "https://files.pythonhosted.org/packages/6d/19/c8514d7f5e0659a96ff2bc7377119dca6d420e2761c5fc69ae37618734f8/coverage-7.17.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:12cb8a20ad904a5811e6660d829a586b216e5254b9b34b76dde8a4d65d827dc6", size = 225477, upload-time = "2026-10-22T16:05:12.761Z" }, + { url = "https://files.pythonhosted.org/packages/81/74/ef9433ce599f07d8633b4ce6e6d6921505c4ef3f7f9006f8eb66e2c278cf/coverage-7.17.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:bb5ca66937c55bc85d1a5e9c5900a3e4693a8edf2280a9b3672b7b2661a84b95", size = 225787, upload-time = "2026-10-22T16:05:14.394Z" }, + { url = "https://files.pythonhosted.org/packages/d8/66/26e58981940e8b3944c6a9941bec5c7f50840e7009a046712e3f3bcd9e6a/coverage-7.17.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:5cc15faaf318ee5040d835a62c75bc6f122d837274f3b5efd47bf9fbd4ac5b85", size = 226019, upload-time = "2026-10-22T16:05:15.798Z" }, + { url = "https://files.pythonhosted.org/packages/e9/63/fe175310815cc8357b83b773c9b4c20943c269f217dd825a37c4b4bfe324/coverage-7.17.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:7d6d9d396c8b7bcaf76d774a10140b218111484e54c73a5f2a1b75655381748e", size = 226169, upload-time = "2026-10-22T16:05:17.188Z" }, + { url = "https://files.pythonhosted.org/packages/2c/b0/ac84761f5b4bd226696b0ec4606903d2ef7208abda0aec06c14a4218def4/coverage-7.17.0-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:42e6f170d1ac04b7eb3363310a6c41cdfd82c1fe6bf9ffbc4a86f380bdf091c7", size = 258092, upload-time = "2026-10-22T16:05:18.691Z" }, + { url = "https://files.pythonhosted.org/packages/e9/58/d8bdd65a661913bc12eaaa3e59742b2cb8b94f2e7f78e62184f0a58ca493/coverage-7.17.0-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:acdb5f3c96046efff9dc6c572a6acb2b8d0da09372d7b9a9ed7249cf7d17434f", size = 261096, upload-time = "2026-10-22T16:05:20.089Z" }, + { url = "https://files.pythonhosted.org/packages/7b/15/5d82e2f0a75ea149e0a6412e137e1d610538772fdfd7199737db7fe234da/coverage-7.17.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3393209f6894713161ee82a9b0e673e58237e0f9733c6638e7a6c0c620ef0b33", size = 262080, upload-time = "2026-10-22T16:05:21.568Z" }, + { url = "https://files.pythonhosted.org/packages/0f/cc/e3ebfd295d1dd47c7bc84a17d73f150465a37ab388dd68fe82b495dc79a0/coverage-7.17.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e9494d9e99389d39e6a96227749af898d8924ff591cb63944f8a98e68582bf0e", size = 264558, upload-time = "2026-10-22T16:05:22.906Z" }, + { url = "https://files.pythonhosted.org/packages/e0/9d/e2adff76d6544bbc6df7366d74967923c2de37ba889a006bfb875bf5857b/coverage-7.17.0-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:032d4c2db4c677eb71d43f2f8b80bf64e9346f7a3f7b6ebb13f1782f3cc6c274", size = 257855, upload-time = "2026-10-22T16:05:24.343Z" }, + { url = "https://files.pythonhosted.org/packages/d9/fd/ecd1e9cddf1c36b5d03c438568b1f53e63db84bb3656539d71b454484c7d/coverage-7.17.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:4343a9378b964eb3cb80fa90406385de91c7e982a769ca9314d848f93bc0d3f2", size = 259933, upload-time = "2026-10-22T16:05:25.79Z" }, + { url = "https://files.pythonhosted.org/packages/40/b6/e17a19dccdbcad494c64691f71c1dba3e55c30625bc29ffbb88ac5a0aa8b/coverage-7.17.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:b609df3d6d900464207532c5170a65c6ede8d2e0b91f151f90008a866395cd76", size = 257972, upload-time = "2026-10-22T16:05:27.185Z" }, + { url = "https://files.pythonhosted.org/packages/6a/0d/37dac534709a3fa43a937c8b0604839bb44b8670b83d143a4b78fec20d1e/coverage-7.17.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:a01ef92caeedfa4f2ad171a3a8e6fece4565de871d8bca428151bf9929f41048", size = 262271, upload-time = "2026-10-22T16:05:28.619Z" }, + { url = "https://files.pythonhosted.org/packages/cb/17/7c05186925b4d64a46c271dfd944e3f0c1ad89db9095c31a6df157a75df2/coverage-7.17.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:a3914e2c98c34b94c26d2add3d2a61ccfd3534fc4eaa879ee290bdbb5184ba6c", size = 257604, upload-time = "2026-10-22T16:05:30.07Z" }, + { url = "https://files.pythonhosted.org/packages/95/65/c241cdee66a0ff86c68f6d37328e735d10e3d30cd6ff66f356039eb688d2/coverage-7.17.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:e8a72b26dc5c9a9422e45472b1d67ba5caf273c524c8668cc04b23062663755e", size = 259714, upload-time = "2026-10-22T16:05:31.402Z" }, + { url = "https://files.pythonhosted.org/packages/db/09/e93233fe2405574ce0a31ed201c0cba3a860e085db5d2828bf12b7dc0790/coverage-7.17.0-cp314-cp314-win32.whl", hash = "sha256:03baf0688e14a5a44ffd1eefa8928dfc92b14565e521174e072b31b971aa1657", size = 227914, upload-time = "2026-10-22T16:05:32.78Z" }, + { url = "https://files.pythonhosted.org/packages/d7/77/04979eb217edb94b94afabb4bcc179f60c746f68d3fd0b12688ba9195730/coverage-7.17.0-cp314-cp314-win_amd64.whl", hash = "sha256:3a4459b0f389dd1e51aac3dbfc4ca35b30cae7dc99caa3c0a086d1997a985709", size = 228543, upload-time = "2026-10-22T16:05:34.118Z" }, + { url = "https://files.pythonhosted.org/packages/a0/38/155f2f2c6769f8c43eca8a5e581443faa0908828650090b97103b4a41376/coverage-7.17.0-cp314-cp314-win_arm64.whl", hash = "sha256:a32ee5831b15ba239bef5d272ae586ca8acea4a661b5f2941f5a0d380e518b23", size = 228396, upload-time = "2026-10-22T16:05:35.951Z" }, + { url = "https://files.pythonhosted.org/packages/52/55/d6751572d9445485c4c5a51ec6ae11caa5c11b7ef2448cdccc884f7f54d9/coverage-7.17.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:9c083e253e96fdeeaf48133fe836cac86ace2db215d235700e7f7532420a200c", size = 226486, upload-time = "2026-10-22T16:05:37.363Z" }, + { url = "https://files.pythonhosted.org/packages/2f/a8/82d37760e537bfdb9fdee285fa1e1b45bc153293d23e2331c7befedd1990/coverage-7.17.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:b88fb4d4d7608cbcebdc7ff7dbffc2115e3b81d6c33a79147cd79d54fcb080f4", size = 226861, upload-time = "2026-10-22T16:05:38.777Z" }, + { url = "https://files.pythonhosted.org/packages/24/96/70ec00a9db6d7bdd3f914eba12057360769f5cb78a98456e71fe1937deb5/coverage-7.17.0-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:2bf3ee38a40c0570c316b9e7507c472469586906a20f289705898528c31aaea8", size = 266849, upload-time = "2026-10-22T16:05:40.285Z" }, + { url = "https://files.pythonhosted.org/packages/77/d8/d99be3bfbb208512f039c80b31e404138a86678fc929a17acd5b7135a6ae/coverage-7.17.0-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:7c570293f001ea91598b90e90cb9b98fddee833e90bbede432cbd218331ccd8e", size = 269402, upload-time = "2026-10-22T16:05:41.585Z" }, + { url = "https://files.pythonhosted.org/packages/82/50/6cbe8b5a5060edbd21ce8109c4e900f8b7656e8b72ede6dc4d82dc0b8bfb/coverage-7.17.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:791f9ee09814dd30d9562b4f4eb037a371485ec1c830b53999d2320d20c4e764", size = 272025, upload-time = "2026-10-22T16:05:42.921Z" }, + { url = "https://files.pythonhosted.org/packages/bd/05/d3633add01e5518dfdacfdbd1be41cb5c581412dd8bccd663edf6ef7fec8/coverage-7.17.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c7c5672ecf1b497de7a0b37743bef40e3b58e3207e9fe2ed6682cb0420836a88", size = 273515, upload-time = "2026-10-22T16:05:44.324Z" }, + { url = "https://files.pythonhosted.org/packages/57/d9/b83387780271bf9ea9f96a3aeb01c1c302c4494a927923c4fa70cd170a3f/coverage-7.17.0-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:337f87e62e1c5f43c8198d879f677c342d152f84d3f157add12558853cbb8eaf", size = 266725, upload-time = "2026-10-22T16:05:45.713Z" }, + { url = "https://files.pythonhosted.org/packages/e4/be/8e80b29764ae3f4d70d44952eee3f3927eb36897490fe799e400adb7f5da/coverage-7.17.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:c8a58baa8eb6154c6645eadcc19b6c089deab78c24a612a83206f2a8a5992a7c", size = 269312, upload-time = "2026-10-22T16:05:47.054Z" }, + { url = "https://files.pythonhosted.org/packages/b8/c9/e702f588fe8d83ca844f133270ad0a49bf650c8cf260551ff6f11a6b6bbf/coverage-7.17.0-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:fb5745048ee20f796b0e3ab61831049e749acff188563e2af32f58ede957ac1b", size = 266740, upload-time = "2026-10-22T16:05:48.343Z" }, + { url = "https://files.pythonhosted.org/packages/8a/11/ee13ad14b714d75ef07c490b6e43969574a8b8b8c6d67bc741fcbacbf9c3/coverage-7.17.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:7b5abcae74750302ac4bc77351eaee87c450e4abd3b6225eacde3858e260f8b2", size = 271036, upload-time = "2026-10-22T16:05:49.727Z" }, + { url = "https://files.pythonhosted.org/packages/e2/bc/854e8912c11c54aaf416e74602000589c6efbbe1f251dc90202fdabddcb6/coverage-7.17.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:4e153f0bd6509dbf837fd585f489b3328bfd98f2a718a0b15ae63032148b285c", size = 266158, upload-time = "2026-10-22T16:05:51.141Z" }, + { url = "https://files.pythonhosted.org/packages/a8/dd/96432830ddf63bc42654de54eee77a0cd6949cc65769bda06f353323f23a/coverage-7.17.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:e42a835271922fe7bf98c7a05c78d4032f4e52bf5b07fafa1eadfdfc2f0ad22f", size = 268088, upload-time = "2026-10-22T16:05:52.92Z" }, + { url = "https://files.pythonhosted.org/packages/ce/ee/230c6ec065a6629d41cc89323ceb54d37e2650c104f70fc108f4ee7c5051/coverage-7.17.0-cp314-cp314t-win32.whl", hash = "sha256:5900055351ec89fdffb34b27898815106529ded29a9496464c1ebd6313fd40fa", size = 228229, upload-time = "2026-10-22T16:05:54.255Z" }, + { url = "https://files.pythonhosted.org/packages/f0/1d/23eb6aadcb3c00764ff2961c0ff55f5dfa2543856d932b949bbc04fa68e9/coverage-7.17.0-cp314-cp314t-win_amd64.whl", hash = "sha256:5ee9c8bd13a59e301dc2aefe7aa8384633406808de051b1982775667094eb673", size = 229154, upload-time = "2026-10-22T16:05:55.682Z" }, + { url = "https://files.pythonhosted.org/packages/3e/f1/7df117e8e016b7a3ee459e833b3456f3610505afefd071f8727702d98439/coverage-7.17.0-cp314-cp314t-win_arm64.whl", hash = "sha256:0a9cb9a0fb58e6fff5442969af36094ecd6976f4907f179e33b27e15355d8f74", size = 228726, upload-time = "2026-10-22T16:05:57.179Z" }, + { url = "https://files.pythonhosted.org/packages/9f/a1/f9c24911755f91e6f451ca2e2c1804fbcbe9fecbc3310e341bbbbfa1d85b/coverage-7.17.0-cp315-cp315-android_24_arm64_v8a.whl", hash = "sha256:adabc1079ba844b096799ba8638d8df434aca1613f95bb7d5791af320cf05559", size = 227196, upload-time = "2026-10-22T16:05:58.543Z" }, + { url = "https://files.pythonhosted.org/packages/b0/a5/2b665b40c626e6bb40e5e8e23d91170b48b5f2f57186f46b07f2fac42386/coverage-7.17.0-cp315-cp315-android_24_x86_64.whl", hash = "sha256:6b4593d2871c4c801656dd5861e759f8f5156febd6b6061b02f6a72d64618b7b", size = 227315, upload-time = "2026-10-22T16:06:00.409Z" }, + { url = "https://files.pythonhosted.org/packages/3d/8c/22b518d61b930b8a4d62905d046c76a03675ef19b466462011dfb16439a3/coverage-7.17.0-cp315-cp315-ios_13_0_arm64_iphoneos.whl", hash = "sha256:c2c48a36bb970b7265b196032bfd4237c0b373f2b25ef7012d7e905b5b35c19c", size = 225490, upload-time = "2026-10-22T16:06:01.743Z" }, + { url = "https://files.pythonhosted.org/packages/08/5b/d3b141455944e030cd4987f055f8c84a44a1a0b28d81ae93389f44051691/coverage-7.17.0-cp315-cp315-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:f8bc2cd4c6aa58be1ca243fb8cddb6c3c42031fe4403991ad9cf2b3baed03d9f", size = 225797, upload-time = "2026-10-22T16:06:03.423Z" }, + { url = "https://files.pythonhosted.org/packages/03/24/aab2156983f7022361352640c583d08f0238e4cd7463eb921172001b4cfe/coverage-7.17.0-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:cbf0b83ce927de6053279ba61342d9f8892afde1385fcba9144cd05d4023d109", size = 226025, upload-time = "2026-10-22T16:06:04.76Z" }, + { url = "https://files.pythonhosted.org/packages/87/db/94697b9ccf4ec94cdad07bf94e2a21b9702247cb212d2d5f18d2b9c3d0de/coverage-7.17.0-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:66efc0fb99cee34626fc17d3adac9c3cd816bfdad563bb1e983c3ef432f566d4", size = 226193, upload-time = "2026-10-22T16:06:06.167Z" }, + { url = "https://files.pythonhosted.org/packages/a0/fd/db1c4ade4219831d590fff5aa5b7d9ceaf84586e8e60da6fff71b6182251/coverage-7.17.0-cp315-cp315-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:58a304efdf2253fb2ea03d45d0a04cc174a92a0cd5279c28c5bb893ce7410968", size = 258046, upload-time = "2026-10-22T16:06:07.51Z" }, + { url = "https://files.pythonhosted.org/packages/95/ee/84ed138990ed752abd73db4adffc1842d6152017d8e21809cc109a413fda/coverage-7.17.0-cp315-cp315-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:aed06129a9f530a948928260d7d2d81c47261355b0127e3c782251b38958463c", size = 261735, upload-time = "2026-10-22T16:06:08.861Z" }, + { url = "https://files.pythonhosted.org/packages/64/ad/c1e4367bbd1e7d562d6bc979c82b542f2040544e95a8cd995b694caf0e1a/coverage-7.17.0-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f6cdb47a9be2d3313829bed1d296086287682a3ee901ff987b3432b45705b771", size = 262819, upload-time = "2026-10-22T16:06:10.49Z" }, + { url = "https://files.pythonhosted.org/packages/b5/3a/e8e8509b0940403eab15d86f70022790a01fce3e971a161919c544953f7a/coverage-7.17.0-cp315-cp315-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:804fbaf81d09b34e2f8b1159d01e211a50713d743d32fd9b2ca6cf0ae507a66f", size = 265194, upload-time = "2026-10-22T16:06:11.847Z" }, + { url = "https://files.pythonhosted.org/packages/1f/77/d64bbdbb734b1059ebcde2b5df567f142a9948079e51af9b0abf8c9ec17b/coverage-7.17.0-cp315-cp315-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b2afebb96da9a1bbca142f8cc12320c14e5c215e0a49cded4f191f5a31506a28", size = 258410, upload-time = "2026-10-22T16:06:13.234Z" }, + { url = "https://files.pythonhosted.org/packages/e2/29/7cf81f08a254532c074eb52cfe6b12bb753d08f2997269c9fac8de359541/coverage-7.17.0-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:bd04fd23206be9ad3de13b0b39313094fafa66a1f6baf70f5471c004a9fac47d", size = 260697, upload-time = "2026-10-22T16:06:14.595Z" }, + { url = "https://files.pythonhosted.org/packages/7f/56/bacea6dee2d5ccb31ab0cfffcee87ca7efa80ff048840209c8013aeaf02e/coverage-7.17.0-cp315-cp315-musllinux_1_2_i686.whl", hash = "sha256:5ebfbc87836529a31d25c2dec1262c38eea852ec1c46c0f690a515a7339d9c5b", size = 258649, upload-time = "2026-10-22T16:06:16.265Z" }, + { url = "https://files.pythonhosted.org/packages/a4/97/fd8a6a63740f9ee177c53b59e2f4d0752369bb0fd154086a1e69aa1fb255/coverage-7.17.0-cp315-cp315-musllinux_1_2_ppc64le.whl", hash = "sha256:a59c3b470ac3b2432af3598a996affbeb10236cd899a005c1c4e517fcff8b592", size = 262989, upload-time = "2026-10-22T16:06:17.85Z" }, + { url = "https://files.pythonhosted.org/packages/c1/37/7cd3d3520d45db3788456452b65446182c08f2a7c276f40c9dd11966be02/coverage-7.17.0-cp315-cp315-musllinux_1_2_riscv64.whl", hash = "sha256:7fef12cec4b96f59e09761cbd4fc9bdf4788062d1ae435b920241fc16c8a9549", size = 257955, upload-time = "2026-10-22T16:06:19.563Z" }, + { url = "https://files.pythonhosted.org/packages/c1/b9/80343a55e07d854927a915cc21a52e350ae1db9ead33f0248507b960785f/coverage-7.17.0-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:a14e8d69464da9cbe1dc29e7116dbd37296d1d51a695327c478e250349085089", size = 260512, upload-time = "2026-10-22T16:06:21.055Z" }, + { url = "https://files.pythonhosted.org/packages/f3/32/e4c816f0cb3efdc6392bb276ff88359729f9dec605bdd3c1c7a44776f759/coverage-7.17.0-cp315-cp315-win32.whl", hash = "sha256:9a66074231799d8884d267234d49f60c90b43a945dfe069646b72f3058cd4e67", size = 227913, upload-time = "2026-10-22T16:06:22.407Z" }, + { url = "https://files.pythonhosted.org/packages/43/14/283120142ac281834808584249b89957de997c28573ddd65ae106b622121/coverage-7.17.0-cp315-cp315-win_amd64.whl", hash = "sha256:79cf71e5f9a48877ccfed5baa2c72f45af4feeaa596249e23eb15b473134830e", size = 228532, upload-time = "2026-10-22T16:06:23.756Z" }, + { url = "https://files.pythonhosted.org/packages/b9/bc/83dbc6c3bd58042afaefcb0cfd7d7a2e3076989a2f3d9bc6e6edac819784/coverage-7.17.0-cp315-cp315-win_arm64.whl", hash = "sha256:09e563b49c81cd0a33734e926d7a4a6ef1778d74304cde2dcc3aff8b30ea1036", size = 228393, upload-time = "2026-10-22T16:06:25.163Z" }, + { url = "https://files.pythonhosted.org/packages/00/ed/9463d1cb1766a2570343a372c2871c7fb4d561e5051f029399f4d57ce947/coverage-7.17.0-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:a5e2e0625dfc789af0b2e10acfc106b352f314fffbd19e6392a7961c8cbe5d9a", size = 226481, upload-time = "2026-10-22T16:06:26.563Z" }, + { url = "https://files.pythonhosted.org/packages/ae/59/6536c7fa87be13236b4184fc395f14d18f622a5b8ef7448a6bfdf5f24ec0/coverage-7.17.0-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:194562ab1eff0d85aee8a17ce4c5c0bc1ebf70a629a7145257123156ad100fd6", size = 226876, upload-time = "2026-10-22T16:06:27.921Z" }, + { url = "https://files.pythonhosted.org/packages/b0/bc/4ffae6038f78382c9ab6a4969cb5fc01d50b083d119851085807629abe88/coverage-7.17.0-cp315-cp315t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:85682ca89bc4601125dcdff23fa70e6274e13c9ad502fba845e6c274ab2c6569", size = 267241, upload-time = "2026-10-22T16:06:29.356Z" }, + { url = "https://files.pythonhosted.org/packages/c0/60/cae4b708b4217d33ebf66b9c57f8910a257346b3eaf054a78d73605ddc4c/coverage-7.17.0-cp315-cp315t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:0d0fac604a962ff546beb038cba9a5a09d88e6e3f01612fb46f7bd4fe5dc8dca", size = 270079, upload-time = "2026-10-22T16:06:30.736Z" }, + { url = "https://files.pythonhosted.org/packages/a9/84/1225ab04e55a6a2e483e903a5eb0f8f081cb5325df833594534aa288d91c/coverage-7.17.0-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:72edd4f9483ae30cd1ba4f230687840ef7b58a16afb1c9e81131fd6d64655f36", size = 272775, upload-time = "2026-10-22T16:06:32.247Z" }, + { url = "https://files.pythonhosted.org/packages/94/ad/567f8dd9246a553aa8753732c41f0e0654e98e9a617d6282e64eae0e6aa8/coverage-7.17.0-cp315-cp315t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:91e487ad07c511da77b574820141a7c2da09bc2a4b4090b5be2424295d100697", size = 274110, upload-time = "2026-10-22T16:06:33.859Z" }, + { url = "https://files.pythonhosted.org/packages/66/5f/e0ee76895df40601e4aedffb6b8e71ea48e276a629fb4488b92cffffca14/coverage-7.17.0-cp315-cp315t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:1cabb31531002fef5197de5df3c84730aef01e750e25154b9813a2583435aba2", size = 267775, upload-time = "2026-10-22T16:06:35.193Z" }, + { url = "https://files.pythonhosted.org/packages/19/6d/d19c6e45e940e5b9ba697b7be9a8e63cef73d90bdea45b670a59df466717/coverage-7.17.0-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:c9ac886242bbd50fb35a2242ba79555a99891e657da33f98b7ca968e3488a350", size = 270268, upload-time = "2026-10-22T16:06:36.597Z" }, + { url = "https://files.pythonhosted.org/packages/27/a3/b2731bb7aef2db800b330603675c3fa58a6497a05d5b5f8625f3b6285f68/coverage-7.17.0-cp315-cp315t-musllinux_1_2_i686.whl", hash = "sha256:d45cdc495671fc99268ff3653e0da4244972e9fc0e1fc3ea9de18781ad28963a", size = 267298, upload-time = "2026-10-22T16:06:37.973Z" }, + { url = "https://files.pythonhosted.org/packages/50/75/d0c90a81b6b6607933109914dc8b0ef0a5f4cacf9482ca868ae773ee5d0b/coverage-7.17.0-cp315-cp315t-musllinux_1_2_ppc64le.whl", hash = "sha256:12961f738d717a046d4bf3f8d1727eb3ebe2ed5478ff96cd6233d4dfc39323d3", size = 271899, upload-time = "2026-10-22T16:06:39.389Z" }, + { url = "https://files.pythonhosted.org/packages/64/1e/f038f3a2614e9ef839a40a184713aca40b8ed79e626ab5cdc388771045df/coverage-7.17.0-cp315-cp315t-musllinux_1_2_riscv64.whl", hash = "sha256:1ee0a982b0e1230b3edf768a7dd022bd2377303763895327ac62deb5efacee03", size = 267380, upload-time = "2026-10-22T16:06:40.97Z" }, + { url = "https://files.pythonhosted.org/packages/c9/30/8d636e639b2d8b1934128573402a7caf66c8cfbc794fa02061d84f115c34/coverage-7.17.0-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:460679f53893081edb94eeed011d26be37228ba649b853fd2aed7e4394fb1f35", size = 268993, upload-time = "2026-10-22T16:06:42.361Z" }, + { url = "https://files.pythonhosted.org/packages/73/7a/219f6b391cb0636242d06c390a8add5af91afa3e92afd0380dbedd34f487/coverage-7.17.0-cp315-cp315t-win32.whl", hash = "sha256:600e121e412cd6bdb52ec105b95c6a04da0c9710d1534f6e64a42db667907802", size = 228216, upload-time = "2026-10-22T16:06:43.714Z" }, + { url = "https://files.pythonhosted.org/packages/f0/3f/514d294319c5eac11494394545de209079f33993e14526c220eab29b2a96/coverage-7.17.0-cp315-cp315t-win_amd64.whl", hash = "sha256:c3e11b81faafb1f063248735c3e2c5c4183fc1db2c7f46782170dd7760dffcbc", size = 229146, upload-time = "2026-10-22T16:06:45.262Z" }, + { url = "https://files.pythonhosted.org/packages/f6/82/c0d8bb0a67df838fc7bc6f4060ffc404fd1154fd18e624d83e6abc41aeef/coverage-7.17.0-cp315-cp315t-win_arm64.whl", hash = "sha256:d472ee1c64868a917a47f02ffb4ac8c22b477f7589d57a3eb0c58b1ad4f0137c", size = 228716, upload-time = "2026-10-22T16:06:47.097Z" }, + { url = "https://files.pythonhosted.org/packages/c9/17/48032919343afa2906112d458aceda2043afdb24a2ee8eb4c60904d8ddf1/coverage-7.17.0-py3-none-any.whl", hash = "sha256:bac487dce0d2aff28ddf1a533a916f771416eee0e493960f611e49037b878e9d", size = 217383, upload-time = "2026-10-22T16:06:48.471Z" }, +] + +[[package]] +name = "cryptography" +version = "50.0.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cffi", marker = "platform_python_implementation != 'PyPy'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9d/af/182eb91b0df3fe75c4d9f26fe70684569566745f6ba7e5c9c73a862c5252/cryptography-50.0.2.tar.gz", hash = "sha256:7b46165bb56eb4704e2eaaf86f3c940d19154535d9b0ca7d6d590b04060e00d5", size = 880623, upload-time = "2026-09-30T15:30:04.884Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e5/56/d194340cc4a57535e82e1bee9e89667ac4b7c13b5d3f59686deae3094dd5/cryptography-50.0.2-cp311-abi3-macosx_11_0_arm64.whl", hash = "sha256:fa8f5efb344d6908a1ce62f4a24e2e5780f825d6f53f5f50ec5ffacac72936cb", size = 3914904, upload-time = "2026-09-30T14:43:44.339Z" }, + { url = "https://files.pythonhosted.org/packages/d9/69/c9bd862c3bf43d6399c433caf002df16e2dffd4be49bdf515cda38038711/cryptography-50.0.2-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:79def8d059362e7831389ed3be0ecdf58a89386e1271e35dd9f5af84e81bffd0", size = 4731146, upload-time = "2026-09-30T14:43:47.113Z" }, + { url = "https://files.pythonhosted.org/packages/21/69/64cef1f702bf6657e0cc186ed1a2891d50d29fb41586b254e1c07adea261/cryptography-50.0.2-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:630ebfea3bf689d075f82316324ff7433dc447fe6bc1bfc76524b74b4a9567d2", size = 4719841, upload-time = "2026-09-30T14:43:49.01Z" }, + { url = "https://files.pythonhosted.org/packages/38/6b/61a3f8d8c5e1e49a6cddccafc4015cc1c0021360ab0acb4080e7a423644a/cryptography-50.0.2-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:f9f6143a8c75945eb960d9eb98905a441394abfa24afaae239d514ffb2586480", size = 4738340, upload-time = "2026-09-30T14:43:50.932Z" }, + { url = "https://files.pythonhosted.org/packages/7b/2e/7212ca32fd43dc91f2f41db20160b268098874b4c9a0e7be94d6835f5b2e/cryptography-50.0.2-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:a582ab2ae1d34f67112cadc86702774c9ea4374df6bca6afe672817203c99134", size = 5367029, upload-time = "2026-09-30T14:43:52.911Z" }, + { url = "https://files.pythonhosted.org/packages/1a/f1/b474e930c4d910328780e3940da76f5aa5cbc48ce1fc14e44d239d9ea9db/cryptography-50.0.2-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:4061c0079120205fb760c58acab6443e217307dcf05e3702cf970e0689972856", size = 4753050, upload-time = "2026-09-30T14:43:55.272Z" }, + { url = "https://files.pythonhosted.org/packages/7c/52/9af10e80ac16b0fcc2123f9cbd5e7afbd0fd5075bb7a607c592258a39cda/cryptography-50.0.2-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:ac9ed99d81760c62fe89d5f0815cdfa1ba9a35141cf30f1c2d044f04b4803d2e", size = 4376724, upload-time = "2026-09-30T14:43:57.24Z" }, + { url = "https://files.pythonhosted.org/packages/71/37/6202e488cc1eb625ea110c292c6bda92823176e023f427d8d5660ce8d632/cryptography-50.0.2-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:87e9ce85beb6b328ba370cc6e6aea483c92617b4c95b1d33a49297eb662bfb04", size = 4737859, upload-time = "2026-09-30T14:43:59.541Z" }, + { url = "https://files.pythonhosted.org/packages/8f/30/e86d7d518489b0ae2497091a35287abcb1a2ce4037837a34afbe9b1d6964/cryptography-50.0.2-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:f265528741e048bce55c3463ed721fb0aa45a5888d8add8cfeccb3035451bbdc", size = 5324103, upload-time = "2026-09-30T14:44:01.901Z" }, + { url = "https://files.pythonhosted.org/packages/d3/69/2c833a049475e0a3444e94c7d0aca0aa51d166374a449b09e92ac98138de/cryptography-50.0.2-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:9dab55f57c74c3cad24c323bacbbd04be4705ba6eb0d92e920b1fc4837ed5079", size = 4752576, upload-time = "2026-09-30T14:44:04.545Z" }, + { url = "https://files.pythonhosted.org/packages/6c/5d/906970b83bbfc1f5bbfb677a143c181f2801f23b6a7204a3b47c42c97e65/cryptography-50.0.2-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:25784ce8b9621c90c643efb9e1e2162ab3b0224cae446ad5e70e7fcb1ce18b51", size = 4870819, upload-time = "2026-09-30T14:44:06.884Z" }, + { url = "https://files.pythonhosted.org/packages/68/e3/f2298d3bb55e0c4a91841ec4d01b3f020ba8c5fbf15ccdcc6dcf03f97025/cryptography-50.0.2-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:85d0d9a31b9098e98534226d5686b47264b95e62ce459dc2e62fdfc809f9fe93", size = 5030152, upload-time = "2026-09-30T14:44:09.443Z" }, + { url = "https://files.pythonhosted.org/packages/9a/4f/adfc442765721292fff86d314ce385d3249d22db42295c0dd057727b60f3/cryptography-50.0.2-cp311-abi3-win_amd64.whl", hash = "sha256:7afa5a6602a9f29af1f3a2965f831bae7c9d5d597b7cbb716d41ab3b7d89879c", size = 3824692, upload-time = "2026-09-30T14:44:11.671Z" }, + { url = "https://files.pythonhosted.org/packages/ce/cb/52eb3770c0d0be2702a98c6e96065ddc0a2877cf0845aa9c23397c142cd4/cryptography-50.0.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:f785f6161f202ab04d8ca194158968798e480ca058943907972da5f12e2881e8", size = 3892731, upload-time = "2026-09-30T14:44:13.485Z" }, + { url = "https://files.pythonhosted.org/packages/19/8e/aa1fc533d4546b127b45de8aa024eb5933d23eff9debfe25931e56861095/cryptography-50.0.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:0ecbc5652bdb6fc9eaf89a7d196e20941adfe812f43bc4ca05d9150496821047", size = 4710431, upload-time = "2026-09-30T14:44:15.427Z" }, + { url = "https://files.pythonhosted.org/packages/6a/64/72bc3f75176e7e406b748a3e3830432b8c51297b38368713df04dc04898a/cryptography-50.0.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ab50ee449bf968271e820086f10a33d101dd060370abc10bcd22279be2656539", size = 4694824, upload-time = "2026-09-30T14:44:17.69Z" }, + { url = "https://files.pythonhosted.org/packages/4e/c6/62c77550edfa5ca3f14bf44a1e6739b9fa09d6e998a11d97ed8213bccc98/cryptography-50.0.2-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:a9f7355e6fab51f6c369b86fb7571cffa05edee2c2121e0380a37fb9ac1cd5c1", size = 4716967, upload-time = "2026-09-30T14:44:19.661Z" }, + { url = "https://files.pythonhosted.org/packages/f4/37/cce70f150c432914460157a6ecc161752e053aa5ec0ef3b3f7dc6e31039a/cryptography-50.0.2-cp314-cp314t-manylinux_2_28_ppc64le.whl", hash = "sha256:94e5e9f108ee10471288214d3d233fbfbb492840a8457eb85178d643ddeb32c7", size = 5328676, upload-time = "2026-09-30T14:44:21.744Z" }, + { url = "https://files.pythonhosted.org/packages/aa/9a/6f2f0304d634ceafdeaf23e84537336664ac419b5d07611675c2ad3f6b7a/cryptography-50.0.2-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:241449bf940a5d27309bd317e6f9a2af6932113818bb2b8f5c59ddc7ef16da18", size = 4727698, upload-time = "2026-09-30T14:44:24.178Z" }, + { url = "https://files.pythonhosted.org/packages/1d/de/66bcf9244d118663b2e1aaded8990f4640e3d7b7411870a5765f252074d2/cryptography-50.0.2-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:d8947001be83df1394050758ce0e745dd74fb134eef0a4b5124208dfc3a68c37", size = 4354821, upload-time = "2026-09-30T14:44:26.263Z" }, + { url = "https://files.pythonhosted.org/packages/bd/e6/db28a28c7b6c676addce89136de3d8db49ea825a8c863472e36e42ead4ad/cryptography-50.0.2-cp314-cp314t-manylinux_2_34_aarch64.whl", hash = "sha256:4a20ce1e5cb4284a86692fdcba7cb8754185c6b2e5c56fcef3751cf451d3cdc2", size = 4716748, upload-time = "2026-09-30T14:44:28.447Z" }, + { url = "https://files.pythonhosted.org/packages/30/96/01546c7f69ea0e2ab790a2e4f0934a4052fb9b388147fbf83c2fd72f1e57/cryptography-50.0.2-cp314-cp314t-manylinux_2_34_ppc64le.whl", hash = "sha256:84f964e537f916e2cc85199e5a88742e964939b575ac8598b3f9d6cc416cdaf1", size = 5285085, upload-time = "2026-09-30T14:44:30.704Z" }, + { url = "https://files.pythonhosted.org/packages/6c/01/03263395f74d50b071e9e66daace3f8bef80493e5d410726f2ba8554736b/cryptography-50.0.2-cp314-cp314t-manylinux_2_34_x86_64.whl", hash = "sha256:828d49b0ff5a0e3975865571c5d91dbbdd0d38d8289b249a163e9425413a5e05", size = 4727268, upload-time = "2026-09-30T14:44:32.92Z" }, + { url = "https://files.pythonhosted.org/packages/eb/94/2bfe8f29ec0cc9c0d99359c4161adf32858e4934b72c6d100d2ac0bbe962/cryptography-50.0.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:deb9fde5c60e437ee4821bc9bc39ff31b42135c27e1dc61ef0a629389c1de62e", size = 4849503, upload-time = "2026-09-30T14:44:34.969Z" }, + { url = "https://files.pythonhosted.org/packages/54/44/e80651ecbf0e42b62e2bb5f5768916e07eea72e1297338956a61df361f88/cryptography-50.0.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:8c71ba2cd31fc93748c38e1b613200ff1c2665cbfd5341fe3a61cfde35a1430e", size = 5004057, upload-time = "2026-09-30T14:44:37.064Z" }, + { url = "https://files.pythonhosted.org/packages/f8/cc/1d33befb3cd7ea7e77d2d73f43f2066471da1b21f24a6156efcaabf6d2e8/cryptography-50.0.2-cp314-cp314t-win_amd64.whl", hash = "sha256:78198641e5be9521beea5aa782bb551a58068d10e6eb04c9c680c1b69f2e7d45", size = 3795868, upload-time = "2026-09-30T14:44:39.71Z" }, + { url = "https://files.pythonhosted.org/packages/2d/49/93f6a6e7a87c9aa68d44d3e1cdb5fe8f60c90d5d2f46acae9a56892816b8/cryptography-50.0.2-cp315-abi3.abi3t-macosx_11_0_arm64.whl", hash = "sha256:edc3342adf8f697fc5f59c887a304356f147b397809440ed64e2fa6af2f50f37", size = 4133708, upload-time = "2026-09-30T14:44:41.807Z" }, + { url = "https://files.pythonhosted.org/packages/8c/75/32ac2a56243d778805c16ca6a32b8f74fb757df7e28d7ecb560afafb59cf/cryptography-50.0.2-cp315-abi3.abi3t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d370b8d1dfcdf7130178137f6fbee6140774a1acc6cacefc4b42643ec11d0a3a", size = 4956267, upload-time = "2026-09-30T14:44:43.693Z" }, + { url = "https://files.pythonhosted.org/packages/aa/a4/2c8d734e43d97f0842ee9f1b7b4bfb3d0cf5e19edebf43c2afe6675c2320/cryptography-50.0.2-cp315-abi3.abi3t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f2f9bd7f90c64fe89253f0a2c05e3c4856072660429ce8831b4235bf29403a67", size = 4966465, upload-time = "2026-09-30T14:44:45.769Z" }, + { url = "https://files.pythonhosted.org/packages/c2/58/ee288c829a6f41f6235ae9dd33d82fd19b45442b65b4c8a3da36963d9f7a/cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_28_aarch64.whl", hash = "sha256:e275096ea1e60cc595cda2836fd4a6c725d1125108b868be17f53684d164e2cc", size = 4959356, upload-time = "2026-09-30T14:44:48.211Z" }, + { url = "https://files.pythonhosted.org/packages/92/20/9ded6d51ddd9897f6b6e81fb9ebea7951d7cc5d6c890b0ed8abf77a51a80/cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_28_ppc64le.whl", hash = "sha256:b13478603dcd0a2479ff8e87e2c19a7d525734686fe3c49542472293a204212d", size = 5548822, upload-time = "2026-09-30T14:44:50.86Z" }, + { url = "https://files.pythonhosted.org/packages/02/a8/8df951850d6b31d2a00218f19e2b3f999523437ed7a819df7fa427942fca/cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_28_x86_64.whl", hash = "sha256:58a0c478eeca76fe5e07993c5a0703def34a6dc6a0cda4f5564639b33112ffe7", size = 5001199, upload-time = "2026-09-30T14:44:53.379Z" }, + { url = "https://files.pythonhosted.org/packages/8b/f9/36b3022218ce75b7cdf068fb95f809f9bd0d820e4955ef43b90c255cc7ac/cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_31_armv7l.whl", hash = "sha256:d38cdff612d06fa6a32840d5e1b1f7a27cee4a349aa9085d94a67789d6bfd408", size = 4629333, upload-time = "2026-09-30T14:44:55.635Z" }, + { url = "https://files.pythonhosted.org/packages/8c/72/20f99a219f6af47cdd1cbd978c243b92d71496e168a746138af44ded4f29/cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_34_aarch64.whl", hash = "sha256:fdd28f912fccfec1846a94e2e1e8f9b0012f557f0c46fe4f3eb0d7a87afcf90b", size = 4958822, upload-time = "2026-09-30T14:44:59.639Z" }, + { url = "https://files.pythonhosted.org/packages/f2/20/196f112617fb08eb4d608a2a6c422373d46f9cc2857f38fc0667033c0899/cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_34_ppc64le.whl", hash = "sha256:cbc8738fd8526d80f35cb3a40d41f41a2e7030bb3b18b09a6778ef63d291c2fd", size = 5506351, upload-time = "2026-09-30T14:45:02.267Z" }, + { url = "https://files.pythonhosted.org/packages/24/95/83378121ef3eaaaf71d4b781577ff794acb39b9e1b87a3f156898c8497ed/cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_34_x86_64.whl", hash = "sha256:e105ab60406787da31fccc883fc0f733af1efd78f0136a4599692c4083a73d0c", size = 5000859, upload-time = "2026-09-30T14:45:05.009Z" }, + { url = "https://files.pythonhosted.org/packages/22/f7/70fd7ae4d1dbfa7ba29b02e1b9068771519a86027756510b700ce81086a8/cryptography-50.0.2-cp315-abi3.abi3t-musllinux_1_2_aarch64.whl", hash = "sha256:6f8700550aa1474a91e5dc07049c46f98b423b5b1ddd0483e0b51362eeeaf5be", size = 5092151, upload-time = "2026-09-30T15:29:15.932Z" }, + { url = "https://files.pythonhosted.org/packages/d4/be/688367b74de86984bd58d8efacfc7c9e68b89a6a22ced0fb4f38db50254a/cryptography-50.0.2-cp315-abi3.abi3t-musllinux_1_2_x86_64.whl", hash = "sha256:c71be1cbfa5cd9a41ee452acf1eccd82b2c05950358b106ec8ceb83411d1a020", size = 5286120, upload-time = "2026-09-30T15:29:18.309Z" }, + { url = "https://files.pythonhosted.org/packages/39/d1/55f8a3f2ef5d1529e16835ef10cf0fe3d559ce237b46dddc440c0bba3649/cryptography-50.0.2-cp315-abi3.abi3t-win_amd64.whl", hash = "sha256:c423ab384a46c4dff7217b2ea5ba2e11cffdeab6441acd04cf65a369caf0366c", size = 4111557, upload-time = "2026-09-30T15:29:20.155Z" }, + { url = "https://files.pythonhosted.org/packages/23/ad/ac987755d00e1e64273760228d2635ae38dae2be83e3c6e0d3289d91dec3/cryptography-50.0.2-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:0ec5f09541743261e66e291b4a0cbf0fb2997aeaab6d9e9c740b9dba1b58d1c2", size = 3943588, upload-time = "2026-09-30T15:29:22.265Z" }, + { url = "https://files.pythonhosted.org/packages/d5/8d/6d585339bedf85d45044c85d8412dac53f2bb6f918e8b7777efba1787844/cryptography-50.0.2-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:c5e67125c7dca78d199ec4e116aa93dbb83494808ecbb8211a2cb09b1bf41dbd", size = 4756166, upload-time = "2026-09-30T15:29:24.58Z" }, + { url = "https://files.pythonhosted.org/packages/bf/f1/1c1f6874e8550cfddd4b688ceb38cefb6ed15ceed224d56f133f3d88c214/cryptography-50.0.2-cp39-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ee247f5c245c9a2fe7c8e2214e295918838e44e00a45a6718451e4004219e767", size = 4749145, upload-time = "2026-09-30T15:29:26.807Z" }, + { url = "https://files.pythonhosted.org/packages/c1/63/61b15dc1a8de03fe0adbe3fd7608b3ad5c73bf50993bbcb1faaa930afe33/cryptography-50.0.2-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:dfe9763530994147d9af1def057a5b9658b00e8f8fe8743d144d1e0911c2e454", size = 4763638, upload-time = "2026-09-30T15:29:28.588Z" }, + { url = "https://files.pythonhosted.org/packages/fc/35/b345bdfa40c9126df1a9d33236aa98418367931b8725f84fc3ae2b98dc59/cryptography-50.0.2-cp39-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:58ddb5a8e3179d12f19e4ea34d2d32e9d63a4baa142c875c1eb59f41b7243acd", size = 5382217, upload-time = "2026-09-30T15:29:30.589Z" }, + { url = "https://files.pythonhosted.org/packages/4f/87/ef344a9e616871f2519c22d6afcda79ddd5d35e9592d95eb6e677608d055/cryptography-50.0.2-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:f21e8a22c8605750c7af886bab299a363721264061b4ac0a30efb73cfd58efc5", size = 4781387, upload-time = "2026-09-30T15:29:32.605Z" }, + { url = "https://files.pythonhosted.org/packages/90/5b/f2fdb13cd0b96f6f932c8627bb292a45f11c64d21620a8e120aee9a3b848/cryptography-50.0.2-cp39-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:9c8402a82ea0dc4ceeab793db05f0fafa8ca139ca34fcde5df0f596103c74107", size = 4403790, upload-time = "2026-09-30T15:29:34.374Z" }, + { url = "https://files.pythonhosted.org/packages/bc/ce/7e4f662b1e3c393513569e402cfc85ac7da0bd3d5435e122a3140219eb2d/cryptography-50.0.2-cp39-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:0ddc924c04591c2811ca024d62ecad4f7f6f08af8939c211438f48a16bd23602", size = 4764319, upload-time = "2026-09-30T15:29:36.149Z" }, + { url = "https://files.pythonhosted.org/packages/3c/3f/86ff33ce34cc0de6847fb96e035a1a760d81652e38643f617c02ad32ef7a/cryptography-50.0.2-cp39-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:a6557e5f38e065ca9fbdaf7cfc7435ecb1d113aa81a022d1b51921ee7432e227", size = 5338560, upload-time = "2026-09-30T15:29:39.053Z" }, + { url = "https://files.pythonhosted.org/packages/40/cf/6b5c8e2fd9202d98988ab7cb5cc5c991704c4ad55f492ff408e4969f83f1/cryptography-50.0.2-cp39-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:1981f1db4630889b9ef7803fadef12b056f428cb6b85c27ba57b774793b6093c", size = 4780973, upload-time = "2026-09-30T15:29:41.251Z" }, + { url = "https://files.pythonhosted.org/packages/10/bf/8d6ebc7dded797bd0f0160d52188021211f011a2b164ef0ae1dac4587465/cryptography-50.0.2-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:7a8701d6b584d76e909e3d305b7d126b41439876a5aaf76cddc67fc230eafa2e", size = 4897738, upload-time = "2026-09-30T15:29:43.106Z" }, + { url = "https://files.pythonhosted.org/packages/d4/aa/f3f6e0de7e6253b8baa8b2d8fb9d50924fa75cee3d4624bd4bc1208ee923/cryptography-50.0.2-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:ce47f66801c20ec6c6632453bb5960fe38939e9306970b48b3a5a26de7745d94", size = 5058280, upload-time = "2026-09-30T15:29:44.827Z" }, + { url = "https://files.pythonhosted.org/packages/f6/b6/a1faf3a27ae9405fb34b1713cc73b2d8a26b04d5c561578fa2e6ef3e5bb9/cryptography-50.0.2-cp39-abi3-win_amd64.whl", hash = "sha256:4e81d95e5bafc2d6e34e4bed780e53e4d5b9a2f928573428aa4d35fbec1eb0de", size = 3854095, upload-time = "2026-09-30T15:29:46.782Z" }, +] + +[[package]] +name = "exscript" +source = { editable = "." } +dependencies = [ + { name = "lxml" }, + { name = "ncclient" }, + { name = "paramiko" }, + { name = "pycryptodomex" }, +] + +[package.optional-dependencies] +otel = [ + { name = "opentelemetry-api" }, + { name = "opentelemetry-instrumentation" }, + { name = "opentelemetry-sdk" }, +] + +[package.dev-dependencies] +dev = [ + { name = "opentelemetry-instrumentation" }, + { name = "opentelemetry-sdk" }, + { name = "pytest" }, + { name = "pytest-cov" }, +] +test = [ + { name = "opentelemetry-instrumentation" }, + { name = "opentelemetry-sdk" }, + { name = "pytest" }, + { name = "pytest-cov" }, +] + +[package.metadata] +requires-dist = [ + { name = "lxml" }, + { name = "ncclient", specifier = ">=0.7,<1" }, + { name = "opentelemetry-api", marker = "extra == 'otel'", specifier = ">=1.12" }, + { name = "opentelemetry-instrumentation", marker = "extra == 'otel'", specifier = ">=0.42b0" }, + { name = "opentelemetry-sdk", marker = "extra == 'otel'", specifier = ">=1.12" }, + { name = "paramiko", specifier = ">=3.4,<6" }, + { name = "pycryptodomex" }, +] +provides-extras = ["otel"] + +[package.metadata.requires-dev] +dev = [ + { name = "opentelemetry-instrumentation", specifier = ">=0.42b0" }, + { name = "opentelemetry-sdk", specifier = ">=1.12" }, + { name = "pytest", specifier = ">=9.2" }, + { name = "pytest-cov", specifier = ">=7.1" }, +] +test = [ + { name = "opentelemetry-instrumentation", specifier = ">=0.42b0" }, + { name = "opentelemetry-sdk", specifier = ">=1.12" }, + { name = "pytest", specifier = ">=9.2" }, + { name = "pytest-cov", specifier = ">=7.1" }, +] + +[[package]] +name = "iniconfig" +version = "2.3.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/01/e1/2069291243c926a2ff1cd706c7f3eeb9b62144bf60f77c9fb9ff2fb26bd3/iniconfig-2.3.1.tar.gz", hash = "sha256:67f4b9c50da0dedf52af349e7749a80a9057a5031199791b906c3bb3ae878960", size = 21209, upload-time = "2026-10-06T22:48:38.076Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/56/43/4ca9e49d27a1fcf6bece6f6aec0ea46bb9112489b93d4b688fb415457bdb/iniconfig-2.3.1-py3-none-any.whl", hash = "sha256:9121e2c1fdb355232495be3194c8dfe87ccc2d5dee45947b78e68f499790d7a7", size = 7552, upload-time = "2026-10-06T22:48:36.959Z" }, +] + +[[package]] +name = "invoke" +version = "3.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/33/f6/227c48c5fe47fa178ccf1fda8f047d16c97ba926567b661e9ce2045c600c/invoke-3.0.3.tar.gz", hash = "sha256:437b6a622223824380bfb4e64f612711a6b648c795f565efc8625af66fb57f0c", size = 343419, upload-time = "2026-04-07T15:17:48.307Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5a/de/bbc12563bbf979618d17625a4e753ff7a078523e28d870d3626daa97261a/invoke-3.0.3-py3-none-any.whl", hash = "sha256:f11327165e5cbb89b2ad1d88d3292b5113332c43b8553b494da435d6ec6f5053", size = 160958, upload-time = "2026-04-07T15:17:46.875Z" }, +] + +[[package]] +name = "lxml" +version = "6.1.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/23/ad/28ecd7cb894d172f3c9c80a075eeeb2017ac62e3632cee05a5f9493547eb/lxml-6.1.3.tar.gz", hash = "sha256:45222d94ddd511536f3b2f7d9deae3b2339b4ce0f075f1ca25703b07cad9dd21", size = 4211198, upload-time = "2026-09-02T14:48:02.287Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dd/1f/a180b57d9eeabaab77f9d5aa30356898ea749c4795596a8f66d1eb6bef2e/lxml-6.1.3-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:0c0710ac085a157b593c38fbcacd950f15c4afa8e2057527185875ab302752bc", size = 8602094, upload-time = "2026-09-02T14:47:26.054Z" }, + { url = "https://files.pythonhosted.org/packages/a8/25/070c92013a1c029a602b03560d68772313d918268667fa993da7961759c9/lxml-6.1.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:623c8799c17128753c65699f1c3aa32402657393a9ad6db09ed8b98ddf76611d", size = 4638308, upload-time = "2026-09-02T14:47:29.587Z" }, + { url = "https://files.pythonhosted.org/packages/1e/1c/722e88883173097a1a375153e3c2447eba3060d0231522cf6596e99f4195/lxml-6.1.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f683dc6300317700025e41d89a43e0276692ded16113a3c43eab704d605c58e5", size = 4939696, upload-time = "2026-09-02T14:47:32.997Z" }, + { url = "https://files.pythonhosted.org/packages/db/36/aa413bc214dc4f785ad2b2ddd8cc99aae7062d49ab155e91e6011af00daf/lxml-6.1.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:379f8a75cf6eb7eef0af074b55f49ab73b868388a98de14646abcdfa4564bb11", size = 5105247, upload-time = "2026-09-02T14:47:36.734Z" }, + { url = "https://files.pythonhosted.org/packages/a3/a0/a1f7f1313795bfec67b77f01ef3b1128d49f2d7f66a8413fa55d47f4e25f/lxml-6.1.3-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b37772102d44bb6628186accca3a121b1fa3a6b3d97518a8c29a5229ca4c0d0a", size = 5011915, upload-time = "2026-09-02T14:47:39.846Z" }, + { url = "https://files.pythonhosted.org/packages/b9/78/840e7e3f1d0cc7a5cfac5d8505b97e25b6427fd774ac4bae672aaebfb4b5/lxml-6.1.3-cp312-cp312-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ddcf547bea2aee967d6a77779376a45e77e610e8465147a1f3d7e20d539d6e32", size = 5638175, upload-time = "2026-09-02T14:47:43.644Z" }, + { url = "https://files.pythonhosted.org/packages/0a/20/e022dbc6b4753a9bc9fc5fb28a27163430c1731b9913997f6544c1b2518c/lxml-6.1.3-cp312-cp312-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:909f4e927bb051f7740d6367285fc60cdcfdaf0258c2dba4ff5ba7eadadc250c", size = 5244675, upload-time = "2026-09-02T14:47:47.635Z" }, + { url = "https://files.pythonhosted.org/packages/99/83/82cde81d2b5eb38d1539fdfdf318abdd014a7e604f4df01c9cd3deb18f2a/lxml-6.1.3-cp312-cp312-manylinux_2_28_i686.whl", hash = "sha256:a5c18810318303ce9afb3f95e2ddb54834f96fa699a8600433fd5a93dcf44c56", size = 5358205, upload-time = "2026-09-02T14:47:50.306Z" }, + { url = "https://files.pythonhosted.org/packages/d2/a1/f3b057371c8cb29f2a9c9c44ea320592446e40b74a4b0af68c3d8e65bc73/lxml-6.1.3-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:3e42265103fb385d8642a78672edf376c6f7e1d3598a7a4f9cb1278f2f6b5f6f", size = 4704495, upload-time = "2026-09-02T14:47:53.251Z" }, + { url = "https://files.pythonhosted.org/packages/1a/a4/230eb28be5d412152ffc3c679b51fe1aeede5a53f3a8eb6e9748f2f4754f/lxml-6.1.3-cp312-cp312-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:21402998e4b78e7cce237d2788841aaa21ac9a4d1574d04dc2d12ee41ae807b5", size = 5255117, upload-time = "2026-09-02T14:47:55.963Z" }, + { url = "https://files.pythonhosted.org/packages/a3/18/1969f56763af24ce42ea156007b0b2d73fddea552e283b2010416394f0f4/lxml-6.1.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:38fc4e4e4e084e0bd491949482527d406788045c546d4f8789e93fc527b91385", size = 5054424, upload-time = "2026-09-02T14:47:58.131Z" }, + { url = "https://files.pythonhosted.org/packages/f4/d4/2a90acc1f6fabaa3a8db9340437822bd8d041b205d626a4b3e8621aaa390/lxml-6.1.3-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:5609efdb0d3c95499c00046bc53648b3482ec2175b5503d6e611b3f0555dc71d", size = 4785572, upload-time = "2026-09-02T14:48:01.029Z" }, + { url = "https://files.pythonhosted.org/packages/a5/1e/b90e845b1dcd0f2f3f26b98283d857f25909223aacd265eee032c34ab8b1/lxml-6.1.3-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:97ce49699d87ebf8aad631b55d65b33219a4f1bfefbbf5bff19dc9af160aeaf9", size = 5656516, upload-time = "2026-09-02T14:48:03.419Z" }, + { url = "https://files.pythonhosted.org/packages/eb/ab/0a1b802c57f3fba5c4efd77d5c6b78adaa8f7b681f0c90456b140fe8bf6c/lxml-6.1.3-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:48542c9acba9ff9450bd18d871d2c2c8787fdb283572b623d206f1b927cd7d9e", size = 5245982, upload-time = "2026-09-02T14:48:06.109Z" }, + { url = "https://files.pythonhosted.org/packages/da/ee/2c016fbceb3778137459292538d9dfa7e3ad9070fe409c15254ddd90d2cc/lxml-6.1.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c55e71a9b1db1f107efb60da49c093689b74c5c31a708e5379e2fd9439d4fbb5", size = 5267340, upload-time = "2026-09-02T14:48:08.374Z" }, + { url = "https://files.pythonhosted.org/packages/9c/b1/736d18fd6f0835761923b7bac1f0c27d60c1200384e9093f05d8c5100525/lxml-6.1.3-cp312-cp312-win32.whl", hash = "sha256:b3ff39654f0ce6ebd4db154211136dbe7e8157bcc3bed2344c87f32c7c6ecb6c", size = 3602606, upload-time = "2026-09-02T14:48:10.384Z" }, + { url = "https://files.pythonhosted.org/packages/3a/5b/6ed903e4e6278a020c8a6f0dbbe78030d041840a6b4a64ea441a1e414077/lxml-6.1.3-cp312-cp312-win_amd64.whl", hash = "sha256:3e9a00d1c2c30936f7add097c41afc5da6556c580909104aafd382cac92a855c", size = 4005999, upload-time = "2026-09-02T14:48:12.51Z" }, + { url = "https://files.pythonhosted.org/packages/e4/1b/7bcebb7b6332cb3ae85e9c13b139adb6f23f75c71d84041c56a5005d9a29/lxml-6.1.3-cp312-cp312-win_arm64.whl", hash = "sha256:1aeca87830c4fe649dcf93fe2b059525b71c72587f21be4ae4af7103082a79fa", size = 3666631, upload-time = "2026-09-02T14:48:14.567Z" }, + { url = "https://files.pythonhosted.org/packages/52/05/3ef45db776baea068044c799bbba68f3ca00a440c0e930a17c572f3d9639/lxml-6.1.3-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:3a48093cdb058a93af842ede9703520e810b05dcd0fc6d7190a06376c3bfb6bd", size = 8590357, upload-time = "2026-09-02T14:48:17.413Z" }, + { url = "https://files.pythonhosted.org/packages/8c/a5/eee2fc77eee5ea68e4a4334b1def1781a3beaeefd3d98e81b4a38dc447b7/lxml-6.1.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:887c021d9a977cff89cb273047c1352997b772a8908a25c21836861f69b92be1", size = 4632616, upload-time = "2026-09-02T14:48:20.745Z" }, + { url = "https://files.pythonhosted.org/packages/35/42/df27b56848acd29d8a720acc28977911aab36f2a09df4208d5502e887415/lxml-6.1.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:611a51e61c92f62345a50b0035df6fc0d678f9299f33728826d831598862f59d", size = 4936186, upload-time = "2026-09-02T14:48:22.94Z" }, + { url = "https://files.pythonhosted.org/packages/ab/8d/8a7b91df0b54d09d25f5f44885d6b3e0a6d6643a8c070191580318d20c42/lxml-6.1.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:b477912f42c5c33405a10c759d22f80cf5af043ae02d95b9d8e5e5bc555739ed", size = 5093324, upload-time = "2026-09-02T14:48:25.132Z" }, + { url = "https://files.pythonhosted.org/packages/c6/7e/8f340ddcd43790332fb0de8a26628d571a492da3300cd191821698407c96/lxml-6.1.3-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5cffe18571ccc51d742cd08cbb3f8b756de9311d18c7ea98f5d92f37b8fb60c2", size = 4998850, upload-time = "2026-09-02T14:48:27.394Z" }, + { url = "https://files.pythonhosted.org/packages/c5/c1/9c5bb572f1f09ec9e4322bd4a4e9f4ad48347fc56ef94cf4df58a5279dc8/lxml-6.1.3-cp313-cp313-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:75cc6569e86be5785b6188ef1642670c6adbc984e81ec35e224842ecd9eefcc8", size = 5626813, upload-time = "2026-09-02T14:48:29.61Z" }, + { url = "https://files.pythonhosted.org/packages/ac/7d/8bf1fd8bae8247743968bb76d027a1ac5bd2c4b44495fba6a71b30d10706/lxml-6.1.3-cp313-cp313-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d85dfab42dd672f87a7f76e9de7172962aee69fa12044f0d6e1a23cbd53fb80e", size = 5232385, upload-time = "2026-09-02T14:48:31.969Z" }, + { url = "https://files.pythonhosted.org/packages/7b/2e/6cef69ed81cb7df0d03b0dd09d08e6e2cf5061a743ff6f42f0b741548e9b/lxml-6.1.3-cp313-cp313-manylinux_2_28_i686.whl", hash = "sha256:42632b4024ab24a6b488f559ac851312509888b6b80ae2aa11cf29a646a0d245", size = 5347088, upload-time = "2026-09-02T14:48:34.13Z" }, + { url = "https://files.pythonhosted.org/packages/5f/e1/8e5fd8ddc8c7d685badb0f2db149e3c9da84eefc2827c01c658df2c4e3cb/lxml-6.1.3-cp313-cp313-manylinux_2_31_armv7l.whl", hash = "sha256:febd35ef45f603c2d74b74655efdbf45e14f55fc0aef4ac82b663ca829b283e0", size = 4707227, upload-time = "2026-09-02T14:48:36.62Z" }, + { url = "https://files.pythonhosted.org/packages/7a/7e/00041382a11be40a88bf405ebff11c8efabd3de79f2691e1638b1c47a8a0/lxml-6.1.3-cp313-cp313-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a43b3bdf11e477dc7770609d3477316f974354dfc8425d596f64f471cc8daf6e", size = 5240208, upload-time = "2026-09-02T14:48:38.893Z" }, + { url = "https://files.pythonhosted.org/packages/fd/fe/316538b5cff0936fa63d45d421c655730fcbb5a28dcac728c175083002bc/lxml-6.1.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:5d582042c69857c364e8153de6e18e0da9b7b515a6a8113caf69a6ec8e0520f2", size = 5050271, upload-time = "2026-09-02T14:48:41.213Z" }, + { url = "https://files.pythonhosted.org/packages/c9/91/455bcccb3ac725373007344d351151810cd19762d1673b64b811f4359a42/lxml-6.1.3-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:8e49a646acfab83c68974f4aa1d0a2acca9e88d7d627ae0fc13201b14b76d310", size = 4780433, upload-time = "2026-09-02T14:48:43.779Z" }, + { url = "https://files.pythonhosted.org/packages/cb/f6/580440e2f52cf00bba5c5e1080bfa88cdfcde73be71a11d95170ddbb663f/lxml-6.1.3-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:0dee106e9aa97fb00541b1ed7827070564d0549c3d3fba8920e6b20fd980f748", size = 5645928, upload-time = "2026-09-02T14:48:46.187Z" }, + { url = "https://files.pythonhosted.org/packages/f6/dc/d123c1f244306543d545f62443f794959e4f1ea709fe100f8740d514e74a/lxml-6.1.3-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:dd5e90f34cffcfed97f36cf066325773d2b6021c60c29942e53a18b028501b1d", size = 5231184, upload-time = "2026-09-02T14:48:48.691Z" }, + { url = "https://files.pythonhosted.org/packages/c3/3c/fe55b2bd5c6113c906511cd88f6a470195c5fbff1124f19970ab706c3477/lxml-6.1.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:d9b3e7d71bf6acff341233417abbdface29c647e3113892d9aaedc02eb4aa2bc", size = 5255814, upload-time = "2026-09-02T14:48:50.948Z" }, + { url = "https://files.pythonhosted.org/packages/e7/a7/485df55acf55dc35e4ca89d2f48f03889e5a3241826b18b85102b32ce9d8/lxml-6.1.3-cp313-cp313-win32.whl", hash = "sha256:160fcf381f76c3aeac28a756bec44f48942a8f7245a87aa28e3a523b4d90cd87", size = 3602214, upload-time = "2026-09-02T14:48:53.236Z" }, + { url = "https://files.pythonhosted.org/packages/c0/28/e46a7702bd95e9043291f7c3539b6184cba66f96cea9936f20939b284eeb/lxml-6.1.3-cp313-cp313-win_amd64.whl", hash = "sha256:e477aca0bc0d19f3b4ae9e4f2a1cfd687c31bf772d78734910658186b40b2477", size = 4004091, upload-time = "2026-09-02T14:48:55.699Z" }, + { url = "https://files.pythonhosted.org/packages/8a/1d/154c78e20479a43916e63f19cb720d83f44f024b03228be44c92d9a97b24/lxml-6.1.3-cp313-cp313-win_arm64.whl", hash = "sha256:b1cc980905221a5d8b3c476330730b3adb40ff80add71ffbdb6215ba055656f1", size = 3665468, upload-time = "2026-09-02T14:48:57.703Z" }, + { url = "https://files.pythonhosted.org/packages/0c/15/fc75a70b0af6021d0ea16811f1fc71cc42cd06ce90fe10f007a69b2eed84/lxml-6.1.3-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:2bec13085dc8ef48a3fe62f7dfcacfeda2c785cdf19cc8eeda2bb9ed081da165", size = 8609725, upload-time = "2026-09-02T14:49:00.156Z" }, + { url = "https://files.pythonhosted.org/packages/84/ef/398fcf9018f881ec9aeaafae1ddd6586dfb13314a35d35e899de373dcae0/lxml-6.1.3-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:4f4db7c7e954d289d71878938348b3d91b904a3e8210a11939359fb758a58e7d", size = 4639629, upload-time = "2026-09-02T14:49:02.81Z" }, + { url = "https://files.pythonhosted.org/packages/a7/2d/49b6a6ad7ce8f64b07b9fe852ff0c6d3fcbb26db61bee4f63d4120180a1c/lxml-6.1.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:2cae5d5c90a62d9139c512a0cb1aad1d182b022b5740daea2617eb5bf7fc658e", size = 4965074, upload-time = "2026-09-02T14:49:05.133Z" }, + { url = "https://files.pythonhosted.org/packages/66/bc/6230cf80e4331c33383b0b6b73dc31a393dd76edd4cb73d761de5123034d/lxml-6.1.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c6c0c13128a32eb04a51357e56a094e13aa8e6d3d1884de2e9ae923f6915e1a8", size = 5099355, upload-time = "2026-09-02T14:49:07.343Z" }, + { url = "https://files.pythonhosted.org/packages/ac/cf/d1143d9b7717e07a82f158a1fc9ce6e581fdad1226734950af869e3ffde4/lxml-6.1.3-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2221e88679d1351e9a40aaee54bc65679b9795bbd0160bc3d5e36b163344eb75", size = 5036795, upload-time = "2026-09-02T14:49:09.65Z" }, + { url = "https://files.pythonhosted.org/packages/31/6f/194bb00ffb89712c30f5a7e1b8e685590e140fad6c8261fec172c09a3dc0/lxml-6.1.3-cp314-cp314-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cfb398886a7eb4c719161c3efcff2a1248febc53a4d8e5072d2d8a87fed84ac9", size = 5658740, upload-time = "2026-09-02T14:49:11.9Z" }, + { url = "https://files.pythonhosted.org/packages/e9/44/27e3cee3dcdb3b7bc09727b642bdbfcd098490ea77df04611db9060d7722/lxml-6.1.3-cp314-cp314-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a7eb78ba28b187e1e9203a55c60fcf70df2d22cb205fe6d51b9383d6097419f0", size = 5245991, upload-time = "2026-09-02T14:49:14.154Z" }, + { url = "https://files.pythonhosted.org/packages/ca/e9/8312560579fc980bbd2233a8a673cc46f7d613d3633f2bf08a21e8f4ad13/lxml-6.1.3-cp314-cp314-manylinux_2_28_i686.whl", hash = "sha256:ea6b1e9105b4b24a34c722432d9fb578f9ed83af21fa1abda639011e0f22bbb6", size = 5354136, upload-time = "2026-09-02T14:49:16.459Z" }, + { url = "https://files.pythonhosted.org/packages/74/d8/eda60f4f73a9c780b5d6e1175484f66e6c81a2c93346e2906a1fec9c7a02/lxml-6.1.3-cp314-cp314-manylinux_2_31_armv7l.whl", hash = "sha256:e8b17e23df3e827a69d25af70990ca2420e92668aaffaeeb3cd2351d7916a023", size = 4704379, upload-time = "2026-09-02T14:49:19.032Z" }, + { url = "https://files.pythonhosted.org/packages/ba/c8/c9cc60057be78ac34bd2b842e45e6e88edbfe5e532e82c3b82381b7aab49/lxml-6.1.3-cp314-cp314-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:1b7c37339d7e75cab9a123a04248e243cefefb302ad6db566ea0c77cbcde421e", size = 5258676, upload-time = "2026-09-02T14:49:21.306Z" }, + { url = "https://files.pythonhosted.org/packages/41/7b/66894008fee8d1785b8db129747ae963fd427b68f456918df7f2f24a8b98/lxml-6.1.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:83e3a51e7933db700a0da0db31849db3a24022d9970da9bb73001e1d0326fd92", size = 5090069, upload-time = "2026-09-02T14:49:23.562Z" }, + { url = "https://files.pythonhosted.org/packages/8b/31/c1b60404859f4c3cd1f41f29c65a24e25cea78fde822d9574a21f66810be/lxml-6.1.3-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:9bde9ae026a55b9a192078dfa6e27dd0ca4a050171ab6272e92f97b757dfdf48", size = 4741958, upload-time = "2026-09-02T14:49:26.037Z" }, + { url = "https://files.pythonhosted.org/packages/23/b8/6285f0cf546f14da2554cabdeaf7c2c2ff3190c74807f0de2e8810a786f9/lxml-6.1.3-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:1a635e837b50a1819bebfedaac5916498ea024120969da8790500148fb0a894d", size = 5683245, upload-time = "2026-09-02T14:49:28.438Z" }, + { url = "https://files.pythonhosted.org/packages/d3/f6/2168cab44336dcb15fed0f0b78577225b83297cdf0dee349c95420c3dcb0/lxml-6.1.3-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:d0c5c362bc94f1929dc7e96e715bbe7bd17037f802e6d8f0d1545df9133c0559", size = 5246087, upload-time = "2026-09-02T14:49:30.955Z" }, + { url = "https://files.pythonhosted.org/packages/f5/89/32f5de69a0a31f30e6164981851f87b37ecb2c4ee838e504b88d49d4818e/lxml-6.1.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:c59e4265608da6a041f54646ecc0c9ecdbb19aaf14c4c684bb6c2114998cc415", size = 5269352, upload-time = "2026-09-02T14:49:33.502Z" }, + { url = "https://files.pythonhosted.org/packages/a2/a1/741d952ed3a7ef7a50055c6415aec3f067015e97f72f4389ce77b09657ba/lxml-6.1.3-cp314-cp314-win32.whl", hash = "sha256:2e62c569ec7531b679b184cbfe335c501c1d13c4b363560013019962eb630e6d", size = 3662783, upload-time = "2026-09-02T14:50:23.751Z" }, + { url = "https://files.pythonhosted.org/packages/0f/bc/5811cc73cac05e324e05ba9b0924e1a163a317a167ede8a9c748b11db30a/lxml-6.1.3-cp314-cp314-win_amd64.whl", hash = "sha256:66299564c046bc7e0cc5de5106601eae907e9fa5904cd68a323380a8502f7861", size = 4073951, upload-time = "2026-09-02T14:50:26.348Z" }, + { url = "https://files.pythonhosted.org/packages/92/18/3768c8b01ac3a9bed1914715e6011711b00e2a11628ffa6f7fa37f8e0269/lxml-6.1.3-cp314-cp314-win_arm64.whl", hash = "sha256:ebd054ad1737a68fb7c5c073d405cef2b88bb824e294de3b4a4e995b47f0e376", size = 3749279, upload-time = "2026-09-02T14:50:28.749Z" }, + { url = "https://files.pythonhosted.org/packages/72/38/84684784738d9451db2b330de2483f496690c3a5c642071df24135739b37/lxml-6.1.3-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:5a143e6207579de8baeded4eaac9134413200359f1969d636f0bfb98ee8c3c8f", size = 8860296, upload-time = "2026-09-02T14:49:36.346Z" }, + { url = "https://files.pythonhosted.org/packages/24/b7/fc4c50bb1b38e864010ea396046cabe85129bf9e65b11edcfbc37d356241/lxml-6.1.3-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:a1cec0f99b9b914d39176347a93b7610dc09324491aee1cbc57cd291a41a1d55", size = 4755190, upload-time = "2026-09-02T14:49:39.872Z" }, + { url = "https://files.pythonhosted.org/packages/94/e2/ee9aa6ed2b666b2db1f6f7fd48964ff9da39ebe827ef5eac0ab881f639d9/lxml-6.1.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f6b9d2aad499c769ee8287609ab0e6de99d8bcea99c6e6c2e64945259fd52fb2", size = 4979517, upload-time = "2026-09-02T14:49:42.153Z" }, + { url = "https://files.pythonhosted.org/packages/29/e3/e7763d1661b283ddd4fa36f91b9a497db6b8d2aff55028b16c7f642e0755/lxml-6.1.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:28a23fefdb345b2d4d0ff2860571b5ff9a89a28b6a120f720e8fb0324d346626", size = 5115270, upload-time = "2026-09-02T14:49:44.493Z" }, + { url = "https://files.pythonhosted.org/packages/2d/cd/22205d5b4d177e3f4156f780412426ee7c7f8107809f119f0dcc40fa51e3/lxml-6.1.3-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:545ccc14fb05485f48b4439ec35beb16d5b5280eb6c81c658bd4707a2a119414", size = 5032449, upload-time = "2026-09-02T14:49:46.841Z" }, + { url = "https://files.pythonhosted.org/packages/da/43/06a4626c3bb79ef8c501b674afab8100d64e798665bb2a97d1c960636a49/lxml-6.1.3-cp314-cp314t-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:93476b6514b373fc6ca67d26c442784f7807c86f00635bfe79f935c3eab2af17", size = 5603325, upload-time = "2026-09-02T14:49:49.664Z" }, + { url = "https://files.pythonhosted.org/packages/d0/9c/733682a0c2de9f5779ba207bbb3f3f6be8c6bda863fc01739b186b38783a/lxml-6.1.3-cp314-cp314t-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8db38ff3fb7aee7d6a82ae4da2eef1178656fe1216841fbd24870062a9d60473", size = 5229023, upload-time = "2026-09-02T14:49:52.447Z" }, + { url = "https://files.pythonhosted.org/packages/c6/8a/e69cdaca3fd33a647942925664f01b20908d41a6968c182305be9c38fb11/lxml-6.1.3-cp314-cp314t-manylinux_2_28_i686.whl", hash = "sha256:25f4118c438f96bb466e83108506d03d5c31b1bd2387e83e5b070bda6ded9c37", size = 5317811, upload-time = "2026-09-02T14:49:55.25Z" }, + { url = "https://files.pythonhosted.org/packages/2e/b2/0c397588174403c2ab68fc464abf97e03e7324f9c6cb6a99023104707195/lxml-6.1.3-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:1beb0f9909b26cee938df9ba56b15252a84429b1fc30ce6fca161390b9789a70", size = 4646516, upload-time = "2026-09-02T14:49:57.761Z" }, + { url = "https://files.pythonhosted.org/packages/56/7e/cfea25afafbe49db8b225764f7f74bb37c2a7f5e717d917d3d4a5e098ed4/lxml-6.1.3-cp314-cp314t-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3a27ac6c780c8b8a1cd231b58407634cafc1c4cc28cd6c7141362df0f36351e7", size = 5240626, upload-time = "2026-09-02T14:50:00.279Z" }, + { url = "https://files.pythonhosted.org/packages/a1/75/7a587771bb52ebb0e2c57b6dbe9fd96a70fbb54d72ddd97d54c5f8ec18d5/lxml-6.1.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:a1932d7ce78a561367512c594fe66eac2b2ec9b9264cfd9b5f950622f4a116e2", size = 5086619, upload-time = "2026-09-02T14:50:03.245Z" }, + { url = "https://files.pythonhosted.org/packages/1e/01/94c0ebe6d831861542d251e038052e52bf6d33f1d18f1cfffdc82851065a/lxml-6.1.3-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:7d0f5976aa2701996f759b30172925829867547bb073af0ae67d1307a0f0262c", size = 4758828, upload-time = "2026-09-02T14:50:05.873Z" }, + { url = "https://files.pythonhosted.org/packages/1f/f1/938d67bd0e5b1fdfa52be28aefdffbad57e1f6b8e921c2aab88542c75f40/lxml-6.1.3-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:c5e7ce578aa8a80910a72a8ca0bbea3baae10100827249001999726a788456d8", size = 5627083, upload-time = "2026-09-02T14:50:08.555Z" }, + { url = "https://files.pythonhosted.org/packages/d8/65/4e51522f6c214650db0abb7b16ccd11b1238b8a05a8d59aa4ebed59c9f67/lxml-6.1.3-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:d97c5227621af74b111882a290b10f371780a38eef9d9e730408fba2259b52fb", size = 5235170, upload-time = "2026-09-02T14:50:11.255Z" }, + { url = "https://files.pythonhosted.org/packages/92/c2/e73d19365665f6b16ef84df21199befc3b06e4c539046ad2d9595f6fb9ea/lxml-6.1.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:da707f14ea3c35ee463d50acd596d6488e4b2b4ae7cf77a5bf93f55c023d63e8", size = 5252273, upload-time = "2026-09-02T14:50:13.782Z" }, + { url = "https://files.pythonhosted.org/packages/48/a9/7f386c84c9fe2854e1ca6e231c285e1c8f392971ac353c6865e6ec49faff/lxml-6.1.3-cp314-cp314t-win32.whl", hash = "sha256:9efe56a68179f3adc4de41861c9358931db03837c48dd5e1c78077b84dd07f3a", size = 3902712, upload-time = "2026-09-02T14:50:16.171Z" }, + { url = "https://files.pythonhosted.org/packages/82/a6/8a3eb793f7900ef01c7f99e6f5fcbcfbdff35251cfaef66b32a4c16352d6/lxml-6.1.3-cp314-cp314t-win_amd64.whl", hash = "sha256:c9389b3784b56c58d933b5e0aecdf28f901b073ff385358d8a7d40907f6e14b2", size = 4400979, upload-time = "2026-09-02T14:50:18.621Z" }, + { url = "https://files.pythonhosted.org/packages/cc/c4/3807bea283b4fe9e9d9f5dde46a73df91178472b335d2778e10b2a37aa22/lxml-6.1.3-cp314-cp314t-win_arm64.whl", hash = "sha256:32a409be3190b088f960ac92bfedfbef2f86c49ff940765e1548177592d20026", size = 3823401, upload-time = "2026-09-02T14:50:21.119Z" }, + { url = "https://files.pythonhosted.org/packages/e1/8e/4614fcd65496054cfb7172662f3576a59200278739506433b8c241ea422a/lxml-6.1.3-cp315-cp315-macosx_10_15_universal2.whl", hash = "sha256:6ea2f13dce778ca072ccee598bca46a092ce192e8fd907b6c1f0e52c800529a0", size = 8609378, upload-time = "2026-09-02T14:50:31.772Z" }, + { url = "https://files.pythonhosted.org/packages/f2/51/2cdce3c65fa99a6195dd8fbd512d33407c1000ad99f63e0a285b63d7a8eb/lxml-6.1.3-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:c581b1d68b3845fb86c6b2983e755b29bf001461c59fa411d2c26a911b6559a9", size = 4640022, upload-time = "2026-09-02T14:50:34.41Z" }, + { url = "https://files.pythonhosted.org/packages/52/09/0b30084e9eb1c546a4be3d9c56df70058d116b1a320400a59b0f7da87bf0/lxml-6.1.3-cp315-cp315-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2e01125896585139453cab8cb235893644d8815d7509520da95ae3ee8d1c1f79", size = 5037928, upload-time = "2026-09-02T14:50:37.007Z" }, + { url = "https://files.pythonhosted.org/packages/b8/0e/5c37275a3e361f6138dc06db748ea565c1fe8a5f4ee5e2ddd80047c81a89/lxml-6.1.3-cp315-cp315-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:290f66b97ede0e552e1cb44a0fd8a74f9753ee635b50830a0b122fb72788d015", size = 5661932, upload-time = "2026-09-02T14:50:39.777Z" }, + { url = "https://files.pythonhosted.org/packages/70/c5/b71ffb289b15e2642e2a3cf6d468c44da39ea119061a99e5b05e3d10f217/lxml-6.1.3-cp315-cp315-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:73fc05988ed20809450474ba760a87c8ad4e455fc09783c02195e56ec634b41a", size = 5249209, upload-time = "2026-09-02T14:50:42.141Z" }, + { url = "https://files.pythonhosted.org/packages/81/ea/9910da149a23932f9301652e57661cd9e42b0df18f12be21159b7255f92b/lxml-6.1.3-cp315-cp315-manylinux_2_31_armv7l.whl", hash = "sha256:dc3a44689eea43eab836e5c98a8ab015dc2419987d1ea6eafc7c590cdff86bed", size = 4704543, upload-time = "2026-09-02T14:50:44.634Z" }, + { url = "https://files.pythonhosted.org/packages/76/07/9290329cd188c62e22021f79df04ee0cc33d9a93b0d38bd65ccd452ad9d0/lxml-6.1.3-cp315-cp315-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:209c3ccbfe35a04ac6d24f0611f9d1cbf8025d49991b14acd935236234d6c156", size = 5261298, upload-time = "2026-09-02T14:50:47.301Z" }, + { url = "https://files.pythonhosted.org/packages/c9/0c/aba78bd3401cd99b73a0aed8e2b9b43e14be94fab3603d4bbc8a62365f2a/lxml-6.1.3-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:2f5b2a2b9811b853b39bfa41367c6d78747b8e3e80e07fc5a24aae295c1a4d7d", size = 5090453, upload-time = "2026-09-02T14:50:49.952Z" }, + { url = "https://files.pythonhosted.org/packages/8d/dc/fa4426c3355aa0216cbeb3911495b5f65a26e0df85859a89928fe28f0396/lxml-6.1.3-cp315-cp315-musllinux_1_2_armv7l.whl", hash = "sha256:6a406d0b3cb207b0fa460ed4dc93e866f44f105da0169361cb18ff998a44c7f0", size = 4744709, upload-time = "2026-09-02T14:50:52.394Z" }, + { url = "https://files.pythonhosted.org/packages/be/2b/224fe7918658ab7c532ac2412f3c1eb28f71e6364fb07566262d0cc6a7b6/lxml-6.1.3-cp315-cp315-musllinux_1_2_ppc64le.whl", hash = "sha256:53258656846f5c48996b882fb4b135885e088a3ad3d96b4bc0530f95124d1f69", size = 5685802, upload-time = "2026-09-02T14:50:55.043Z" }, + { url = "https://files.pythonhosted.org/packages/21/44/7d480819b9adcae5f84dd8ac529132c6b7a578544398225cd20321adcd91/lxml-6.1.3-cp315-cp315-musllinux_1_2_riscv64.whl", hash = "sha256:aa633613ff907ea91b9b0489a1f0da1b8725d8c6ccec6b77e8a1c9c235044bb0", size = 5249019, upload-time = "2026-09-02T14:50:57.985Z" }, + { url = "https://files.pythonhosted.org/packages/72/83/385a267ea1b6b283f2249dd827ef360a295e9db14e13ef4665a120c60d64/lxml-6.1.3-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:90f709b9accab6b2e4d14f5c8718203877a0486bcb3afd74d8b539ecd1e961d4", size = 5271886, upload-time = "2026-09-02T14:51:01.667Z" }, + { url = "https://files.pythonhosted.org/packages/d8/0d/f967b0eb172ae876855a402d6d9b11fa86e3e0c89ca9bbfeadf7ffbfa719/lxml-6.1.3-cp315-cp315-win32.whl", hash = "sha256:b4fc6b03b9d9d90557274f571ab30e7fbbfc527955536935d96f98b6817a86e4", size = 3662894, upload-time = "2026-09-02T14:51:45.173Z" }, + { url = "https://files.pythonhosted.org/packages/f4/48/d8a8c4160a29e663109ad520bac2deb37fcd014756d024561e8bc3e611ec/lxml-6.1.3-cp315-cp315-win_amd64.whl", hash = "sha256:33cadd956b667997e4de1635fce9541f2e8ede2038fcde8cf55aa14d571d1bad", size = 4074626, upload-time = "2026-09-02T14:51:47.77Z" }, + { url = "https://files.pythonhosted.org/packages/25/20/3e1395d34d19f9254625d0b567b81cf70d37d3417be074f4d63b94a2be3c/lxml-6.1.3-cp315-cp315-win_arm64.whl", hash = "sha256:8a330c0ee5fa318c7b5cbbaad882baeca3f570357e7eb25ab34bf31008150758", size = 3749495, upload-time = "2026-09-02T14:51:50.663Z" }, + { url = "https://files.pythonhosted.org/packages/8f/c6/7465ffd9c43883526a382df6fa4846c9d8d419214f7effbf65270e795471/lxml-6.1.3-cp315-cp315t-macosx_10_15_universal2.whl", hash = "sha256:0bf5a3e397df2ec4258eb5eea4c1ac6cf013ca1abd04a176903bff20a70021fe", size = 8857677, upload-time = "2026-09-02T14:51:05.109Z" }, + { url = "https://files.pythonhosted.org/packages/ed/eb/1f3a917e299df43c8162c3e6f64fc2cea3bcf277910f35bff5b8e5d39901/lxml-6.1.3-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:13d22c0d57355366b393936acf6b98a5e0edeadddd3fccbc6a846c50a76b8741", size = 4754522, upload-time = "2026-09-02T14:51:08.137Z" }, + { url = "https://files.pythonhosted.org/packages/d7/f9/f81b4bdb6efb7a596be29603d8758154d00a5f545db9f3cef9d9041c8f64/lxml-6.1.3-cp315-cp315t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cad7617727a96d189bd6f979d0fadf765198c7934e85f4edaba9bf3ad919a300", size = 5033744, upload-time = "2026-09-02T14:51:10.633Z" }, + { url = "https://files.pythonhosted.org/packages/c8/0f/26d9bfaacb319c86e0eca8a1a0bf1130d36a7afbd318883e23caea63763d/lxml-6.1.3-cp315-cp315t-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cae82b5ca24b0c2beedb269f6e2a96f466acd926879ab00ae19f1a65cbf9ffb0", size = 5615269, upload-time = "2026-09-02T14:51:13.357Z" }, + { url = "https://files.pythonhosted.org/packages/5d/90/73675f3f4141350ed65d6fec533b107d4e802c5caa340cf111771edd86e0/lxml-6.1.3-cp315-cp315t-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:69cafd61aea04ebb3502c93c2aaa568b12931ca0802231e0b5de76bf8b6e74bd", size = 5236280, upload-time = "2026-09-02T14:51:16.051Z" }, + { url = "https://files.pythonhosted.org/packages/fd/be/ed260767e7977de463a0f91f3f4fffcab85c0a2a024a21ffe1fa442c2c79/lxml-6.1.3-cp315-cp315t-manylinux_2_31_armv7l.whl", hash = "sha256:dc205732d593118cf701d986f40e9de7801bb2e371cb189ddbda9b7348f4d97e", size = 4650718, upload-time = "2026-09-02T14:51:19.102Z" }, + { url = "https://files.pythonhosted.org/packages/d0/fd/e9839d03b1e767f2725cf7d7d81b80d5f3f9fdc10ad8827e2479311b046e/lxml-6.1.3-cp315-cp315t-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:88e719b9437f148f7e1465df845c758dd1598618cbea3a2fd1e61a715542f2b2", size = 5243376, upload-time = "2026-09-02T14:51:21.606Z" }, + { url = "https://files.pythonhosted.org/packages/34/a5/4606e347e2788c301f677004aa83e28d24da9fe663a24380122af57be6fc/lxml-6.1.3-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:40983eabefd13da003e68170928c7acc011f0d095eefce5871a3c71c9385fb9a", size = 5092340, upload-time = "2026-09-02T14:51:24.21Z" }, + { url = "https://files.pythonhosted.org/packages/ea/99/3314a8661cdf30f493c55a87db283961dfaae08451976a2ca418958e1804/lxml-6.1.3-cp315-cp315t-musllinux_1_2_armv7l.whl", hash = "sha256:fad67b12ffe0f71e02b4932b04883cbc76a9072bbd30731409d3523cf058b011", size = 4758768, upload-time = "2026-09-02T14:51:26.813Z" }, + { url = "https://files.pythonhosted.org/packages/30/58/3bdc577f78ea8b7d72d39a84506f7001d5b28728f43e5b84891e3b7d9a4a/lxml-6.1.3-cp315-cp315t-musllinux_1_2_ppc64le.whl", hash = "sha256:6cd11e7550d89e551a87dcec30f04b1fca32e86b68708aa01a4daa455d8605e5", size = 5649546, upload-time = "2026-09-02T14:51:29.453Z" }, + { url = "https://files.pythonhosted.org/packages/6a/e4/652633de1a2395949ebb7a8fc7d089aba12a2b45f0fefbc9d29e3e3ab3cf/lxml-6.1.3-cp315-cp315t-musllinux_1_2_riscv64.whl", hash = "sha256:ca0ec532ad2f5ba1e5ec120ac157769c57f01855b3d8bf37213f5d88abd9ba0a", size = 5234874, upload-time = "2026-09-02T14:51:32.262Z" }, + { url = "https://files.pythonhosted.org/packages/65/a6/c4581d171de30449304b4859bbd3607e9b40da13c0f88b68e6097c8d785e/lxml-6.1.3-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:e99e09ab7741f1281e2677f4c0058c7f5267d182530b09c87e4f6aa26adf3887", size = 5260043, upload-time = "2026-09-02T14:51:34.841Z" }, + { url = "https://files.pythonhosted.org/packages/b8/d7/ed6ee6186a89e69ca4ea9658b2a278f46a5efe8b5d4db56c7197f18653fe/lxml-6.1.3-cp315-cp315t-win32.whl", hash = "sha256:ace1d2c83b2bd24db5940600541140e87a325e119cb32d5fa9ad720d7e76648e", size = 3901093, upload-time = "2026-09-02T14:51:37.234Z" }, + { url = "https://files.pythonhosted.org/packages/67/9d/11d10257a4a048d04195d638bb61f0246ce2448eb05f682bcbab25a257a8/lxml-6.1.3-cp315-cp315t-win_amd64.whl", hash = "sha256:b49638355ea3bebba70da783ccbc630fd72afa16bc46c54474bfa1f9a915bbc6", size = 4395446, upload-time = "2026-09-02T14:51:39.884Z" }, + { url = "https://files.pythonhosted.org/packages/f8/b7/44edd7de434181c582892e68d1ffe6775ca403ce14aea07cb5a218a936cf/lxml-6.1.3-cp315-cp315t-win_arm64.whl", hash = "sha256:5a721a98c649855963811b59b55755b30566e7f7fc40bdc9803d66dee9f811cf", size = 3822836, upload-time = "2026-09-02T14:51:42.471Z" }, +] + +[[package]] +name = "ncclient" +version = "0.7.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "lxml" }, + { name = "paramiko" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ef/92/e36995ee7ce151814335583571bed95f1a5a491b7ea4d5487cb67377800a/ncclient-0.7.1.tar.gz", hash = "sha256:60dabb6ac1a2d84fbc3349cb104ef7d0c5d12cf8eee43fefc715410d70410ddc", size = 123086, upload-time = "2026-03-15T16:05:49.317Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5b/7c/c0c06d1696b03b901fa5a854b03e16f35c5c5cc8f00a0f43217dc467cd39/ncclient-0.7.1-py3-none-any.whl", hash = "sha256:47beeeee6074bd70a9215c4d353b51c7237af3c5c15269d81692810f2aa15147", size = 94833, upload-time = "2026-03-15T16:05:47.879Z" }, +] + +[[package]] +name = "opentelemetry-api" +version = "1.45.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/19/14/cf3a6b6448849656e1e5f4e2101f3988dc6789724050096aba1bc5ccf592/opentelemetry_api-1.45.2.tar.gz", hash = "sha256:d7b0008e21e5fd61cbe7a7f34ca3b6905a78ab677d9bc395e299359c75cda6b4", size = 73277, upload-time = "2026-10-29T17:29:37.888Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a9/2d/610db864d7513158cebb8f36cc88e67e73bd6c7aa1905ef18d89e1281c81/opentelemetry_api-1.45.2-py3-none-any.whl", hash = "sha256:2cbd737377e710ee740d261c94e93fa0f08e6d037c18f17513cc0a2d94479a34", size = 60335, upload-time = "2026-10-29T17:29:06.955Z" }, +] + +[[package]] +name = "opentelemetry-instrumentation" +version = "0.66b2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-api" }, + { name = "opentelemetry-semantic-conventions" }, + { name = "packaging" }, + { name = "wrapt" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/76/b9/46a136199fa6928eecefec8418867a460d94f01c608a85d22bd2ed4e8d48/opentelemetry_instrumentation-0.66b2.tar.gz", hash = "sha256:02b094150e53b76f5906d75e628dbb3833bbc6e62fa109b5c7ba0c081865e923", size = 43305, upload-time = "2026-10-29T17:38:11.618Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3b/41/014d40e2b08a5d9d9058ed4967ad992657c203ce0c292ded97364ee7b82a/opentelemetry_instrumentation-0.66b2-py3-none-any.whl", hash = "sha256:fa9ccfdab147a14af8107d19ef0a7b5015675273c5adced7a1fc38f28b42ec4c", size = 36902, upload-time = "2026-10-29T17:37:06.994Z" }, +] + +[[package]] +name = "opentelemetry-sdk" +version = "1.45.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-api" }, + { name = "opentelemetry-semantic-conventions" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7b/1a/74a21d4d6969fb8b432ac397e66112a8167a6deb267ea4559dbaf2fe91ef/opentelemetry_sdk-1.45.2.tar.gz", hash = "sha256:7eec4de13dd21ff3a1670dcb0e02079e692736f94d733dfdce5af461d884248d", size = 221705, upload-time = "2026-10-29T17:29:54.685Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9f/a5/9a3b793ef48245f2336a16beb2324c2397dbb3f987cd68942f77cca06156/opentelemetry_sdk-1.45.2-py3-none-any.whl", hash = "sha256:ff974ea70ee4f1d51e2ec152db37c2b626c7bf82bb3f7d86114fd8704e7199c6", size = 141756, upload-time = "2026-10-29T17:29:33.722Z" }, +] + +[[package]] +name = "opentelemetry-semantic-conventions" +version = "0.66b2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-api" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/37/3f/50f95972b50e8e353f5b9342034c9cd17730f1956f009ce6a3fbbd89f9ce/opentelemetry_semantic_conventions-0.66b2.tar.gz", hash = "sha256:7da5f1a5dc4383eb66711d4d3d3e71824d9403ac1559fea021cc103c1a4c4976", size = 150246, upload-time = "2026-10-29T17:29:55.682Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/67/bd/6bf33ed31521c565931c037e9e81bac47c1acb000e850c6e24220e2a0c00/opentelemetry_semantic_conventions-0.66b2-py3-none-any.whl", hash = "sha256:4cbe61e5f9275d57ab31708446e7938ef4291ebbe6a7fca6f7a629c5acae87d8", size = 206280, upload-time = "2026-10-29T17:29:35.062Z" }, +] + +[[package]] +name = "packaging" +version = "26.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7d/fa/3944b40b07da9ce895c0e6303a5ab7d53da063554f534556b134a54d6093/packaging-26.3.tar.gz", hash = "sha256:94edc256424af38762eb31306eed28beb9f0efc50a8837492c9d6fd6004aed79", size = 313412, upload-time = "2026-08-04T18:15:28.737Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/63/34/ba1c580383c9eada3711951fef0795c80b829a078d72188184bcab9dd527/packaging-26.3-py3-none-any.whl", hash = "sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c", size = 129956, upload-time = "2026-08-04T18:15:27.159Z" }, +] + +[[package]] +name = "paramiko" +version = "5.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "bcrypt" }, + { name = "cryptography" }, + { name = "invoke" }, + { name = "pynacl" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/76/cb/b1fde76618d84b1acf54d3d03d2b833ba5878afd12b9899346cc080f209d/paramiko-5.1.0.tar.gz", hash = "sha256:239b5f9b2a53006cd54f83cf8495ee742083708ead947882c8dd2786de79f39b", size = 1579549, upload-time = "2026-10-18T02:38:51.931Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/13/45/fa0e8d70cb25d2dae06bc238e580a84834f6ca53a4e2408cccc19a78db7b/paramiko-5.1.0-py3-none-any.whl", hash = "sha256:d9fe6d67a9d0bbf3eb85b051d26ab850b678c02d060f32915b944e5ca6d0d9e1", size = 211623, upload-time = "2026-10-18T02:38:49.903Z" }, +] + +[[package]] +name = "pluggy" +version = "1.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/bf/db/7fc19e6f2dc92a966727031389fc2e08b558f0f25eb7403c1119ad4713cd/pluggy-1.7.0.tar.gz", hash = "sha256:d1eaa46ebb595891b860ab086b4d09c8588af65ebd4361b8e8f4bb8920b90ba8", size = 123304, upload-time = "2026-10-15T09:50:58.343Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/40/9e/2b38731e0fc536806f16490e1a12d7f0dc2a1235aa8cc07bcc75416a7daa/pluggy-1.7.0-py3-none-any.whl", hash = "sha256:7dd7b0d8832ba3cb632c306926ded123429211b83641b35dc5c41ad2d34f9bec", size = 27082, upload-time = "2026-10-15T09:50:56.808Z" }, +] + +[[package]] +name = "pycparser" +version = "3.12" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/cb/27/eeb9f307bf07060f3d9f9aa0fb8c15b01bdfded0a941e4b2719c7f94cacd/pycparser-3.12.tar.gz", hash = "sha256:f03387877eab82ce01bbdfd0ba7252ee55a90b1c708d5129e2ce915475cf1db6", size = 158785, upload-time = "2026-10-22T13:40:03.56Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b6/ae/315bbeee126a769cbde43cf4908acb4b40408d8bf8bbf334dd7368d8dadf/pycparser-3.12-py3-none-any.whl", hash = "sha256:a076f3f489b39b43ed13563a1dafe42447445bec2e294154ec5c57fcdda4c85e", size = 62600, upload-time = "2026-10-22T13:40:02.021Z" }, +] + +[[package]] +name = "pycryptodomex" +version = "3.24.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f4/e7/50e72301e32acb73a314e663c2014e1ddefc019d5f434ff14ec89861d664/pycryptodomex-3.24.1.tar.gz", hash = "sha256:09081666ffc599976c0b8b29caf2cf82212c0f05bed233c8f8ed53e4f6e1d356", size = 4934627, upload-time = "2026-10-11T19:26:49.624Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/86/2a/1129e2245c474edc9dde1eb3ee5dacc100b896977339117b30e29078784d/pycryptodomex-3.24.1-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:5966f829f64c833cc72a8694cfee05ad50446a44167842c92dec4ebf6b84d72c", size = 2477074, upload-time = "2026-10-11T19:25:36.458Z" }, + { url = "https://files.pythonhosted.org/packages/2d/5e/147a034ada6ecfb38effd375b55c27523851f18509a8312bc617bd0de7de/pycryptodomex-3.24.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:7f30261641dac5ae60e0af2fb11ad7c87200575b0ce403f7531576b0f38a52be", size = 1644417, upload-time = "2026-10-11T19:25:38.567Z" }, + { url = "https://files.pythonhosted.org/packages/19/49/49fc9000d594fd76f9cfe15b6bbf0f9e8d0550202f5a7fa8ddf16846f965/pycryptodomex-3.24.1-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:03e2027f81fe6b700e7ff614d79111559bfb05ed8c4a9de9d2152d1a0a0768de", size = 2195698, upload-time = "2026-10-11T19:25:40.313Z" }, + { url = "https://files.pythonhosted.org/packages/85/40/f64fb00b5bac898d40b1806c4fdb0c6a40e4977a7b5178a38818adcf6043/pycryptodomex-3.24.1-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c20fb5e8cf874dc182091d6df122b8b588501d23e7b500acf662c49899afdd0f", size = 2281501, upload-time = "2026-10-11T19:25:42.037Z" }, + { url = "https://files.pythonhosted.org/packages/96/74/15119b835617b9dc02e78f4706b366e941959b535776b588f1e46ac569bb/pycryptodomex-3.24.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:147c742f73bbe8791d8c454b9972330d7ea501de94e8172957af375f9bdb4a7f", size = 2187040, upload-time = "2026-10-11T19:25:43.795Z" }, + { url = "https://files.pythonhosted.org/packages/3b/c5/d0a7b0613bc3cd9632c5ee9bd85b8458312a217d4c38c8477527eddf7592/pycryptodomex-3.24.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:cdec09935db4cbd74da5b577ea7d9959c73374060b22abf7d505429c0da7ed63", size = 2279237, upload-time = "2026-10-11T19:25:45.724Z" }, + { url = "https://files.pythonhosted.org/packages/3c/3d/da0548d495dc2ccf8118e3902d321ed0fa025ff8a7319e9e97716d0d522c/pycryptodomex-3.24.1-cp313-cp313t-win32.whl", hash = "sha256:6159dc74824c591b4c294f8f96b74a71c3b5e9f637dab2f2da14cfa32dc24d47", size = 1793507, upload-time = "2026-10-11T19:25:47.825Z" }, + { url = "https://files.pythonhosted.org/packages/9b/4a/ddd59bff4eb0a62a5c484b3ad7335b2c8de4399c45d2b49bfba44268d0ac/pycryptodomex-3.24.1-cp313-cp313t-win_amd64.whl", hash = "sha256:edc1deb28fceab6b78e12bae506eaef62ee2fc1b3cabc96f7932f0d51e368acd", size = 1826054, upload-time = "2026-10-11T19:25:50.031Z" }, + { url = "https://files.pythonhosted.org/packages/e1/c0/a9c5270bdaeadc7b0e4674ffb993aadd6aaada37230bbc362423f75cff5a/pycryptodomex-3.24.1-cp313-cp313t-win_arm64.whl", hash = "sha256:5b37b86a3771d6aa21cccf9f8448460e4be11e68bb8e699133ce632cb986f521", size = 1760465, upload-time = "2026-10-11T19:25:51.871Z" }, + { url = "https://files.pythonhosted.org/packages/48/3b/aeef17f479f6c7e367a9f6ad9974866ea311719b2f976e8b26cecd1b27de/pycryptodomex-3.24.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:514d685de4b0227d35d7114047fbd575bf162bff373e07af665aa41eee089dd0", size = 2477998, upload-time = "2026-10-11T19:25:53.619Z" }, + { url = "https://files.pythonhosted.org/packages/18/c1/d26287bf0e1f8c62ed7d448320a0d6704ddbb27f26506e66ffb4431e8ec5/pycryptodomex-3.24.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:7a81e9be7084af3591912475b22c5d0646a9160f3974b6b7300138781fffdc0d", size = 1646567, upload-time = "2026-10-11T19:25:55.45Z" }, + { url = "https://files.pythonhosted.org/packages/82/a6/cf6db80b637d9480f7429797002779d5b57ecbeefb91169e0a70826359ca/pycryptodomex-3.24.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:e87928d8f37952ba53d215836cb3d89c63c1367f037d055c3a5b3f4403f4c6b6", size = 2195699, upload-time = "2026-10-11T19:25:57.591Z" }, + { url = "https://files.pythonhosted.org/packages/2d/20/969e8df9f253b2b9586582e911c2b14ac0a8a0b2cc4f22abbe34c0b1d77c/pycryptodomex-3.24.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f3ed97cdde1d96894057778095238bad3dd94d3e46dc07d6d618e6d4b7700cce", size = 2281499, upload-time = "2026-10-11T19:25:59.648Z" }, + { url = "https://files.pythonhosted.org/packages/fe/33/6a0fe299a475a6f4dfc4dfd34ce70b158556d31e1a043eefa83651152d90/pycryptodomex-3.24.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:48d9058dd4c8f3af83e048b6ff83d9c6300841c42579f77fdfab19e49156a512", size = 2187039, upload-time = "2026-10-11T19:26:01.794Z" }, + { url = "https://files.pythonhosted.org/packages/5f/6f/3580b64c6a166a00f4de5abe977bd4d9a0de893ce1fbe86ddb47851ca7d6/pycryptodomex-3.24.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:9dd5104a1d8ccf725a3c1b40b21d5ed47530b6499abba30955c174d895c1be2f", size = 2279238, upload-time = "2026-10-11T19:26:03.558Z" }, + { url = "https://files.pythonhosted.org/packages/4d/55/ed1fdab2642548db89a3153143f4557315f1cba9e0c66166054bc07c2eaa/pycryptodomex-3.24.1-cp314-cp314t-win32.whl", hash = "sha256:569fdbeff936cbd5beb852b3969dc2f58fb255bd221b3f7a8999c035c30ec958", size = 1809532, upload-time = "2026-10-11T19:26:05.592Z" }, + { url = "https://files.pythonhosted.org/packages/d8/88/8ebf1ae58911beeb7316502decfb0b86913f00a983eecda6cf37ce0c61b6/pycryptodomex-3.24.1-cp314-cp314t-win_amd64.whl", hash = "sha256:3bbcc1807502da4b5d66c94357a99589c8547aa9ad2f9ecfa537b2e9a347538b", size = 1842970, upload-time = "2026-10-11T19:26:07.842Z" }, + { url = "https://files.pythonhosted.org/packages/ce/e2/32a1ffba448295e642cb51f6fb8a5b796cdbefa23bc61379e7b1bea191ac/pycryptodomex-3.24.1-cp314-cp314t-win_arm64.whl", hash = "sha256:794f32227a480ab3b39971ac4a53dfa8ed444e28c0ddecbdc35f26d038bb46da", size = 1779090, upload-time = "2026-10-11T19:26:09.75Z" }, + { url = "https://files.pythonhosted.org/packages/bf/ae/48679004ab908818326a62c35586b51ccd6ba0e3a03d0ae637cd13e8df99/pycryptodomex-3.24.1-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:9bb353c764c144a9fc03302ef3763ad0c3e75bc7ca41f445cc98455f36da6c59", size = 2477923, upload-time = "2026-10-11T19:26:11.828Z" }, + { url = "https://files.pythonhosted.org/packages/11/3f/14abc2a50521961dc864266d95898e87e073379ee9407f65c9d259f75018/pycryptodomex-3.24.1-cp37-abi3-macosx_10_9_x86_64.whl", hash = "sha256:eeac2c9acbd2d9f0ca493fdf692ce8245ca37cbebee08c496025868d4b8ef70f", size = 1645300, upload-time = "2026-10-11T19:26:13.583Z" }, + { url = "https://files.pythonhosted.org/packages/3b/5f/44916f6aa08a08b34b3ce41e2282d36149aa416a8edca9dbecd7c07e0a10/pycryptodomex-3.24.1-cp37-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:9a732b9f5603b153dcdc071d6b342f192981907519827f2c1e4aeac41e2b34c2", size = 2194142, upload-time = "2026-10-11T19:26:15.535Z" }, + { url = "https://files.pythonhosted.org/packages/cf/93/a954b7a8746980e31648943e29772aca0311d0d8a407a3adf5788083b693/pycryptodomex-3.24.1-cp37-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:9b94a8c647c1f80b4c82c7ac9642dda94a71abbb4efa29ad652c410ad7a39879", size = 2280275, upload-time = "2026-10-11T19:26:17.671Z" }, + { url = "https://files.pythonhosted.org/packages/90/8b/bb2bcc9da5cb1b70415bbfaa4d9dc837a677d021b4da16ed99b0159b42b1/pycryptodomex-3.24.1-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:bd06ab1d9cf90e8b198daae3b364e9ac23a640d210bb8faddad6627cf2e33be7", size = 2185457, upload-time = "2026-10-11T19:26:19.818Z" }, + { url = "https://files.pythonhosted.org/packages/e9/54/c123cfe95a7caba1eee9cb92e8482d59f74638b92d9035dff673246c45a2/pycryptodomex-3.24.1-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:8321c315c4418dfe45a811b792380438ec81185deaaf202dbcd77711b4f51796", size = 2277787, upload-time = "2026-10-11T19:26:21.765Z" }, + { url = "https://files.pythonhosted.org/packages/5b/58/b06b695ff0f31261e841a25fc1342c8e774167e0d6db5d0a569fe1a137f6/pycryptodomex-3.24.1-cp37-abi3-win32.whl", hash = "sha256:cc64f4e1fc07a155ef31eb9815183a6a8eb13dfd6876ab049c01125223aa0c40", size = 1793402, upload-time = "2026-10-11T19:26:23.589Z" }, + { url = "https://files.pythonhosted.org/packages/13/4c/2c1963e4154da534a3b3a4d6ee9549f5cac4ecffe986d3b470d08b9c4df6/pycryptodomex-3.24.1-cp37-abi3-win_amd64.whl", hash = "sha256:82eb0dd8a95be97f03527b108ee49f9b86a7c534fa47fa7a510be3eab8ea9acd", size = 1825955, upload-time = "2026-10-11T19:26:25.258Z" }, + { url = "https://files.pythonhosted.org/packages/57/d2/a3a2158e44f468993583c2a0ae07f8b2a58f349e8cac1b4f039745d80945/pycryptodomex-3.24.1-cp37-abi3-win_arm64.whl", hash = "sha256:a692d2484ca8fa2c69f45c63f2b30cd00b8512834e3a522fc297d981dde5b34c", size = 1760463, upload-time = "2026-10-11T19:26:27.457Z" }, +] + +[[package]] +name = "pygments" +version = "2.21.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/49/2e/ced460408999b33da6b31b0021b0f37d329e202d4169aeb164493778f25b/pygments-2.21.0.tar.gz", hash = "sha256:610ca751c9bc2492b38eb9a38a7fbc93edbbb2d7182edaf34e66ae493dee5c8c", size = 5005329, upload-time = "2026-08-17T08:02:48.824Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/71/46/17f022dd3e953bf20a04a028a21ec746d942f8d2af30fa0f124fa0e6a684/pygments-2.21.0-py3-none-any.whl", hash = "sha256:2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9", size = 1250147, upload-time = "2026-08-17T08:02:44.912Z" }, +] + +[[package]] +name = "pynacl" +version = "1.6.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cffi", marker = "platform_python_implementation != 'PyPy'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d9/9a/4019b524b03a13438637b11538c82781a5eda427394380381af8f04f467a/pynacl-1.6.2.tar.gz", hash = "sha256:018494d6d696ae03c7e656e5e74cdfd8ea1326962cc401bcf018f1ed8436811c", size = 3511692, upload-time = "2026-01-01T17:48:10.851Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4b/79/0e3c34dc3c4671f67d251c07aa8eb100916f250ee470df230b0ab89551b4/pynacl-1.6.2-cp314-cp314t-macosx_10_10_universal2.whl", hash = "sha256:622d7b07cc5c02c666795792931b50c91f3ce3c2649762efb1ef0d5684c81594", size = 390064, upload-time = "2026-01-01T17:31:57.264Z" }, + { url = "https://files.pythonhosted.org/packages/eb/1c/23a26e931736e13b16483795c8a6b2f641bf6a3d5238c22b070a5112722c/pynacl-1.6.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d071c6a9a4c94d79eb665db4ce5cedc537faf74f2355e4d502591d850d3913c0", size = 809370, upload-time = "2026-01-01T17:31:59.198Z" }, + { url = "https://files.pythonhosted.org/packages/87/74/8d4b718f8a22aea9e8dcc8b95deb76d4aae380e2f5b570cc70b5fd0a852d/pynacl-1.6.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:fe9847ca47d287af41e82be1dd5e23023d3c31a951da134121ab02e42ac218c9", size = 1408304, upload-time = "2026-01-01T17:32:01.162Z" }, + { url = "https://files.pythonhosted.org/packages/fd/73/be4fdd3a6a87fe8a4553380c2b47fbd1f7f58292eb820902f5c8ac7de7b0/pynacl-1.6.2-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:04316d1fc625d860b6c162fff704eb8426b1a8bcd3abacea11142cbd99a6b574", size = 844871, upload-time = "2026-01-01T17:32:02.824Z" }, + { url = "https://files.pythonhosted.org/packages/55/ad/6efc57ab75ee4422e96b5f2697d51bbcf6cdcc091e66310df91fbdc144a8/pynacl-1.6.2-cp314-cp314t-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:44081faff368d6c5553ccf55322ef2819abb40e25afaec7e740f159f74813634", size = 1446356, upload-time = "2026-01-01T17:32:04.452Z" }, + { url = "https://files.pythonhosted.org/packages/78/b7/928ee9c4779caa0a915844311ab9fb5f99585621c5d6e4574538a17dca07/pynacl-1.6.2-cp314-cp314t-manylinux_2_34_aarch64.whl", hash = "sha256:a9f9932d8d2811ce1a8ffa79dcbdf3970e7355b5c8eb0c1a881a57e7f7d96e88", size = 826814, upload-time = "2026-01-01T17:32:06.078Z" }, + { url = "https://files.pythonhosted.org/packages/f7/a9/1bdba746a2be20f8809fee75c10e3159d75864ef69c6b0dd168fc60e485d/pynacl-1.6.2-cp314-cp314t-manylinux_2_34_x86_64.whl", hash = "sha256:bc4a36b28dd72fb4845e5d8f9760610588a96d5a51f01d84d8c6ff9849968c14", size = 1411742, upload-time = "2026-01-01T17:32:07.651Z" }, + { url = "https://files.pythonhosted.org/packages/f3/2f/5e7ea8d85f9f3ea5b6b87db1d8388daa3587eed181bdeb0306816fdbbe79/pynacl-1.6.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:3bffb6d0f6becacb6526f8f42adfb5efb26337056ee0831fb9a7044d1a964444", size = 801714, upload-time = "2026-01-01T17:32:09.558Z" }, + { url = "https://files.pythonhosted.org/packages/06/ea/43fe2f7eab5f200e40fb10d305bf6f87ea31b3bbc83443eac37cd34a9e1e/pynacl-1.6.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:2fef529ef3ee487ad8113d287a593fa26f48ee3620d92ecc6f1d09ea38e0709b", size = 1372257, upload-time = "2026-01-01T17:32:11.026Z" }, + { url = "https://files.pythonhosted.org/packages/4d/54/c9ea116412788629b1347e415f72195c25eb2f3809b2d3e7b25f5c79f13a/pynacl-1.6.2-cp314-cp314t-win32.whl", hash = "sha256:a84bf1c20339d06dc0c85d9aea9637a24f718f375d861b2668b2f9f96fa51145", size = 231319, upload-time = "2026-01-01T17:32:12.46Z" }, + { url = "https://files.pythonhosted.org/packages/ce/04/64e9d76646abac2dccf904fccba352a86e7d172647557f35b9fe2a5ee4a1/pynacl-1.6.2-cp314-cp314t-win_amd64.whl", hash = "sha256:320ef68a41c87547c91a8b58903c9caa641ab01e8512ce291085b5fe2fcb7590", size = 244044, upload-time = "2026-01-01T17:32:13.781Z" }, + { url = "https://files.pythonhosted.org/packages/33/33/7873dc161c6a06f43cda13dec67b6fe152cb2f982581151956fa5e5cdb47/pynacl-1.6.2-cp314-cp314t-win_arm64.whl", hash = "sha256:d29bfe37e20e015a7d8b23cfc8bd6aa7909c92a1b8f41ee416bbb3e79ef182b2", size = 188740, upload-time = "2026-01-01T17:32:15.083Z" }, + { url = "https://files.pythonhosted.org/packages/be/7b/4845bbf88e94586ec47a432da4e9107e3fc3ce37eb412b1398630a37f7dd/pynacl-1.6.2-cp38-abi3-macosx_10_10_universal2.whl", hash = "sha256:c949ea47e4206af7c8f604b8278093b674f7c79ed0d4719cc836902bf4517465", size = 388458, upload-time = "2026-01-01T17:32:16.829Z" }, + { url = "https://files.pythonhosted.org/packages/1e/b4/e927e0653ba63b02a4ca5b4d852a8d1d678afbf69b3dbf9c4d0785ac905c/pynacl-1.6.2-cp38-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8845c0631c0be43abdd865511c41eab235e0be69c81dc66a50911594198679b0", size = 800020, upload-time = "2026-01-01T17:32:18.34Z" }, + { url = "https://files.pythonhosted.org/packages/7f/81/d60984052df5c97b1d24365bc1e30024379b42c4edcd79d2436b1b9806f2/pynacl-1.6.2-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:22de65bb9010a725b0dac248f353bb072969c94fa8d6b1f34b87d7953cf7bbe4", size = 1399174, upload-time = "2026-01-01T17:32:20.239Z" }, + { url = "https://files.pythonhosted.org/packages/68/f7/322f2f9915c4ef27d140101dd0ed26b479f7e6f5f183590fd32dfc48c4d3/pynacl-1.6.2-cp38-abi3-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:46065496ab748469cdd999246d17e301b2c24ae2fdf739132e580a0e94c94a87", size = 835085, upload-time = "2026-01-01T17:32:22.24Z" }, + { url = "https://files.pythonhosted.org/packages/3e/d0/f301f83ac8dbe53442c5a43f6a39016f94f754d7a9815a875b65e218a307/pynacl-1.6.2-cp38-abi3-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8a66d6fb6ae7661c58995f9c6435bda2b1e68b54b598a6a10247bfcdadac996c", size = 1437614, upload-time = "2026-01-01T17:32:23.766Z" }, + { url = "https://files.pythonhosted.org/packages/c4/58/fc6e649762b029315325ace1a8c6be66125e42f67416d3dbd47b69563d61/pynacl-1.6.2-cp38-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:26bfcd00dcf2cf160f122186af731ae30ab120c18e8375684ec2670dccd28130", size = 818251, upload-time = "2026-01-01T17:32:25.69Z" }, + { url = "https://files.pythonhosted.org/packages/c9/a8/b917096b1accc9acd878819a49d3d84875731a41eb665f6ebc826b1af99e/pynacl-1.6.2-cp38-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:c8a231e36ec2cab018c4ad4358c386e36eede0319a0c41fed24f840b1dac59f6", size = 1402859, upload-time = "2026-01-01T17:32:27.215Z" }, + { url = "https://files.pythonhosted.org/packages/85/42/fe60b5f4473e12c72f977548e4028156f4d340b884c635ec6b063fe7e9a5/pynacl-1.6.2-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:68be3a09455743ff9505491220b64440ced8973fe930f270c8e07ccfa25b1f9e", size = 791926, upload-time = "2026-01-01T17:32:29.314Z" }, + { url = "https://files.pythonhosted.org/packages/fa/f9/e40e318c604259301cc091a2a63f237d9e7b424c4851cafaea4ea7c4834e/pynacl-1.6.2-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:8b097553b380236d51ed11356c953bf8ce36a29a3e596e934ecabe76c985a577", size = 1363101, upload-time = "2026-01-01T17:32:31.263Z" }, + { url = "https://files.pythonhosted.org/packages/48/47/e761c254f410c023a469284a9bc210933e18588ca87706ae93002c05114c/pynacl-1.6.2-cp38-abi3-win32.whl", hash = "sha256:5811c72b473b2f38f7e2a3dc4f8642e3a3e9b5e7317266e4ced1fba85cae41aa", size = 227421, upload-time = "2026-01-01T17:32:33.076Z" }, + { url = "https://files.pythonhosted.org/packages/41/ad/334600e8cacc7d86587fe5f565480fde569dfb487389c8e1be56ac21d8ac/pynacl-1.6.2-cp38-abi3-win_amd64.whl", hash = "sha256:62985f233210dee6548c223301b6c25440852e13d59a8b81490203c3227c5ba0", size = 239754, upload-time = "2026-01-01T17:32:34.557Z" }, + { url = "https://files.pythonhosted.org/packages/29/7d/5945b5af29534641820d3bd7b00962abbbdfee84ec7e19f0d5b3175f9a31/pynacl-1.6.2-cp38-abi3-win_arm64.whl", hash = "sha256:834a43af110f743a754448463e8fd61259cd4ab5bbedcf70f9dabad1d28a394c", size = 184801, upload-time = "2026-01-01T17:32:36.309Z" }, +] + +[[package]] +name = "pytest" +version = "9.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/79/42/bb4641652e58dedeac2ec51abc1f8cae4fee4ec63899a0999016229a0f2a/pytest-9.2.0.tar.gz", hash = "sha256:97c4f3b1f3c7414aa36465e09b8220f1d03627993bd51b1282e193753c80557b", size = 1797555, upload-time = "2026-10-27T21:30:20.728Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/51/39/1f5d8c8877a0fa327db959c19341d4be61660de1ab6b7391f9eb805bcd67/pytest-9.2.0-py3-none-any.whl", hash = "sha256:07acdcf88ec174f907dc01af70bf52d49b014bbab03faeeade31c96b18c2e588", size = 411776, upload-time = "2026-10-27T21:30:19.131Z" }, +] + +[[package]] +name = "pytest-cov" +version = "7.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "coverage" }, + { name = "pluggy" }, + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b1/51/a849f96e117386044471c8ec2bd6cfebacda285da9525c9106aeb28da671/pytest_cov-7.1.0.tar.gz", hash = "sha256:30674f2b5f6351aa09702a9c8c364f6a01c27aae0c1366ae8016160d1efc56b2", size = 55592, upload-time = "2026-03-21T20:11:16.284Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9d/7a/d968e294073affff457b041c2be9868a40c1c71f4a35fcc1e45e5493067b/pytest_cov-7.1.0-py3-none-any.whl", hash = "sha256:a0461110b7865f9a271aa1b51e516c9a95de9d696734a2f71e3e78f46e1d4678", size = 22876, upload-time = "2026-03-21T20:11:14.438Z" }, +] + +[[package]] +name = "typing-extensions" +version = "4.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f6/cc/6253133b5bb138fc3306cebfbda2c520f545d36b5be2c7255cc528bb45d6/typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5", size = 113555, upload-time = "2026-07-02T08:40:05.92Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8", size = 45571, upload-time = "2026-07-02T08:40:04.659Z" }, +] + +[[package]] +name = "wrapt" +version = "2.5.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9c/b0/f752d19e1e83ab8bd64aeccfde32ff086aee3ef08520f7340b69a8d0a4d2/wrapt-2.5.2.tar.gz", hash = "sha256:551cb49e88b3722faad780303833bc4fa70685fb1773270fb98b40d39d9fddbf", size = 186066, upload-time = "2026-10-29T23:06:39.868Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4b/fe/9ccae0a147d0732397f54ff232b1a0da148a4099f17c1b156b9d3e79127e/wrapt-2.5.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:bec3b6b1ae2f7ed763fb0ba1776b9b5f11283e6ff40c9cbe3a855ea652c326cf", size = 106628, upload-time = "2026-10-29T23:04:44.962Z" }, + { url = "https://files.pythonhosted.org/packages/d5/ca/a61d994df49fa422abd859e525e04b95ef2888e3818a0f451aa7d4d3527f/wrapt-2.5.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:e57d79e3338b3cecee1dd80d6ef8f945f8ea7b92f0d8d5591f840ffa0afc14dc", size = 106264, upload-time = "2026-10-29T23:04:46.307Z" }, + { url = "https://files.pythonhosted.org/packages/dd/26/a6caa9a84ef5db4a273787949cea7ab51f6001d2548768af4c8e3c49debd/wrapt-2.5.2-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:092472b7b1494deef40dac5e9f6295013a9d51b112b5fbc4350f64dc2574e46a", size = 250614, upload-time = "2026-10-29T23:04:47.735Z" }, + { url = "https://files.pythonhosted.org/packages/c7/28/2189571e711ab31cb7f682c739d05f033ae0e83e12eb4d5d21473ca2cd1a/wrapt-2.5.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d108bd98ab2c5ccbf55cf2154e0a73341df930030df2e01ee35d71a44a99d3ad", size = 250770, upload-time = "2026-10-29T23:04:49.183Z" }, + { url = "https://files.pythonhosted.org/packages/80/8e/46a6cc2e8215df7fae292178a355df729547ff9b912686823201d38be269/wrapt-2.5.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:858511731d6ede4217960c5fecdb5ecbf90e306dfd93b837395658d61374a498", size = 230031, upload-time = "2026-10-29T23:04:50.641Z" }, + { url = "https://files.pythonhosted.org/packages/97/38/dff531d0125fe880e05fe5fcb89f144d25ffbe95aa9a2c50539eabde7567/wrapt-2.5.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3b6885250b809532a634acd56e360d57c201c1cdc75cbf2b5e106241ece6cbc3", size = 247325, upload-time = "2026-10-29T23:04:52.16Z" }, + { url = "https://files.pythonhosted.org/packages/f0/5b/f4a78df4c52175d53ff095f2f5a54206a2d16e542b1dd2cdcaed901fe432/wrapt-2.5.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:c60a5cd8b7b510056f2643d94f1c144533a568ade2891a9dd35733f5e0055777", size = 226900, upload-time = "2026-10-29T23:04:53.559Z" }, + { url = "https://files.pythonhosted.org/packages/20/6f/8eab144c717300f76f1337ba58091f4de5da0f4d772727c9f41135b325c1/wrapt-2.5.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:40e03506f09dbf4854fd69848e843dadd9b7fbbdd217837cb27b37fb8f0b5bc3", size = 239017, upload-time = "2026-10-29T23:04:55.293Z" }, + { url = "https://files.pythonhosted.org/packages/59/1a/98e278ef2f8952a34f847a91f75899f6385762b07577c51a5ef82c768dfb/wrapt-2.5.2-cp312-cp312-win32.whl", hash = "sha256:ac00ba7d22d55d3c8d085005ce4942ff3bb118d480f889b5984f1d3454052917", size = 100660, upload-time = "2026-10-29T23:04:56.703Z" }, + { url = "https://files.pythonhosted.org/packages/7d/0a/ab645f8b4c594cac2db49bbc805bff97a9d3f2944061b865f67e0351b6c8/wrapt-2.5.2-cp312-cp312-win_amd64.whl", hash = "sha256:36021ef761856bdbc10d5f3da9a95af93ef60d6acf9c72c8f849937ba2b902aa", size = 106220, upload-time = "2026-10-29T23:04:58.081Z" }, + { url = "https://files.pythonhosted.org/packages/37/1e/232868b68b925ab29ca5acbec4ad54ee42a853fcd8f596bf7e0e0e3850f1/wrapt-2.5.2-cp312-cp312-win_arm64.whl", hash = "sha256:81bdf0bd141d9b1706a0751b2ee068c7afe6be4042712d427b32b3072d5c40b9", size = 103377, upload-time = "2026-10-29T23:04:59.366Z" }, + { url = "https://files.pythonhosted.org/packages/1b/7a/38f08a89ca8faa9b788a857f109b5e4cd1f25671a5157e2218072f64aa76/wrapt-2.5.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:88f982c2bb306732271aa15cded4002ce28b5d9023de9d11e927da4dcf65e441", size = 106242, upload-time = "2026-10-29T23:05:00.746Z" }, + { url = "https://files.pythonhosted.org/packages/e9/8b/2fb9b2522b7e1e7534a72fec0040bbf3d273129ec0a98316e7d37b09aa35/wrapt-2.5.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:269321b17467a10997cc6e14793bf1816e6ce488f9711c74b72552466d206739", size = 106366, upload-time = "2026-10-29T23:05:02.191Z" }, + { url = "https://files.pythonhosted.org/packages/c2/1e/770d73e0cc82a41602cb2cb184769c28bdbaa9b998944d577a1097c18da3/wrapt-2.5.2-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:8b039b3a296762db1908d6108365225393c4fb33c1dbc5f00aa36f8302eaeaa0", size = 248696, upload-time = "2026-10-29T23:05:03.593Z" }, + { url = "https://files.pythonhosted.org/packages/ac/2b/b4362e803d870451b5e701df5e30ded2d6fb1d9e0c01eec4dc7fa8332b57/wrapt-2.5.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e1af6136ac5bcfab17190a70b25d8437eb8d8f9ca2b821a0655d39275a8cb7e7", size = 248663, upload-time = "2026-10-29T23:05:05.084Z" }, + { url = "https://files.pythonhosted.org/packages/87/15/244dbd2d86285857898be5f7632e8375161953e645f250ac9778c058ab9b/wrapt-2.5.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9d4e1cdf2efdc7181c8177f6090886e22278e70f7257df12863d2f52a9dcbbd7", size = 227568, upload-time = "2026-10-29T23:05:06.836Z" }, + { url = "https://files.pythonhosted.org/packages/62/0e/15ae5812d510fbfb65acb3a63aecbd8ffb4b944b2fd0131d968e31df89d6/wrapt-2.5.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e33578755e07edf3a70df3241f4cf376dad507e2cc73e737311bb00b5067f4d7", size = 245147, upload-time = "2026-10-29T23:05:08.322Z" }, + { url = "https://files.pythonhosted.org/packages/f6/c2/e611b881c05f7f2a5c88f9d89236afb091b8adba3ac89ff9040b6a837be4/wrapt-2.5.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:2bf23cde96e0f771a6a2c68973c1ded45c9707f8bd519ede30ecf075f3c720f6", size = 225491, upload-time = "2026-10-29T23:05:10.102Z" }, + { url = "https://files.pythonhosted.org/packages/f3/f4/bdb253208ec907097809e34c25d199cd66686a7974257e08b57d96e3176c/wrapt-2.5.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eff91af619ad5d587f8855f6c02164e12d33d205c702eb73d487deb499520265", size = 237319, upload-time = "2026-10-29T23:05:11.625Z" }, + { url = "https://files.pythonhosted.org/packages/67/86/74205217834201f27bc73a76b4db67f488fd07787e12a045fb2b39f09c29/wrapt-2.5.2-cp313-cp313-win32.whl", hash = "sha256:11655df8e17658a67628cdc97fe95d6b263e023213cf18067f21e5ec0b0285f3", size = 100634, upload-time = "2026-10-29T23:05:13.073Z" }, + { url = "https://files.pythonhosted.org/packages/e0/56/7c732aed39116853d74485b4431a0ea37b05bd2bf250bdddc55ceb64f8c0/wrapt-2.5.2-cp313-cp313-win_amd64.whl", hash = "sha256:5726faa7342ba473041bc1a276486d232b42e3927d35427729b266b1b5ecaa42", size = 106044, upload-time = "2026-10-29T23:05:14.372Z" }, + { url = "https://files.pythonhosted.org/packages/0c/a7/c4f7ae891bbfef566aa63effe846f6804dac4c0461601993f4c59deaf955/wrapt-2.5.2-cp313-cp313-win_arm64.whl", hash = "sha256:e1fb4af8aac3573770ed3e64efc39d953342b12bcb6e6479512a945eb5557523", size = 103077, upload-time = "2026-10-29T23:05:15.718Z" }, + { url = "https://files.pythonhosted.org/packages/ac/dc/a259acf2b59eefe400b2a1c6b45bc19b0d0b583722426c9dabb276cc0d19/wrapt-2.5.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:229859aebfc909626ea6b5a3ae9523aba43c63fd828f876f2a813f4a5767726f", size = 106620, upload-time = "2026-10-29T23:05:17.256Z" }, + { url = "https://files.pythonhosted.org/packages/f8/c5/b34ca079b7906eb043c53c06896b4e42f3ca9efa97ac10e4e254c88e2ca6/wrapt-2.5.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c1ea16d7dfe7d235d29435307b1539495b4925e88b135d532892973a04569245", size = 106555, upload-time = "2026-10-29T23:05:18.656Z" }, + { url = "https://files.pythonhosted.org/packages/e2/46/f66042c9ed3626cdcdf641105f091964f6d38cd406cec96479c4a3f70640/wrapt-2.5.2-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:59b7e6d1dea7c3f38bab21954f587499c27c16ea537058eee4add6e4da1780ea", size = 251681, upload-time = "2026-10-29T23:05:20.035Z" }, + { url = "https://files.pythonhosted.org/packages/70/95/1ebe33ff8053a94b3e1a0bafe31ea94b5e9296f9c5098cbcafc4ee2bcdc9/wrapt-2.5.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cfd508866c421687a99bed9ce5b7b4f26fd9b93d3753685d81947ab37d54c496", size = 250919, upload-time = "2026-10-29T23:05:21.501Z" }, + { url = "https://files.pythonhosted.org/packages/1c/bf/ee90a8674c8f2179ad7aeaf8c5b875fa33a12fec7bf1ccf6cbfcc31fbbf0/wrapt-2.5.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:459bb66cd4bc43e35cba1f1ab2bdfb9b507e0d8dc0ec40c93a060c9cb0c5415e", size = 231773, upload-time = "2026-10-29T23:05:22.996Z" }, + { url = "https://files.pythonhosted.org/packages/e8/ee/610ef6bf4f35e4719a5edea6bd0c4d948921d028950bc9669d1fa9234af9/wrapt-2.5.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:a8626135ecbb0b892ca79c388581ab0ef57e087f09dc0e51a2f34185d3682055", size = 248171, upload-time = "2026-10-29T23:05:24.481Z" }, + { url = "https://files.pythonhosted.org/packages/c6/10/6f12cf42bd9a34829c902c8a617c8151a4e16e7d7cb015289eede5813f39/wrapt-2.5.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:7ffa0be31f651da63ee51e5b74701339feac1860b561a985c7b4869d4600e800", size = 228970, upload-time = "2026-10-29T23:05:25.904Z" }, + { url = "https://files.pythonhosted.org/packages/b1/f1/79417514032b9015b772bf5bf006ca542937b5f6c483b9ab092f9ceff8e9/wrapt-2.5.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5ab31d929ca8b2b7bf9320ad2a927edfb1841609951a869a9b98723b285ffbc6", size = 239267, upload-time = "2026-10-29T23:05:27.502Z" }, + { url = "https://files.pythonhosted.org/packages/a1/ce/092f2723758586973c94afa8c38a206ce3720ec5cd3edb64ee9c8b27a1c4/wrapt-2.5.2-cp314-cp314-win32.whl", hash = "sha256:afd99530994449aaac9a58e1e268eb080bb1ffd758be43ab2f8a249ce8970c9c", size = 101091, upload-time = "2026-10-29T23:05:28.955Z" }, + { url = "https://files.pythonhosted.org/packages/23/86/201fb9180290373b7fdff18e8d600151694fdb0b10892cdee77c13eb7e6b/wrapt-2.5.2-cp314-cp314-win_amd64.whl", hash = "sha256:fabc60720d1e4a36bbe8aaf1c32024f04c6674991313f5acaca5eeadb564c600", size = 106417, upload-time = "2026-10-29T23:05:30.395Z" }, + { url = "https://files.pythonhosted.org/packages/73/a8/cc313dc9230f4b5029e24867a1fc1dbcb929b792b02ac5fb2e0d79df7401/wrapt-2.5.2-cp314-cp314-win_arm64.whl", hash = "sha256:1c17ca5ce83c7145eddaa20073398ec2192dad65c8d8ff4d9b78fd2917c46a75", size = 104145, upload-time = "2026-10-29T23:05:31.788Z" }, + { url = "https://files.pythonhosted.org/packages/62/04/15e0c465088fbfdab321a672c88739f68429a18aeb81ad7995a0a5cf2065/wrapt-2.5.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:a60863d874cc6ac917bbf44324e3f195564064dee15c84e83b19e1fbbccd3169", size = 109731, upload-time = "2026-10-29T23:05:33.131Z" }, + { url = "https://files.pythonhosted.org/packages/b1/b6/91d6f77257afabafe19bca698b41d60dde7cf49b59b0126eb25ed6a3a90e/wrapt-2.5.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:d41c45517912610de55546188695e2bd0fbcd70b585087cd759f85a96f47be7e", size = 110032, upload-time = "2026-10-29T23:05:34.589Z" }, + { url = "https://files.pythonhosted.org/packages/9c/84/40901e8ba08f6f18f213fea51daa41a8107ccaae4e0b6e13b842f02f5feb/wrapt-2.5.2-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:48fd5026977adb3e2fda292a6a796967b511935d23f97dc2ae1f127a3a8639d5", size = 284278, upload-time = "2026-10-29T23:05:36.033Z" }, + { url = "https://files.pythonhosted.org/packages/89/af/6bfdffc2688a78019762f0b14171ca47c73fae1421c3324c0850135b212c/wrapt-2.5.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0d7639c3287d4efcdc2c8837f1854d1ecf27aae761714ba3d95e61fe8eb48db3", size = 294060, upload-time = "2026-10-29T23:05:37.515Z" }, + { url = "https://files.pythonhosted.org/packages/56/d1/9a4e3aed00d6d877d2646072d9b6b4c9a558a8ad79a18d3abafc1079cd07/wrapt-2.5.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:eeaa40251c40e18c5c97ef11d5dbbad28a7b7b99fdd7a7469662799100d74dc7", size = 263272, upload-time = "2026-10-29T23:05:39.008Z" }, + { url = "https://files.pythonhosted.org/packages/5f/6f/0086e02aae9144a0fa029fb0351a8b2f91fbd28eb191c8f2ced3831cb1ba/wrapt-2.5.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:b9ce67ec8803e6c56e78c9ebf726f61116516212c28dfa3683d0962e15cf9a62", size = 289219, upload-time = "2026-10-29T23:05:40.459Z" }, + { url = "https://files.pythonhosted.org/packages/d2/ac/e342f997166b7d433abc12720b6e30793fcf3a36c5457d22261558b61411/wrapt-2.5.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:4d5faf5c3e435207b3a51294ed5a00e5e17e3842442ccbbd864ee9bc0efc63fb", size = 260554, upload-time = "2026-10-29T23:05:41.815Z" }, + { url = "https://files.pythonhosted.org/packages/65/64/48ddff54611f6e08ba70adf1b3dbf36c7b08b7aa5dbf7c65a03672b0eed1/wrapt-2.5.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:2861d50a4f330fa140ddc2a9ddee33f7e4f9bdb649a23dc9be8ed6310ffa47fc", size = 278921, upload-time = "2026-10-29T23:05:43.489Z" }, + { url = "https://files.pythonhosted.org/packages/c0/bf/0ca422495491f22656b401832184a87b1e67d1dfb7f31ac8d60525c572ce/wrapt-2.5.2-cp314-cp314t-win32.whl", hash = "sha256:46cee4400437c9f33e755522be9e7ce6ebd097273f464eb6f57bee09787f0c59", size = 103586, upload-time = "2026-10-29T23:05:45.046Z" }, + { url = "https://files.pythonhosted.org/packages/69/17/922d99009a228907cd0a5f5d590bbc176dd70527d3eb06f8b4d74889edf8/wrapt-2.5.2-cp314-cp314t-win_amd64.whl", hash = "sha256:1ad706a42b139ad7f03f32aed57da5e10fe8ed707e29849cfa94f1d49cba1405", size = 110111, upload-time = "2026-10-29T23:05:46.458Z" }, + { url = "https://files.pythonhosted.org/packages/68/45/d168c09643e3207c5618b33a5636137f298b9ded38b3a33fad67dc343403/wrapt-2.5.2-cp314-cp314t-win_arm64.whl", hash = "sha256:450cc93f5a2d58d4a74d384d52db4b41d5728065c0057c9bb4666c6115f2a2c9", size = 106384, upload-time = "2026-10-29T23:05:48.051Z" }, + { url = "https://files.pythonhosted.org/packages/67/8c/3b2b95cd60b78f9450d748854e421fcf48bc350d76ca9e0fe048b87ad796/wrapt-2.5.2-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:2c17ec4310cc06f864799a4c37c1c716d5dcf05ceead2e63ca992a9466903b73", size = 106621, upload-time = "2026-10-29T23:05:49.433Z" }, + { url = "https://files.pythonhosted.org/packages/16/12/c430cd1289711cdf2b6b1c469d5efa527f4815fabab32bff3249545344da/wrapt-2.5.2-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:430070776fb30d925a7a1e13d68086e4731b63eedf4c43ee62fc4c8108920cce", size = 106590, upload-time = "2026-10-29T23:05:50.967Z" }, + { url = "https://files.pythonhosted.org/packages/69/4b/d7230c57ca30023a5630bf5cc0e092ad1cfce4d999a309b8377a36b3c249/wrapt-2.5.2-cp315-cp315-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:99978d196df45c115273c6119e4d0dacbc67bc267a196506625bf7084b741f0b", size = 252192, upload-time = "2026-10-29T23:05:52.344Z" }, + { url = "https://files.pythonhosted.org/packages/fa/74/6dc841c96b9906e0c466aaeed7f51162129cc8745a91bb28168aa3b45194/wrapt-2.5.2-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:217798fb0cfb31479a264f841b0242bc1bed1ffc70d233e8444a876cec57928c", size = 251207, upload-time = "2026-10-29T23:05:54.011Z" }, + { url = "https://files.pythonhosted.org/packages/52/bb/25e710cc9d3bd25d81d46791dd25029d6f9b9d44b990e3d81755b549ff71/wrapt-2.5.2-cp315-cp315-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:62b339565cfdfcc0ba71c527b4ad13ac9de25504d474fa4ae5db95d1c7e4c6a2", size = 233048, upload-time = "2026-10-29T23:05:55.796Z" }, + { url = "https://files.pythonhosted.org/packages/4e/80/672640d799862a36abae740e778146dbe155ec8ffd6211b3a99240f599ca/wrapt-2.5.2-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:84619ca6b6f67a7dd53754ad402cafceb4ddc9d8981f144f62718b783f05786e", size = 248479, upload-time = "2026-10-29T23:05:57.346Z" }, + { url = "https://files.pythonhosted.org/packages/f6/34/91eb6724eb7cf64b77637d720ef71cad1ec6e4ce2d15eb548670c47078fa/wrapt-2.5.2-cp315-cp315-musllinux_1_2_riscv64.whl", hash = "sha256:08d1d7bcb7dd277b7b976fd914455c3c1088f5c101ac7469ae5a5b8c03be0a0a", size = 230013, upload-time = "2026-10-29T23:05:58.745Z" }, + { url = "https://files.pythonhosted.org/packages/bf/f8/f1fc0c3394819c9a6a3c0d0055b06614a3bef7ad133cf18f9c09a2b2141f/wrapt-2.5.2-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:4e86f3543c67d72b4d4c08c3e351523c2d69cd05932df65937d79bc472e9d62f", size = 239808, upload-time = "2026-10-29T23:06:00.266Z" }, + { url = "https://files.pythonhosted.org/packages/56/d1/70f9e71674d7b6c21c8e8cafc3016f773dc3acb31a131f23ab4af8c1effb/wrapt-2.5.2-cp315-cp315-win32.whl", hash = "sha256:035cd71e2aadfc0ad366ab4785b49e62093370885ea4b527b61b308b7a92ff14", size = 101099, upload-time = "2026-10-29T23:06:01.777Z" }, + { url = "https://files.pythonhosted.org/packages/7c/e6/fdea9e35605a6e7fc71556068ea184fb3e9cc09d1e3be8ca8f26d16339c9/wrapt-2.5.2-cp315-cp315-win_amd64.whl", hash = "sha256:b9f41029f9bc6ba80755b9c07f5e77e9e7aafb71c625b9f67614e4ebfb24302a", size = 106423, upload-time = "2026-10-29T23:06:03.076Z" }, + { url = "https://files.pythonhosted.org/packages/d5/50/8c41d8ba868cde9afd5b68e18f19b94300d091c409ef081b80c2aecda653/wrapt-2.5.2-cp315-cp315-win_arm64.whl", hash = "sha256:ae84013e89ec1874415549a46cfea373e28ddecbb72e1f5456cddfa2b76a28e6", size = 104152, upload-time = "2026-10-29T23:06:04.356Z" }, + { url = "https://files.pythonhosted.org/packages/8a/e7/b43d341b036e2862550eb7fffe2b4130015c39c861f30eb34014a2f032d2/wrapt-2.5.2-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:b680bef5200899524667f813fa34dcfde9313fef9f9fa432985d4bc7b4195ba3", size = 109711, upload-time = "2026-10-29T23:06:05.768Z" }, + { url = "https://files.pythonhosted.org/packages/be/52/8012aaf046264e27969f79bc94cd2afb4729338f5434b90edd3d4bb60b8f/wrapt-2.5.2-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:da51f5ac290f2e024ee842799c6592f68cd880c20f6b3a54305d1b6aeab459a4", size = 110047, upload-time = "2026-10-29T23:06:07.309Z" }, + { url = "https://files.pythonhosted.org/packages/54/a7/628c315d19c6027e8b0e700c0dd7d5c3fd662329c7aa1294315064b87cdd/wrapt-2.5.2-cp315-cp315t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:b3ebc07822649eda442382c1d20983bc6ec5aa2bf79a600da11b29ecb62a0fb2", size = 285041, upload-time = "2026-10-29T23:06:08.777Z" }, + { url = "https://files.pythonhosted.org/packages/7a/fd/4b418c9655bacb5ce8b0aed894f8355e12940bde74237ec45d9396e1f87f/wrapt-2.5.2-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bbb6e959c1e2c32bbad291f83af6320c8c46bb64e256801e85f55d88a09522d5", size = 295329, upload-time = "2026-10-29T23:06:10.489Z" }, + { url = "https://files.pythonhosted.org/packages/92/24/dbe9776fded6dd5f2bfa1446ec04a33afe5afac1d5fb89cd7f5e96fbc61d/wrapt-2.5.2-cp315-cp315t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a0541e62f7b9c0d42130e3469d6ac3b1767f019314ae795646a2850a75c64412", size = 265088, upload-time = "2026-10-29T23:06:11.963Z" }, + { url = "https://files.pythonhosted.org/packages/9d/ee/4103576d31bed902224fed8b1d06b94313491504d1d644e42bceba4de880/wrapt-2.5.2-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:18819c05409a161999be68da8e451636470c35f77e25c8201825a33a7d88c89c", size = 290413, upload-time = "2026-10-29T23:06:13.536Z" }, + { url = "https://files.pythonhosted.org/packages/bc/7c/944e95c25232a5805ea9f25cf2ce90386262db025ba04fbb61f451d9523f/wrapt-2.5.2-cp315-cp315t-musllinux_1_2_riscv64.whl", hash = "sha256:2594e0be4ee09b2f19828265a39df90e591def44215fe8c5b8d6b83aa4cc18b5", size = 262383, upload-time = "2026-10-29T23:06:14.887Z" }, + { url = "https://files.pythonhosted.org/packages/5d/b3/a4b66497e3bb63d2e94ea1605f4beaf5b6b5dafbb6ce41abd727a6fcb7b5/wrapt-2.5.2-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:32c0387872e67b9dfdd2b599bf31fe127e597ed8526c5f192cdaca57a491e9dc", size = 280180, upload-time = "2026-10-29T23:06:16.314Z" }, + { url = "https://files.pythonhosted.org/packages/91/bd/28d046dccfcade6ad62ad19f2f805e5b2e3f3463431307c3b8bd54e24e7f/wrapt-2.5.2-cp315-cp315t-win32.whl", hash = "sha256:9cb33f293e3aa04b36f7d165dc05b42db85e0a88e20a8fd81f60426d1f17d535", size = 103551, upload-time = "2026-10-29T23:06:17.732Z" }, + { url = "https://files.pythonhosted.org/packages/0d/d9/aef3035a3e3d918eb51e7db454af9e06919f0c5208fa0a9315548fef7734/wrapt-2.5.2-cp315-cp315t-win_amd64.whl", hash = "sha256:7f89423965b23aa62a90bf21c20353056a829ca9a60c01fde9aec9876c99392e", size = 110099, upload-time = "2026-10-29T23:06:19.293Z" }, + { url = "https://files.pythonhosted.org/packages/5f/ea/56eddba6249fae5361b53eb62e2e25ad86cabec4632d943d356aba8dea1b/wrapt-2.5.2-cp315-cp315t-win_arm64.whl", hash = "sha256:fa642d3781c061579cb138773add4a55b16f99de9ebc41603e89a756cf9e62c6", size = 106397, upload-time = "2026-10-29T23:06:20.826Z" }, + { url = "https://files.pythonhosted.org/packages/77/8b/1ea57b936b70f98e8e7620a352578096d778dbee95b262f73c066d210ffb/wrapt-2.5.2-py3-none-any.whl", hash = "sha256:61eb54e9540ea522c2c4752a57549a83d3c67e6704ed708403d28e067aaf90f0", size = 82025, upload-time = "2026-10-29T23:06:38.56Z" }, +] diff --git a/version.sh b/version.sh deleted file mode 100755 index 011e23a9..00000000 --- a/version.sh +++ /dev/null @@ -1,94 +0,0 @@ -#!/bin/sh -# Tag revisions like this: -# $ git tag -a -m "v0.2" v0.2 -# Supports also pre-/post-/dev-releases and local version identifiers: -# $ git tag -a -m "v0.2rc3.post5.dev7+local.1" v0.2rc3.post5.dev7+local.1 -# Enforcing canonical-like pattern with local version specifier: -# N.N(.N)?[{a|b|rc}N][.postN][.devN][+{a..z0..9}(.{a..z0..9})*] -# See https://packaging.python.org/en/latest/specifications/version-specifiers -VERSION_IN=VERSION.in -VERSION_FILE=Exscript/version.py -REGEX_LOCALSPEC='(\+[a-z0-9]+(\.[a-z0-9]+)?)' -REGEX_CANONICAL='((0|[1-9][0-9]*)\.(0|[1-9][0-9]*))(\.(0|[1-9][0-9]*))?((a|b|rc)(0|[1-9][0-9]*))?(\.post(0|[1-9][0-9]*))?(\.dev(0|[1-9][0-9]*))?'"${REGEX_LOCALSPEC}"'?' -REGEX_DESCRIBE='(\-([1-9][0-9]*)\-g[a-f0-9]+)' -ZERO_DESCRIBE='-0-g0000000' - -# Check that we are actually in a git managed project. -if [ ! -e .git ] && [ -z "$1" ]; then - echo >&2 Not a git repository. - exit 1 -fi - -# Make sure that we have permission to modify the version file. -if [ -r "${VERSION_FILE}" ] && [ ! -w "${VERSION_FILE}" ]; then - echo >&2 No permission to modify ${VERSION_FILE}. - exit 1 -fi - -# If the --reset switch was given, reset the version number to 'DEVELOPMENT'. -if [ "$1" = "--reset" ]; then - VERSION='DEVELOPMENT' -else - # Use static version or get the version number from "git describe". - if [ -n "$1" ]; then - VERSION=$1 - else - HEAD=$(git log -1 --pretty=format:%H HEAD) - VERSION=$(git describe "${HEAD}" --tags --match "v[0-9]*") - fi - - # Remove leading 'v' as it is not allowed for canonical versioning. - VERSION=$(echo "${VERSION}" | sed -E 's/^v//') - - # Set implicit patch version to '.0' to always have a 'major.minor.patch' schema. - VERSION=$(echo "${VERSION}" | sed -E 's/^([0-9]+\.[0-9]+)(\.?[a-z]|\+[a-z0-9]|$)/\1.0\2/') - - # Extract delta count of commits from possible "describe" suffix: -3-g454be62 - # Afterwards remove the "describe" suffix from version. - DELTA=$(echo "${VERSION}" | (grep -Eio '('"${REGEX_DESCRIBE}"'$)' || echo "${ZERO_DESCRIBE}") | cut -f2 -d'-') - VERSION=$(echo "${VERSION}" | sed -E 's/'"${REGEX_DESCRIBE}"'$//') - - # Validations. - if [ -z "${VERSION}" ]; then - echo >&2 No matching tag was found. - exit 1 - elif ! (echo "${VERSION}" | grep -Eiq '^'"${REGEX_CANONICAL}"'$'); then - echo >&2 "Version '${VERSION}' does not match canonical-like pattern 'N.N(.N)?[{a|b|rc}N][.postN][.devN][+{a..z0..9}(.{a..z0..9})*]'." - echo >&2 "See also: https://packaging.python.org/en/latest/specifications/version-specifiers" - exit 1 - fi - - # Extract local version specifier from version as his trailing number is optional. - LOCALSPEC=$(echo "${VERSION}" | grep -Eio '('"${REGEX_LOCALSPEC}"'$)') - VERSION=$(echo "${VERSION}" | sed -E 's/'"${REGEX_LOCALSPEC}"'$//') - - # Increment last number by commit delta. - if [ "${LOCALSPEC}" ] && (echo "${LOCALSPEC}" | grep -Eiq '[0-9]$'); then - INCREASED=$(( $(echo "${LOCALSPEC}" | grep -Eio '[0-9]+$') + DELTA )) - LOCALSPEC=$(echo "${LOCALSPEC}" | sed -E 's/[0-9]+$/'"${INCREASED}"'/') - else - INCREASED=$(( $(echo "${VERSION}" | grep -Eio '[0-9]+$') + DELTA )) - VERSION=$(echo "${VERSION}" | sed -E 's/[0-9]+$/'"${INCREASED}"'/') - fi - - # Restore full version string. - VERSION="${VERSION}${LOCALSPEC}" -fi - -# If there is no version file, we are already done. -echo Version is "${VERSION}" -[ ! -r "${VERSION_FILE}" ] && exit 0 - -# Check whether the version file already contains this number, -# and only touch it if there is a change to avoid changing -# the timestamp. -VERSION_FILE_TMP=$(mktemp) -cat "${VERSION_IN}" | sed "s/@VERSION@/${VERSION}/g" > "${VERSION_FILE_TMP}" -if diff -q "${VERSION_FILE_TMP}" "${VERSION_FILE}"; then - echo Version file unchanged. - rm "${VERSION_FILE_TMP}" - exit 0 -fi - -mv "${VERSION_FILE_TMP}" "${VERSION_FILE}" -echo Version file updated. From c7baea3725f2643d6bd9dc477451d6219afcf7b6 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Fri, 30 Oct 2026 18:15:26 +0100 Subject: [PATCH 005/250] Port to Python 3.15 and annotate for a green mypy baseline util.file.load_lib() uses exec_module(), as Python 3.15 removed load_module(). --- src/Exscript/key.py | 2 +- src/Exscript/logger.py | 6 +- src/Exscript/protocols/drivers/__init__.py | 6 +- src/Exscript/protocols/protocol.py | 257 +++++++-------------- src/Exscript/util/file.py | 9 +- src/Exscript/util/log.py | 2 +- src/Exscript/workqueue/mainloop.py | 2 +- tests/Exscript/workqueue/JobTest.py | 4 +- 8 files changed, 99 insertions(+), 189 deletions(-) diff --git a/src/Exscript/key.py b/src/Exscript/key.py index a9a73443..47e99c97 100644 --- a/src/Exscript/key.py +++ b/src/Exscript/key.py @@ -33,7 +33,7 @@ class PrivateKey: Represents a cryptographic key, and may be used to authenticate using :class:`Exscript.protocols`. """ - keytypes = set() + keytypes: set[str] = set() def __init__(self, keytype='rsa'): """ diff --git a/src/Exscript/logger.py b/src/Exscript/logger.py index 77281e00..d44ae8f0 100644 --- a/src/Exscript/logger.py +++ b/src/Exscript/logger.py @@ -117,7 +117,7 @@ def filter(self, record): # Exscript Logging -logger_registry = weakref.WeakValueDictionary() # Map id(logger) to Logger. +logger_registry: 'weakref.WeakValueDictionary[int, Logger]' = weakref.WeakValueDictionary() # Map id(logger) to Logger. class Log: @@ -243,8 +243,8 @@ def succeeded(self): class Logger: """ - A QueueListener that implements logging for the queue. - Logs are kept in memory, and not written to the disk. + Implements logging for the queue: keeps a log of each attempt of each + job. Logs are kept in memory, and not written to the disk. """ def __init__(self): diff --git a/src/Exscript/protocols/drivers/__init__.py b/src/Exscript/protocols/drivers/__init__.py index ce19cabf..a170b9dc 100644 --- a/src/Exscript/protocols/drivers/__init__.py +++ b/src/Exscript/protocols/drivers/__init__.py @@ -58,9 +58,9 @@ from .zte import ZteDriver as ZteDriver from .zyxel import ZyxelDriver as ZyxelDriver -driver_classes = [] -drivers = [] -driver_map = {} +driver_classes: list[type[Driver]] = [] +drivers: list[Driver] = [] +driver_map: dict[str, Driver] = {} def isdriver(o): diff --git a/src/Exscript/protocols/protocol.py b/src/Exscript/protocols/protocol.py index de12cf0a..e34de863 100644 --- a/src/Exscript/protocols/protocol.py +++ b/src/Exscript/protocols/protocol.py @@ -36,6 +36,7 @@ import time from functools import partial from io import StringIO +from typing import IO from ..util.buffer import MonitoredBuffer from ..util.cast import to_regexs @@ -75,7 +76,7 @@ _skey_re = re.compile(r'(?:s\/key|otp-md4) (\d+) (\S+)(?=\s|[\r\n])') # Needed on module level as each host connection has its own thread. -_connection_monitor = {} +_connection_monitor: dict[str, float] = {} _connection_monitor_lock = threading.RLock() # NOTE: Encoding/Decoding non-ASCII byte-streams might not work properly. @@ -92,168 +93,67 @@ class Protocol: This is the base class for all protocols; it defines the common portions of the API. - The goal of all protocol classes is to provide an interface that - is unified across protocols, such that the adapters may be used - interchangeably without changing any other code. - - In order to achieve this, the main challenge are the differences - arising from the authentication methods that are used. The reason - is that many devices may support the following variety of - authentication/authorization methods: - - 1. Protocol level authentication, such as SSH's built-in - authentication. - - - p1: password only - - p2: username - - p3: username + password - - p4: username + key - - p5: username + key + password - - 2. App level authentication, such that the authentication may - happen long after a connection is already accepted. - This type of authentication is normally used in combination with - Telnet, but some SSH hosts also do this (users have reported - devices from Enterasys). These devices may also combine - protocol-level authentication with app-level authentication. - The following types of app-level authentication exist: - - - a1: password only - - a2: username - - a3: username + password - - 3. App level authorization: In order to implement the AAA protocol, - some devices ask for two separate app-level logins, whereas the - first serves to authenticate the user, and the second serves to - authorize him. App-level authorization may support the same - methods as app-level authentication: - - - A1: password only - - A2: username - - A3: username + password - - We are assuming that the following methods are used: - - - Telnet: - - - p1 - p5: never - - a1 - a3: optional - - A1 - A3: optional - - - SSH: - - - p1 - p5: optional - - a1 - a3: optional - - A1 - A3: optional - - - Netconf: - - - p1 - p5: optional (derived from SSH) - - a1 - a3: never (not supported) - - A1 - A3: never (not supported) - - To achieve authentication method compatibility across different - protocols, we must hide all this complexity behind one single API - call, and figure out which ones are supported. - - As a use-case, our goal is that the following code will always work, - regardless of which combination of authentication methods a device - supports:: - - key = PrivateKey.from_file('~/.ssh/id_rsa', 'my_key_password') - - # The user account to use for protocol level authentication. - # The key defaults to None, in which case key authentication is - # not attempted. - account = Account(name = 'myuser', - password = 'mypassword', - key = key) - - # The account to use for app-level authentication. - # password2 defaults to password. - app_account = Account(name = 'myuser', - password = 'my_app_password', - password2 = 'my_app_password2') - - # app_account defaults to account. - conn.login(account, app_account = None, flush = True) - - Another important consideration is that once the login is complete, the - device must be in a clearly defined state, i.e. we need to - have processed the data that was retrieved from the connected host. - - More precisely, the buffer that contains the incoming data must be in - a state such that the following call to expect_prompt() will either - always work, or always fail. - - We hide the following methods behind the login() call:: - - # Protocol level authentication. - conn.protocol_authenticate(...) - - # App-level authentication. - conn.app_authenticate(...) - - # App-level authorization. - conn.app_authorize(...) via conn.auto_app_authorize(...) - - The code produces the following result:: - - Telnet: - conn.protocol_authenticate -> NOOP - conn.app_authenticate - -> waits for username or password prompt, authenticates, - returns after a CLI prompt was seen. - conn.app_authorize (via conn.auto_app_authorize) - -> delegates to driver.auto_authorize - -> calls specific implementation or conn.app_authorize - -> waits for username or password prompt, authorizes, - returns after a CLI prompt was seen. - - SSH: - conn.protocol_authenticate -> authenticates using user/key/password - conn.app_authenticate -> like Telnet - conn.app_authorize -> like Telnet - - Netconf: - conn.protocol_authenticate -> derived from SSH - conn.app_authenticate -> NOOP (not supported) - conn.app_authorize -> NOOP (not supported) - - We can see the following: - - - protocol_authenticate() must not wait for a prompt, because else - app_authenticate() has no way of knowing whether an app-level - login is even necessary. - - - app_authenticate() must check the buffer first, to see if - authentication has already succeeded. In the case that - app_authenticate() is not necessary (i.e. the buffer contains a - CLI prompt), it just returns. - - app_authenticate() must NOT eat the prompt from the buffer, because - else the result may be inconsistent with devices that do not do - any authentication; i.e., when app_authenticate() is not called. - - - Since the prompt must still be contained in the buffer, - conn.driver.auto_authorize() would somehow need to eat it before it - sends the command for starting the authorization procedure. This - will be handled internally by login(), but only if the call to - Driver.supports_auto_authorize() evaluates to `True` for a specific - driver. Otherwise the driver must fall back to app_authorize(), which - consumes the prompt internally. - - If the user wants to handle app authorization manually, one may rely - on authenticate() instead of login(), which does not authorize - automatically. - - - Once all logins are processed, it makes sense to eat the prompt - depending on the `flush` parameter. It should default to True, - because it's better that the connection stalls waiting forever, - than to risk that an error is not immediately discovered due to - timing issues (this is a race condition that is not detailed here). + A protocol adapter connects to a host and provides this API for talking to + it, the same with every protocol, so the adapters can be used + interchangeably: :class:`~Exscript.protocols.telnet.Telnet`, + :class:`~Exscript.protocols.ssh2.SSH2`, + :class:`~Exscript.protocols.netconf.Netconf`, and + :class:`~Exscript.protocols.dummy.Dummy`, which talks to a virtual device + instead of a host. + + A login has up to three levels: + + 1. Protocol-level authentication: the protocol authenticates the user when + the connection opens, e.g. SSH with a password or a key + (:meth:`protocol_authenticate`). + 2. Application-level authentication: after the connection was accepted, + the device asks for a user name, a password or both + (:meth:`app_authenticate`). + 3. Authorization: to grant privileges, e.g. after "enable", the device + asks for the authorization password of the account + (:meth:`app_authorize`, or :meth:`auto_app_authorize`, which lets the + driver send such a command first). + + The protocols use them as follows: + + - Telnet and Dummy: no protocol-level authentication; the device may ask + for an application-level authentication and an authorization. + - SSH: the protocol-level authentication; the device may also ask for an + application-level authentication and an authorization. + - Netconf: only the protocol-level authentication. + + :meth:`login` performs all levels: it calls :meth:`authenticate`, which + performs the protocol-level authentication, once per connection, and the + application-level authentication, and then :meth:`auto_app_authorize`. + :meth:`authenticate` alone skips the authorization, e.g. to authorize + manually with :meth:`app_authorize`. The methods of the single levels can + be called separately, too: :meth:`protocol_authenticate` waits for no + prompt, and :meth:`app_authenticate` sends nothing if the device shows + its command prompt at once. With flush=True, the default, :meth:`login` + consumes the command prompt after the login from the buffer. + + Example:: + + from Exscript import Account, PrivateKey + from Exscript.protocols import SSH2 + + key = PrivateKey.from_file('/home/myuser/.ssh/id_rsa', 'my_key_password') + + # The account for the protocol-level authentication; SSH uses only the + # key if the account has one. + account = Account('myuser', 'mypassword', key=key) + + # The account for the application-level authentication and the + # authorization; password2 defaults to the password. + app_account = Account('myuser', 'my_app_password', password2='my_app_password2') + + conn = SSH2() + conn.connect('myhost') + conn.login(account, app_account) # app_account defaults to account """ + _log: IO[str] | None # managed by the logfile property + def __init__(self, driver=None, stdout=None, @@ -286,8 +186,9 @@ def __init__(self, an in-memory buffer. :keyword stderr: Where to write debug info. Defaults to stderr. :keyword debug: An integer between 0 (no debugging) and 5 (very - verbose debugging) that specifies the amount of debug info - sent to the terminal. The default value is 0. + verbose debugging) that specifies the amount of debug output, + which the loggers under Exscript.protocols log at the DEBUG + level. The default value is 0. :keyword connect_rate_limit: Connection rate-limit per minute to avoid hitting rate limits of the host in case of multiple parallel connections. :keyword connect_timeout: Timeout for the initial TCP connection attempt @@ -438,9 +339,10 @@ def logfile(self): @logfile.setter def logfile(self, value) -> None: with self._write_lock: - if getattr(self, '_log', None): + old_log = getattr(self, '_log', None) + if old_log: try: - self._log.close() + old_log.close() except Exception: pass self._logfile = value @@ -484,8 +386,9 @@ def stdout(self, value): @property def stderr(self): """ - The stream to which the debug/error output is written. Defaults to - `sys.stderr`. + The error stream of the adapter, `sys.stderr` by default. The adapter + writes nothing to it: its debug output goes to the logging module + (see the debug argument of the constructor). :rtype: file-like object :return: The current stderr stream. @@ -957,7 +860,8 @@ def _enforce_connect_rate_limit(self, rate_limit=0): Uses `self.connect_rate_limit`, if the given rate-limit is zero. :type rate_limit: int - :param rate_limit: The connection rate-limit (connects per minute). + :param rate_limit: The connection rate-limit (connects per minute); + 0 uses connect_rate_limit. """ key = self.host.lower() wait = 0.0 @@ -1045,10 +949,11 @@ def _get_account(self, account): def login(self, account=None, app_account=None, flush=True): """ Log into the connected host using the best method available. - If an account is not given, default to the account that was - used during the last call to login(). If a previous call was not - made, use the account-factory that was passed to the constructor. - If that also fails, raise a TypeError. + If no account is given, the account of the last call of login() or + of an authentication method is used. If the constructor got an + account_factory, it provides the account to use: it gets the given + account, else the one of the last call, or None. Without an account + and without an account factory, a TypeError is raised. The app_account is passed to :meth:`app_authenticate` and :meth:`app_authorize`. If app_account is not given, default @@ -1399,8 +1304,7 @@ def execute(self, command, consume=True): Netconf does not have prompts, so this attempts to use the Netconf 1.0 EOM marker `]]>]]>` or the Netconf 1.1 EOC marker `LF HASH HASH LF` as a prompt-like delimiter. If that fails, a custom prompt may also be - defined using the set_prompt() method, while for Netconf as RFC 6142 is - quite strict on the framing behavior, the need might never arise. + defined using the set_prompt() method. This method also modifies the value of the response (self.response) attribute. For Netconf this refers to the raw RPC reply. Please use the @@ -1523,8 +1427,7 @@ def expect_prompt(self, consume=True): Netconf does not have prompts, so this attempts to use the Netconf 1.0 EOM marker `]]>]]>` or the Netconf 1.1 EOC marker `LF HASH HASH LF` as a prompt-like delimiter. If that fails, a custom prompt may also be - defined using the set_prompt() method, while for Netconf as RFC 6142 is - quite strict on the framing behavior, the need might never arise. + defined using the set_prompt() method. This method also stores the received data in the response (self.response) attribute. For Netconf this refers to the raw RPC reply. Please use the @@ -1534,7 +1437,7 @@ def expect_prompt(self, consume=True): :type consume: boolean :param consume: Whether to consume the response and prompt from the - buffer or not. The later allows consecutive calls to succeed. + buffer or not. The latter allows consecutive calls to succeed. :rtype: int, re.MatchObject :return: The index of the prompt regular expression that matched, and the match object. diff --git a/src/Exscript/util/file.py b/src/Exscript/util/file.py index c3f51493..a8ffa07a 100644 --- a/src/Exscript/util/file.py +++ b/src/Exscript/util/file.py @@ -26,7 +26,8 @@ import base64 import binascii import configparser -import importlib +import importlib.machinery +import importlib.util import os import re @@ -226,6 +227,10 @@ def load_lib(filename): raise IOError('No such file: %s' % filename) name = os.path.splitext(os.path.basename(filename))[0] - module = importlib.machinery.SourceFileLoader(name, filename).load_module() + # load_module() was removed in Python 3.15; an explicit loader also accepts other suffixes. + loader = importlib.machinery.SourceFileLoader(name, filename) + spec = importlib.util.spec_from_file_location(name, filename, loader=loader) + module = importlib.util.module_from_spec(spec) + loader.exec_module(module) return dict((name + '.' + k, v) for (k, v) in list(module.__lib__.items())) diff --git a/src/Exscript/util/log.py b/src/Exscript/util/log.py index 4088e44a..fd7042e8 100644 --- a/src/Exscript/util/log.py +++ b/src/Exscript/util/log.py @@ -26,7 +26,7 @@ from .. import FileLogger from .impl import add_label -_loggers = [] +_loggers: list[FileLogger] = [] def log_to(logger): diff --git a/src/Exscript/workqueue/mainloop.py b/src/Exscript/workqueue/mainloop.py index be95bbf7..cb51ba38 100644 --- a/src/Exscript/workqueue/mainloop.py +++ b/src/Exscript/workqueue/mainloop.py @@ -31,7 +31,7 @@ _logger.addHandler(logging.NullHandler()) # See http://bugs.python.org/issue1731717 -multiprocessing.process._cleanup = lambda: None +multiprocessing.process._cleanup = lambda: None # type: ignore[attr-defined] class MainLoop(threading.Thread): diff --git a/tests/Exscript/workqueue/JobTest.py b/tests/Exscript/workqueue/JobTest.py index 4f5ca020..beae57d4 100644 --- a/tests/Exscript/workqueue/JobTest.py +++ b/tests/Exscript/workqueue/JobTest.py @@ -1,7 +1,7 @@ import unittest -from multiprocessing import Pipe +from multiprocessing import Pipe, get_all_start_methods from pickle import dumps, loads from Exscript.workqueue.job import Job, Process, Thread @@ -32,6 +32,8 @@ def testStart(self): pass # See testRun() +@unittest.skipIf(get_all_start_methods()[0] != 'fork', + 'Process jobs need the "fork" start method (the default only on Linux < 3.14)') class ProcessTest(ThreadTest): CORRELATE = Process From 68bb2d8fa6fc32b6bf1f0c590eb810e9db028142 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Fri, 30 Oct 2026 18:16:58 +0100 Subject: [PATCH 006/250] Keep test data byte-identical across platforms Banners and expected template outputs contain intentional CRLF and mixed line endings; exclude them from line-ending conversion. --- .gitattributes | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitattributes b/.gitattributes index d0d6371d..a2282ca7 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,2 +1,5 @@ * text=auto eol=lf *.png binary +# Device captures and expected outputs must stay byte-identical (CRLF is intentional). +tests/Exscript/protocols/banners/** -text +tests/templates/** -text From 466247c9eb884d2da1b9900bff82a6fa464c76a1 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Fri, 30 Oct 2026 18:21:13 +0100 Subject: [PATCH 007/250] Format the code base with ruff Mechanical, behavior-neutral formatting with line length 100 and the existing quote style. --- README.md | 2 + demos/multiple/multiple.py | 1 + demos/quickstart/quickstart.py | 1 + demos/report/report.py | 5 +- demos/template/template.py | 1 + pyproject.toml | 18 + scripts/exscript | 1 + src/Exscript/__init__.py | 8 +- src/Exscript/account.py | 21 +- src/Exscript/contrib/otel.py | 47 +- src/Exscript/emulators/__init__.py | 8 +- src/Exscript/emulators/command.py | 2 +- src/Exscript/emulators/iosemu.py | 72 +- src/Exscript/emulators/vdevice.py | 25 +- src/Exscript/host.py | 35 +- src/Exscript/interpreter/__init__.py | 7 +- src/Exscript/interpreter/append.py | 1 - src/Exscript/interpreter/assign.py | 1 - src/Exscript/interpreter/code.py | 87 +- src/Exscript/interpreter/enter.py | 1 - src/Exscript/interpreter/exception.py | 4 +- src/Exscript/interpreter/execute.py | 16 +- src/Exscript/interpreter/expression.py | 22 +- src/Exscript/interpreter/extract.py | 1 - src/Exscript/interpreter/fail.py | 1 - src/Exscript/interpreter/functioncall.py | 13 +- src/Exscript/interpreter/ifcondition.py | 1 - src/Exscript/interpreter/loop.py | 9 +- src/Exscript/interpreter/number.py | 2 +- src/Exscript/interpreter/parser.py | 9 +- src/Exscript/interpreter/program.py | 1 - src/Exscript/interpreter/regex.py | 3 +- src/Exscript/interpreter/rpc.py | 15 +- src/Exscript/interpreter/scope.py | 4 +- src/Exscript/interpreter/string.py | 37 +- src/Exscript/interpreter/template.py | 45 +- src/Exscript/interpreter/term.py | 1 - src/Exscript/interpreter/trying.py | 1 - src/Exscript/interpreter/variable.py | 1 - src/Exscript/key.py | 3 +- src/Exscript/logger.py | 17 +- src/Exscript/parselib/__init__.py | 7 +- src/Exscript/parselib/exception.py | 10 +- src/Exscript/parselib/lexer.py | 3 +- src/Exscript/parselib/token.py | 4 +- src/Exscript/protocols/__init__.py | 14 +- src/Exscript/protocols/drivers/__init__.py | 3 + src/Exscript/protocols/drivers/ace.py | 2 +- src/Exscript/protocols/drivers/adtran.py | 2 +- src/Exscript/protocols/drivers/aironet.py | 2 +- src/Exscript/protocols/drivers/aix.py | 2 +- .../protocols/drivers/arbor_peakflow.py | 4 +- src/Exscript/protocols/drivers/aruba.py | 2 +- src/Exscript/protocols/drivers/bigip.py | 2 +- src/Exscript/protocols/drivers/brocade.py | 1 - src/Exscript/protocols/drivers/cienasaos.py | 2 +- src/Exscript/protocols/drivers/driver.py | 65 +- src/Exscript/protocols/drivers/enterasys.py | 2 +- .../protocols/drivers/enterasys_wc.py | 2 +- src/Exscript/protocols/drivers/eos.py | 5 +- .../protocols/drivers/ericsson_ban.py | 2 +- src/Exscript/protocols/drivers/fortios.py | 2 +- src/Exscript/protocols/drivers/generic.py | 2 +- .../protocols/drivers/hp_pro_curve.py | 2 +- src/Exscript/protocols/drivers/icotera.py | 2 +- src/Exscript/protocols/drivers/ios.py | 2 +- src/Exscript/protocols/drivers/ios_xr.py | 2 +- src/Exscript/protocols/drivers/isam.py | 2 +- src/Exscript/protocols/drivers/junos.py | 2 +- src/Exscript/protocols/drivers/junos_erx.py | 2 +- src/Exscript/protocols/drivers/mrv.py | 2 +- src/Exscript/protocols/drivers/nxos.py | 2 +- src/Exscript/protocols/drivers/one_os.py | 2 +- src/Exscript/protocols/drivers/rios.py | 2 +- src/Exscript/protocols/drivers/shell.py | 2 +- .../protocols/drivers/smart_edge_os.py | 2 +- src/Exscript/protocols/drivers/sp_vxoa.py | 2 +- src/Exscript/protocols/drivers/sros.py | 2 +- src/Exscript/protocols/drivers/vrp.py | 2 +- src/Exscript/protocols/drivers/vxworks.py | 2 +- src/Exscript/protocols/drivers/zte.py | 2 +- src/Exscript/protocols/drivers/zyxel.py | 4 +- src/Exscript/protocols/dummy.py | 1 - src/Exscript/protocols/exception.py | 17 +- src/Exscript/protocols/netconf.py | 18 +- src/Exscript/protocols/osguesser.py | 15 +- src/Exscript/protocols/protocol.py | 122 +- src/Exscript/protocols/ssh2.py | 47 +- src/Exscript/protocols/telnet.py | 5 +- src/Exscript/protocols/telnetlib.py | 47 +- src/Exscript/queue.py | 62 +- src/Exscript/servers/__init__.py | 8 +- src/Exscript/servers/httpd.py | 37 +- src/Exscript/servers/server.py | 8 +- src/Exscript/servers/sshd.py | 13 +- src/Exscript/servers/telnetd.py | 2 +- src/Exscript/stdlib/__init__.py | 100 +- src/Exscript/stdlib/list.py | 3 + src/Exscript/stdlib/mysys.py | 10 +- src/Exscript/stdlib/string.py | 1 + src/Exscript/stdlib/util.py | 2 + src/Exscript/util/__init__.py | 8 +- src/Exscript/util/buffer.py | 4 +- src/Exscript/util/cast.py | 4 +- src/Exscript/util/collections.py | 13 +- src/Exscript/util/crypt.py | 2319 +++++++++++++++-- src/Exscript/util/daemonize.py | 1 + src/Exscript/util/decorator.py | 8 + src/Exscript/util/event.py | 2 +- src/Exscript/util/file.py | 14 +- src/Exscript/util/impl.py | 15 +- src/Exscript/util/interact.py | 16 +- src/Exscript/util/ip.py | 1 + src/Exscript/util/ipv4.py | 3 +- src/Exscript/util/log.py | 2 + src/Exscript/util/mail.py | 17 +- src/Exscript/util/match.py | 1 + src/Exscript/util/pidutil.py | 1 + src/Exscript/util/report.py | 5 +- src/Exscript/util/sigint.py | 4 +- src/Exscript/util/sigintcatcher.py | 2 + src/Exscript/util/start.py | 1 + src/Exscript/util/syslog.py | 15 +- src/Exscript/util/template.py | 3 +- src/Exscript/util/tty.py | 4 +- src/Exscript/util/url.py | 12 +- src/Exscript/util/weakmethod.py | 5 +- src/Exscript/util/xml.py | 1 + src/Exscript/version.py | 1 + src/Exscript/workqueue/__init__.py | 7 +- src/Exscript/workqueue/job.py | 13 +- src/Exscript/workqueue/mainloop.py | 10 +- src/Exscript/workqueue/pipeline.py | 5 +- src/Exscript/workqueue/task.py | 2 +- src/Exscript/workqueue/workqueue.py | 38 +- tests/Exscript/AccountManagerTest.py | 2 + tests/Exscript/AccountPoolTest.py | 5 +- tests/Exscript/AccountTest.py | 19 +- tests/Exscript/FileLoggerTest.py | 5 +- tests/Exscript/HostTest.py | 48 +- tests/Exscript/KnownIssuesTest.py | 38 +- tests/Exscript/LexerTest.py | 3 +- tests/Exscript/LogTest.py | 2 + tests/Exscript/LogfileTest.py | 2 + tests/Exscript/LoggerTest.py | 2 + tests/Exscript/QueueTest.py | 78 +- tests/Exscript/TemplateTest.py | 11 +- tests/Exscript/emulators/CommandSetTest.py | 3 + tests/Exscript/emulators/IOSEmulatorTest.py | 10 +- tests/Exscript/emulators/VirtualDeviceTest.py | 20 +- tests/Exscript/emulators/run_suite.py | 10 +- tests/Exscript/protocols/DummyTest.py | 36 +- .../Exscript/protocols/OsGuesserCorpusTest.py | 61 +- tests/Exscript/protocols/OsGuesserTest.py | 42 +- tests/Exscript/protocols/ProtocolTest.py | 151 +- tests/Exscript/protocols/SSH2Test.py | 2 + tests/Exscript/protocols/TelnetTest.py | 2 + tests/Exscript/protocols/pseudodev.py | 15 +- tests/Exscript/protocols/run_suite.py | 10 +- tests/Exscript/run_suite.py | 10 +- tests/Exscript/servers/SSHdTest.py | 2 + tests/Exscript/servers/ServerTest.py | 8 +- tests/Exscript/servers/TelnetdTest.py | 2 + tests/Exscript/servers/run_suite.py | 10 +- tests/Exscript/util/bufferTest.py | 3 + tests/Exscript/util/castTest.py | 15 +- tests/Exscript/util/cryptTest.py | 3 + tests/Exscript/util/decoratorTest.py | 27 +- tests/Exscript/util/eventTest.py | 17 +- tests/Exscript/util/fileTest.py | 18 +- tests/Exscript/util/interactTest.py | 2 + tests/Exscript/util/ipTest.py | 8 +- tests/Exscript/util/ipv4Test.py | 73 +- tests/Exscript/util/ipv6Test.py | 20 +- tests/Exscript/util/mailTest.py | 7 +- tests/Exscript/util/matchTest.py | 5 +- tests/Exscript/util/reportTest.py | 20 +- tests/Exscript/util/run_suite.py | 10 +- tests/Exscript/util/startTest.py | 10 +- tests/Exscript/util/syslogTest.py | 2 + tests/Exscript/util/ttyTest.py | 8 +- tests/Exscript/util/urlTest.py | 123 +- tests/Exscript/util/weakmethodTest.py | 4 +- tests/Exscript/workqueue/JobTest.py | 9 +- tests/Exscript/workqueue/MainLoopTest.py | 2 + tests/Exscript/workqueue/PipelineTest.py | 16 +- tests/Exscript/workqueue/TaskTest.py | 2 + tests/Exscript/workqueue/WorkQueueTest.py | 6 +- tests/Exscript/workqueue/run_suite.py | 10 +- tests/conftest.py | 8 +- tests/templates/builtinvars/pseudodev.py | 3 +- tests/templates/escaping/pseudodev.py | 4 +- tests/templates/netconf/pseudodev.py | 8 +- .../templates/stdlib.connection/pseudodev.py | 3 +- tests/templates/stdlib.crypt/pseudodev.py | 3 +- tests/templates/stdlib.list/pseudodev.py | 3 +- tests/templates/stdlib.string/pseudodev.py | 3 +- tests/templates/syntax/pseudodev.py | 10 +- tests/templates/whitespace/pseudodev.py | 10 +- tools/bench_bulk.py | 22 +- 200 files changed, 3415 insertions(+), 1496 deletions(-) diff --git a/README.md b/README.md index 8bdf49f5..8b57b807 100644 --- a/README.md +++ b/README.md @@ -52,9 +52,11 @@ from Exscript.util.start import start from Exscript.util.file import get_hosts_from_file from Exscript.util.file import get_accounts_from_file + def do_something(job, host, conn): conn.execute('uname -a') + accounts = get_accounts_from_file('accounts.cfg') hosts = get_hosts_from_file('myhosts.txt') start(accounts, hosts, do_something, max_threads=2) diff --git a/demos/multiple/multiple.py b/demos/multiple/multiple.py index 3c029439..b4b3c12b 100644 --- a/demos/multiple/multiple.py +++ b/demos/multiple/multiple.py @@ -23,6 +23,7 @@ def two(job, host, conn): conn.autoinit() conn.execute('show interface POS1/0') + accounts = get_accounts_from_file('accounts.cfg') # Start on one host. diff --git a/demos/quickstart/quickstart.py b/demos/quickstart/quickstart.py index 1a251fa7..a611bfc5 100644 --- a/demos/quickstart/quickstart.py +++ b/demos/quickstart/quickstart.py @@ -8,6 +8,7 @@ def do_something(job, host, conn): files = any_match(conn, r'(\S+)') print("Files found:", files) + # Open a connection (SSH2, by default) to each of the hosts, and run do_something(). To # open the connection via Telnet or SSH Netconf, you may prefix the hostname by the # protocol, e.g.: 'telnet://hostname', 'netconf://hostname', etc. diff --git a/demos/report/report.py b/demos/report/report.py index 5e2cb1f6..50cc4f08 100644 --- a/demos/report/report.py +++ b/demos/report/report.py @@ -13,6 +13,7 @@ def do_something(job, host, conn): conn.execute('show ip int brie') + # Read input data. accounts = get_accounts_from_file('accounts.cfg') hosts = get_hosts_from_file('hostlist.txt') @@ -21,9 +22,9 @@ def do_something(job, host, conn): # round-robin. "verbose=0" instructs the queue to not generate any # output on stdout. queue = Queue(verbose=0, max_threads=5) -queue.add_account(accounts) # Adds one or more accounts. +queue.add_account(accounts) # Adds one or more accounts. queue.run(hosts, do_something) # Asynchronously enqueues all hosts. -queue.shutdown() # Waits until all hosts are completed. +queue.shutdown() # Waits until all hosts are completed. # Print a short report. print(status(logger)) diff --git a/demos/template/template.py b/demos/template/template.py index dfd1c1b7..fc19af90 100644 --- a/demos/template/template.py +++ b/demos/template/template.py @@ -8,4 +8,5 @@ def do_something(job, host, conn): conn.execute('ls -1') eval_file(conn, 'template.exscript', foobar='hello-world') + quickstart('ssh://localhost', do_something) diff --git a/pyproject.toml b/pyproject.toml index 3210dab5..66db3475 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -75,6 +75,24 @@ packages = ["src/Exscript"] "scripts/exscript" = "exscript" "scripts/otp" = "otp" +[tool.ruff] +line-length = 100 +src = ["src", "tests"] +extend-include = ["scripts/exscript", "scripts/otp"] +extend-exclude = ["doc"] + +[tool.ruff.format] +quote-style = "preserve" + +[tool.ruff.lint] +select = ["E", "W", "F", "I", "UP", "B", "S"] +ignore = [ + "UP031", # printf-style formatting is used throughout +] + +[tool.ruff.lint.per-file-ignores] +"tests/**" = ["S101", "S104", "S105", "S106", "S107", "S108", "S311"] + [tool.pytest.ini_options] testpaths = ["tests"] python_files = ["*Test.py"] diff --git a/scripts/exscript b/scripts/exscript index 5c97a808..2e9d6916 100755 --- a/scripts/exscript +++ b/scripts/exscript @@ -1,5 +1,6 @@ #!/usr/bin/env python3 """TODO: Add missing doctring.""" + # Copyright (C) 2007-2010 Samuel Abels. # # This program is free software; you can redistribute it and/or modify diff --git a/src/Exscript/__init__.py b/src/Exscript/__init__.py index 8d621bc1..4bbd70e6 100644 --- a/src/Exscript/__init__.py +++ b/src/Exscript/__init__.py @@ -23,6 +23,7 @@ """ The core module. """ + import inspect from .account import Account, AccountPool @@ -32,5 +33,8 @@ from .queue import Queue from .version import __version__ -__all__ = [name for name, obj in list(locals().items()) - if not (name.startswith('_') or inspect.ismodule(obj))] +__all__ = [ + name + for name, obj in list(locals().items()) + if not (name.startswith('_') or inspect.ismodule(obj)) +] diff --git a/src/Exscript/account.py b/src/Exscript/account.py index 73ce3d73..3e773913 100644 --- a/src/Exscript/account.py +++ b/src/Exscript/account.py @@ -23,6 +23,7 @@ """ Manages user accounts. """ + import multiprocessing from collections import defaultdict, deque @@ -32,17 +33,11 @@ class Account: - """ This class represents a user account. """ - def __init__(self, - name, - password='', - password2=None, - key=None, - needs_lock=True): + def __init__(self, name, password='', password2=None, key=None, needs_lock=True): """ Constructor. @@ -205,7 +200,6 @@ def get_key(self): class AccountProxy: - """ An object that has a 1:1 relation to an account object in another process. @@ -306,11 +300,9 @@ def acquire(self): if response is None: return False - self.account_hash, \ - self.user, \ - self.password, \ - self.authorization_password, \ - self.key = response + self.account_hash, self.user, self.password, self.authorization_password, self.key = ( + response + ) return True def release(self): @@ -364,7 +356,6 @@ def get_key(self): class LoggerProxy: - """ An object that has a 1:1 relation to a Logger object in another process. @@ -398,7 +389,6 @@ def log_succeeded(self, job_id): class AccountPool: - """ This class manages a collection of available accounts. """ @@ -574,7 +564,6 @@ def release_accounts(self, owner): class AccountManager: - """ Keeps track of available user accounts and assigns them to the worker threads. diff --git a/src/Exscript/contrib/otel.py b/src/Exscript/contrib/otel.py index 0ec693be..22d3f891 100644 --- a/src/Exscript/contrib/otel.py +++ b/src/Exscript/contrib/otel.py @@ -33,6 +33,7 @@ ExscriptInstrumentor().instrument() """ + from __future__ import annotations import functools @@ -50,7 +51,7 @@ # TODO (Netconf): Check whether Netconf is now similar enough to Protocol. try: from Exscript.protocols.netconf import Netconf -except ImportError: # ncclient may not be installed +except ImportError: # ncclient may not be installed Netconf = None # type: ignore[misc,assignment] from Exscript.version import __version__ as _exscript_version @@ -101,19 +102,19 @@ def _get_protocol_attributes(proto: Protocol) -> dict: pass try: prompt = proto.get_prompt() - attrs["protocol.prompt"] = [ - getattr(p, "pattern", str(p)) for p in prompt - ] if isinstance(prompt, (list, tuple)) else getattr( - prompt, "pattern", str(prompt) + attrs["protocol.prompt"] = ( + [getattr(p, "pattern", str(p)) for p in prompt] + if isinstance(prompt, (list, tuple)) + else getattr(prompt, "pattern", str(prompt)) ) except Exception: # noqa: BLE001, S110 pass try: err_prompt = proto.get_error_prompt() - attrs["protocol.error_prompt"] = [ - getattr(p, "pattern", str(p)) for p in err_prompt - ] if isinstance(err_prompt, (list, tuple)) else getattr( - err_prompt, "pattern", str(err_prompt) + attrs["protocol.error_prompt"] = ( + [getattr(p, "pattern", str(p)) for p in err_prompt] + if isinstance(err_prompt, (list, tuple)) + else getattr(err_prompt, "pattern", str(err_prompt)) ) except Exception: # noqa: BLE001, S110 pass @@ -192,9 +193,7 @@ def wrapper(self, *args, **kwargs): # --- connect uses a long-lived span (ended in close) --- if action == "connect": - span = tracer.start_span( - name, kind=SpanKind.CLIENT, attributes=span_attributes - ) + span = tracer.start_span(name, kind=SpanKind.CLIENT, attributes=span_attributes) exception = None if callable(request_hook): request_hook(span, self) @@ -287,8 +286,12 @@ def _instrument_class( if getattr(original, _APPLIED_ATTR, False): continue # already instrumented wrapped = _make_wrapper( - original, method_name, tracer, duration_histogram, - request_hook, response_hook, + original, + method_name, + tracer, + duration_histogram, + request_hook, + response_hook, ) setattr(cls, method_name, wrapped) @@ -324,7 +327,9 @@ def instrumentation_dependencies(self) -> Collection[str]: def _instrument(self, **kwargs): tracer_provider = kwargs.get("tracer_provider") tracer = get_tracer( - __name__, _exscript_version, tracer_provider, + __name__, + _exscript_version, + tracer_provider, ) meter_provider = kwargs.get("meter_provider") meter = get_meter(__name__, _exscript_version, meter_provider) @@ -337,11 +342,19 @@ def _instrument(self, **kwargs): response_hook = kwargs.get("response_hook") _instrument_class( - Protocol, tracer, duration_histogram, request_hook, response_hook, + Protocol, + tracer, + duration_histogram, + request_hook, + response_hook, ) if Netconf is not None: _instrument_class( - Netconf, tracer, duration_histogram, request_hook, response_hook, + Netconf, + tracer, + duration_histogram, + request_hook, + response_hook, ) def _uninstrument(self, **kwargs): diff --git a/src/Exscript/emulators/__init__.py b/src/Exscript/emulators/__init__.py index 248535b5..77005dd2 100644 --- a/src/Exscript/emulators/__init__.py +++ b/src/Exscript/emulators/__init__.py @@ -23,11 +23,15 @@ """ Emulating a device for testing your scripts. """ + import inspect from .command import CommandSet from .iosemu import IOSEmulator from .vdevice import VirtualDevice -__all__ = [name for name, obj in list(locals().items()) - if not (name.startswith('_') or inspect.ismodule(obj))] +__all__ = [ + name + for name, obj in list(locals().items()) + if not (name.startswith('_') or inspect.ismodule(obj)) +] diff --git a/src/Exscript/emulators/command.py b/src/Exscript/emulators/command.py index c75035b4..cd2f4347 100644 --- a/src/Exscript/emulators/command.py +++ b/src/Exscript/emulators/command.py @@ -23,12 +23,12 @@ """ Defines the behavior of commands by mapping commands to functions. """ + import re from collections.abc import Iterable class CommandSet: - """ A set of commands to be used by the Dummy adapter. """ diff --git a/src/Exscript/emulators/iosemu.py b/src/Exscript/emulators/iosemu.py index 70d2ad30..39748e6f 100644 --- a/src/Exscript/emulators/iosemu.py +++ b/src/Exscript/emulators/iosemu.py @@ -23,6 +23,7 @@ """ Cisco IOS emulator used for internal testing. """ + import re from .vdevice import VirtualDevice @@ -41,9 +42,11 @@ at '%s' port '/dev/vty0' from '12.34.56.78' ''' + def show_diag(data): slot = re.search(r'(\d+)', data).groups()[0] - return """ + return ( + """ SLOT %s (RP/LC 0 ): 16 Port ISE Packet Over SONET OC-3c/STM-1 Single Mode/IR LC connector MAIN: type 79, 800-19733-08 rev A0 Deviation: 0 @@ -68,10 +71,15 @@ def show_diag(data): TX Packet Memory size: 268435456 bytes, Packet Memory pagesize: 16384 bytes RX Packet Memory size: 268435456 bytes, Packet Memory pagesize: 16384 bytes 0 crashes since restart -""" % slot +""" + % slot + ) + commands = ( -('show version', """ + ( + 'show version', + """ Cisco Internetwork Operating System Software IOS (tm) GS Software (C12KPRP-P-M), Version 12.0(32)SY6c, RELEASE SOFTWARE (fc3) Technical Support: http://www.cisco.com/techsupport @@ -115,9 +123,11 @@ def show_diag(data): 250880K bytes of ATA PCMCIA card at slot 0 (Sector size 512 bytes). 65536K bytes of Flash internal SIMM (Sector size 256K). Configuration register is 0x2102 -"""), - -(r'sh\S* ip int\S* brie\S*', """ +""", + ), + ( + r'sh\S* ip int\S* brie\S*', + """ Interface IP-Address OK? Method Status Protocol Ethernet0 10.108.00.5 YES NVRAM up up Ethernet1 unassigned YES unset administratively down down @@ -126,9 +136,11 @@ def show_diag(data): Serial1 10.108.40.5 YES NVRAM up up Serial2 10.108.100.5 YES manual up up Serial3 unassigned YES unset administratively down down -"""), - -('show interface.*', """ +""", + ), + ( + 'show interface.*', + """ FastEthernet0/2 is administratively down, line protocol is down Hardware is i82545, address is 0001.c9f4.c418 (bia 0001.c9f4.c418) MTU 1500 bytes, BW 100000 Kbit, DLY 100 usec, rely 255/255, load 1/255 @@ -153,29 +165,33 @@ def show_diag(data): 0 babbles, 0 late collision, 0 deferred 1 lost carrier, 0 no carrier 0 output buffer failures, 0 output buffers swapped out -"""), - -(r'show diag \d+', show_diag), - -(r'^!.*', '') +""", + ), + (r'show diag \d+', show_diag), + (r'^!.*', ''), ) + class IOSEmulator(VirtualDevice): - def __init__(self, - hostname, - echo = True, - login_type = VirtualDevice.LOGIN_TYPE_BOTH, - strict = True, - banner = None): + def __init__( + self, + hostname, + echo=True, + login_type=VirtualDevice.LOGIN_TYPE_BOTH, + strict=True, + banner=None, + ): thebanner = iosbanner % (hostname, hostname, hostname) - VirtualDevice.__init__(self, - hostname, - echo = echo, - login_type = login_type, - strict = strict, - banner = banner or thebanner) - self.user_prompt = 'Username: ' + VirtualDevice.__init__( + self, + hostname, + echo=echo, + login_type=login_type, + strict=strict, + banner=banner or thebanner, + ) + self.user_prompt = 'Username: ' self.password_prompt = 'Password: ' - self.prompt = hostname + '#' + self.prompt = hostname + '#' for command, handler in commands: self.add_command(command, handler) diff --git a/src/Exscript/emulators/vdevice.py b/src/Exscript/emulators/vdevice.py index 918b8d10..57d54272 100644 --- a/src/Exscript/emulators/vdevice.py +++ b/src/Exscript/emulators/vdevice.py @@ -23,29 +23,22 @@ """ Defines the behavior of a virtual device, as needed by :class:`Exscript.servers`. """ + from . import CommandSet class VirtualDevice: - """ An object that emulates a remote device. """ - LOGIN_TYPE_PASSWORDONLY, \ - LOGIN_TYPE_USERONLY, \ - LOGIN_TYPE_BOTH, \ - LOGIN_TYPE_NONE = list(range(1, 5)) - - PROMPT_STAGE_USERNAME, \ - PROMPT_STAGE_PASSWORD, \ - PROMPT_STAGE_CUSTOM = list(range(1, 4)) - - def __init__(self, - hostname, - echo=True, - login_type=LOGIN_TYPE_BOTH, - strict=True, - banner=None): + + LOGIN_TYPE_PASSWORDONLY, LOGIN_TYPE_USERONLY, LOGIN_TYPE_BOTH, LOGIN_TYPE_NONE = list( + range(1, 5) + ) + + PROMPT_STAGE_USERNAME, PROMPT_STAGE_PASSWORD, PROMPT_STAGE_CUSTOM = list(range(1, 4)) + + def __init__(self, hostname, echo=True, login_type=LOGIN_TYPE_BOTH, strict=True, banner=None): """ :type hostname: str :param hostname: The hostname, used for the prompt. diff --git a/src/Exscript/host.py b/src/Exscript/host.py index 29367806..cd4c474a 100644 --- a/src/Exscript/host.py +++ b/src/Exscript/host.py @@ -23,6 +23,7 @@ """ Representing a device to connect with. """ + from .account import Account from .util.cast import to_list from .util.ipv4 import clean_ip, is_ip @@ -30,17 +31,11 @@ class Host: - """ Represents a device on which to open a connection. """ - __slots__ = ('protocol', - 'vars', - 'account', - 'name', - 'address', - 'tcp_port', - 'options') + + __slots__ = ('protocol', 'vars', 'account', 'name', 'address', 'tcp_port', 'options') def __init__(self, uri, default_protocol='ssh'): """ @@ -54,7 +49,7 @@ def __init__(self, uri, default_protocol='ssh'): :param default_protocol: The protocol name. """ self.protocol = default_protocol - self.vars = None # To save memory, do not init with a dict. + self.vars = None # To save memory, do not init with a dict. self.account = None self.name = None self.address = None @@ -98,9 +93,7 @@ def set_uri(self, uri): self.set_name(name) self.set_address(name) - if uri.username is not None \ - or uri.password1 is not None \ - or uri.password2: + if uri.username is not None or uri.password1 is not None or uri.password2: account = Account(uri.username, uri.password1, uri.password2) self.set_account(account) @@ -120,9 +113,11 @@ def get_uri(self): url.protocol = self.get_protocol() url.hostname = self.get_address() url.port = self.get_tcp_port() - url.vars = dict((k, to_list(v)) - for (k, v) in list(self.get_all().items()) - if isinstance(v, str) or isinstance(v, list)) + url.vars = dict( + (k, to_list(v)) + for (k, v) in list(self.get_all().items()) + if isinstance(v, str) or isinstance(v, list) + ) if self.account: url.username = self.account.get_name() @@ -144,10 +139,12 @@ def get_dict(self): :rtype: dict :return: The resulting dictionary. """ - return {'hostname': self.get_name(), - 'address': self.get_address(), - 'protocol': self.get_protocol(), - 'port': self.get_tcp_port()} + return { + 'hostname': self.get_name(), + 'address': self.get_address(), + 'protocol': self.get_protocol(), + 'port': self.get_tcp_port(), + } def set_name(self, name): """ diff --git a/src/Exscript/interpreter/__init__.py b/src/Exscript/interpreter/__init__.py index b64a7bc0..94778f6e 100644 --- a/src/Exscript/interpreter/__init__.py +++ b/src/Exscript/interpreter/__init__.py @@ -24,5 +24,8 @@ from .parser import Parser -__all__ = [name for name, obj in list(locals().items()) - if not (name.startswith('_') or inspect.ismodule(obj))] +__all__ = [ + name + for name, obj in list(locals().items()) + if not (name.startswith('_') or inspect.ismodule(obj)) +] diff --git a/src/Exscript/interpreter/append.py b/src/Exscript/interpreter/append.py index b608b771..36262fc3 100644 --- a/src/Exscript/interpreter/append.py +++ b/src/Exscript/interpreter/append.py @@ -25,7 +25,6 @@ class Append(Token): - def __init__(self, lexer, parser, parent): Token.__init__(self, self.__class__.__name__, lexer, parser, parent) diff --git a/src/Exscript/interpreter/assign.py b/src/Exscript/interpreter/assign.py index e63f90b3..10990d2e 100644 --- a/src/Exscript/interpreter/assign.py +++ b/src/Exscript/interpreter/assign.py @@ -25,7 +25,6 @@ class Assign(Token): - def __init__(self, lexer, parser, parent): Token.__init__(self, self.__class__.__name__, lexer, parser, parent) diff --git a/src/Exscript/interpreter/code.py b/src/Exscript/interpreter/code.py index 7475c7b3..3942cb29 100644 --- a/src/Exscript/interpreter/code.py +++ b/src/Exscript/interpreter/code.py @@ -42,56 +42,50 @@ # check if there is some kind of None handling/comparison. If not, we should add it. varname = varname_re.pattern -keywords = ['append', - 'as', - 'else', - 'end', - 'enter', - 'extract', - 'fail', - 'false', - 'from', - 'if', - 'into', - 'loop', - 'try', - 'to', - 'true', - 'until', - 'when', - 'while'] -operators = ['in', - r'not\s+in', - r'is\s+not', - 'is', - 'ge', - 'gt', - 'le', - 'lt', - 'matches'] +keywords = [ + 'append', + 'as', + 'else', + 'end', + 'enter', + 'extract', + 'fail', + 'false', + 'from', + 'if', + 'into', + 'loop', + 'try', + 'to', + 'true', + 'until', + 'when', + 'while', +] +operators = ['in', r'not\s+in', r'is\s+not', 'is', 'ge', 'gt', 'le', 'lt', 'matches'] grammar = ( - ('escaped_data', r'\\.'), - ('line_comment', r'(?:^|(?<=[\r\n{]))[ \t]*#[^\r\n]*(?:[\r\n]|$)'), - ('regex_delimiter', r'/'), - ('string_delimiter', r'"'), - ('open_curly_bracket', r'{'), + ('escaped_data', r'\\.'), + ('line_comment', r'(?:^|(?<=[\r\n{]))[ \t]*#[^\r\n]*(?:[\r\n]|$)'), + ('regex_delimiter', r'/'), + ('string_delimiter', r'"'), + ('open_curly_bracket', r'{'), ('close_curly_bracket', r'}'), - ('close_bracket', r'\)'), - ('comma', r','), - ('whitespace', r'[ \t]+'), - ('keyword', r'\b(?:' + '|'.join(keywords) + r')\b'), - ('comparison', r'\b(?:' + '|'.join(operators) + r')\b'), + ('close_bracket', r'\)'), + ('comma', r','), + ('whitespace', r'[ \t]+'), + ('keyword', r'\b(?:' + '|'.join(keywords) + r')\b'), + ('comparison', r'\b(?:' + '|'.join(operators) + r')\b'), ('arithmetic_operator', r'(?:\*|\+|-|%|\.)'), - ('logical_operator', r'\b(?:and|or|not)\b'), - ('assign', r'='), - ('hex_number', r'\b0x[0-9a-fA-F]+\b'), - ('octal_number', r'\b0o?[0-7]+\b'), - ('number', r'\b\d+\b'), - ('open_function_call', varname + r'(?:\.' + varname + r')*\('), - ('varname', varname), - ('newline', r'[\r\n]'), - ('raw_data', r'[^\r\n{}]+') + ('logical_operator', r'\b(?:and|or|not)\b'), + ('assign', r'='), + ('hex_number', r'\b0x[0-9a-fA-F]+\b'), + ('octal_number', r'\b0o?[0-7]+\b'), + ('number', r'\b\d+\b'), + ('open_function_call', varname + r'(?:\.' + varname + r')*\('), + ('varname', varname), + ('newline', r'[\r\n]'), + ('raw_data', r'[^\r\n{}]+'), ) grammar_c = [] @@ -100,7 +94,6 @@ class Code(Scope): - def __init__(self, lexer, parser, parent): Scope.__init__(self, self.__class__.__name__, lexer, parser, parent) lexer.set_grammar(grammar_c) diff --git a/src/Exscript/interpreter/enter.py b/src/Exscript/interpreter/enter.py index c5f48fcb..3b46ca77 100644 --- a/src/Exscript/interpreter/enter.py +++ b/src/Exscript/interpreter/enter.py @@ -25,7 +25,6 @@ class Enter(Token): - def __init__(self, lexer, parser, parent): Token.__init__(self, self.__class__.__name__, lexer, parser, parent) lexer.expect(self, 'keyword', 'enter') diff --git a/src/Exscript/interpreter/exception.py b/src/Exscript/interpreter/exception.py index 9dc32415..ae16a38e 100644 --- a/src/Exscript/interpreter/exception.py +++ b/src/Exscript/interpreter/exception.py @@ -27,17 +27,17 @@ class ExscriptException(Exception): class FailException(ExscriptException): - """ This exception type is used if the "fail" command was used in a template. """ + pass class PermissionError(ExscriptException): - """ Raised if an insecure function was called when the parser is in secure mode. """ + pass diff --git a/src/Exscript/interpreter/execute.py b/src/Exscript/interpreter/execute.py index f9237a43..6543986d 100644 --- a/src/Exscript/interpreter/execute.py +++ b/src/Exscript/interpreter/execute.py @@ -30,9 +30,9 @@ from .string import String, varstring_re grammar = { - 'leading_blank_lines': r'^(?:[ \t]*[\r\n])+', - 'trailing_blank_lines': r'(?:[\r\n][ \t]*)+$', - 'line_comment': r'(?:^|(?<=[\r\n]))[ \t]*#[^\r\n]*(?:[\r\n]|$)', + 'leading_blank_lines': r'^(?:[ \t]*[\r\n])+', + 'trailing_blank_lines': r'(?:[\r\n][ \t]*)+$', + 'line_comment': r'(?:^|(?<=[\r\n]))[ \t]*#[^\r\n]*(?:[\r\n]|$)', } leading_whitespace_re = re.compile(r'^[ \t\r\n]*') line_comment_re = re.compile(grammar['line_comment']) @@ -40,7 +40,6 @@ class Execute(String): - def __init__(self, lexer, parser, parent, command, escape_offset=0): Token.__init__(self, self.__class__.__name__, lexer, parser, parent) self.delimiter = '' # Command strings are not delimited in Execute tokens. @@ -85,10 +84,7 @@ def __init__(self, lexer, parser, parent, command, escape_offset=0): rpc_command = varstring_re.sub(self.variable_sigil_sub_cb, rpc_command) # Now we can validate the prepared RPC command. - msg = ( - "XML validation of the RPC command has failed. The error was " - "caused by:\n" - ) + msg = "XML validation of the RPC command has failed. The error was caused by:\n" try: validate_xml(rpc_command, silent=False) except UnicodeEncodeError as exc: @@ -102,10 +98,10 @@ def __init__(self, lexer, parser, parent, command, escape_offset=0): # XMLSyntaxError position (line, offset) needs to be converted # to an absolute character offset. Then adjust the start/end marks. lines = rpc_command.splitlines(keepends=True) - exc_start = len(''.join(lines[:exc.lineno - 1])) + exc_start = len(''.join(lines[: exc.lineno - 1])) exc_start += exc.offset # We assume the end positions are either both set or not! - exc_end = len(''.join(lines[:(exc.end_lineno or exc.lineno) - 1])) + exc_end = len(''.join(lines[: (exc.end_lineno or exc.lineno) - 1])) exc_end += exc.end_offset or exc.offset + 1 self.mark_start(original_start + exc_start) self.mark_end(distance=(exc_end - exc_start)) diff --git a/src/Exscript/interpreter/expression.py b/src/Exscript/interpreter/expression.py index 39c4ef22..f087d7af 100644 --- a/src/Exscript/interpreter/expression.py +++ b/src/Exscript/interpreter/expression.py @@ -26,7 +26,6 @@ class ExpressionNode(Token): - def __init__(self, lexer, parser, parent, parent_node=None): # Skip whitespace before initializing the token to make sure that self.start # points to the beginning of the expression (which makes for prettier error @@ -47,20 +46,24 @@ def __init__(self, lexer, parser, parent, parent_node=None): # The expression may end already (a single term is also an expression). lexer.skip(['whitespace', 'newline']) - if not lexer.current_is('arithmetic_operator') and \ - not lexer.current_is('logical_operator') and \ - not lexer.current_is('comparison') and \ - not lexer.current_is('regex_delimiter'): + if ( + not lexer.current_is('arithmetic_operator') + and not lexer.current_is('logical_operator') + and not lexer.current_is('comparison') + and not lexer.current_is('regex_delimiter') + ): # Early end of statement. self.mark_end(rstrip=True) return # Expect the operator. self.op_type, self.op = lexer.token() - if not lexer.next_if('arithmetic_operator') and \ - not lexer.next_if('logical_operator') and \ - not lexer.next_if('comparison') and \ - not lexer.next_if('regex_delimiter'): + if ( + not lexer.next_if('arithmetic_operator') + and not lexer.next_if('logical_operator') + and not lexer.next_if('comparison') + and not lexer.next_if('regex_delimiter') + ): self.mark_start() self.mark_end(distance=len(self.op)) msg = 'Expected operator but got %s %s' % (self.op_type, repr(self.op)) @@ -186,7 +189,6 @@ def dump(self, indent=0): class Expression(Token): - def __init__(self, lexer, parser, parent): Token.__init__(self, self.__class__.__name__, lexer, parser, parent) diff --git a/src/Exscript/interpreter/extract.py b/src/Exscript/interpreter/extract.py index e8181670..3f4e34a2 100644 --- a/src/Exscript/interpreter/extract.py +++ b/src/Exscript/interpreter/extract.py @@ -26,7 +26,6 @@ class Extract(Token): - def __init__(self, lexer, parser, parent): Token.__init__(self, self.__class__.__name__, lexer, parser, parent) self.varnames = [] diff --git a/src/Exscript/interpreter/fail.py b/src/Exscript/interpreter/fail.py index cd4ad0bc..10061809 100644 --- a/src/Exscript/interpreter/fail.py +++ b/src/Exscript/interpreter/fail.py @@ -26,7 +26,6 @@ class Fail(Token): - def __init__(self, lexer, parser, parent): Token.__init__(self, self.__class__.__name__, lexer, parser, parent) self.expression = None diff --git a/src/Exscript/interpreter/functioncall.py b/src/Exscript/interpreter/functioncall.py index 86b78e1e..017713b4 100644 --- a/src/Exscript/interpreter/functioncall.py +++ b/src/Exscript/interpreter/functioncall.py @@ -27,7 +27,6 @@ class FunctionCall(Token): - def __init__(self, lexer, parser, parent): Token.__init__(self, self.__class__.__name__, lexer, parser, parent) self.funcname = None @@ -52,9 +51,7 @@ def __init__(self, lexer, parser, parent): while True: if lexer.next_if('close_bracket'): break - self.arguments.append( - Exscript.interpreter.expression.Expression(lexer, parser, parent) - ) + self.arguments.append(Exscript.interpreter.expression.Expression(lexer, parser, parent)) ttype, token = lexer.token() if not lexer.next_if('comma') and not lexer.current_is('close_bracket'): self.mark_start() @@ -76,17 +73,13 @@ def value(self, context): function = self.parent.get(self.funcname) if function is None: self.mark_end(distance=len(self.funcname)) - self.lexer.runtime_error( - 'Undefined function %s' % repr(self.funcname), self - ) + self.lexer.runtime_error('Undefined function %s' % repr(self.funcname), self) try: result = function(self.parent, *argument_values) except Exception as e: self.mark_end(distance=len(self.funcname)) self.lexer.runtime_error( - '%s while executing function %s: %s' % ( - type(e).__name__, repr(self.funcname), e - ), + '%s while executing function %s: %s' % (type(e).__name__, repr(self.funcname), e), self, ) else: diff --git a/src/Exscript/interpreter/ifcondition.py b/src/Exscript/interpreter/ifcondition.py index 8edb7d81..3aa5873e 100644 --- a/src/Exscript/interpreter/ifcondition.py +++ b/src/Exscript/interpreter/ifcondition.py @@ -27,7 +27,6 @@ class IfCondition(Token): - def __init__(self, lexer, parser, parent): Token.__init__(self, self.__class__.__name__, lexer, parser, parent) diff --git a/src/Exscript/interpreter/loop.py b/src/Exscript/interpreter/loop.py index aaa2994e..fbd23313 100644 --- a/src/Exscript/interpreter/loop.py +++ b/src/Exscript/interpreter/loop.py @@ -28,7 +28,6 @@ class Loop(Token): - def __init__(self, lexer, parser, parent): Token.__init__(self, self.__class__.__name__, lexer, parser, parent) self.during = None @@ -41,9 +40,11 @@ def __init__(self, lexer, parser, parent): # Expect one ore more lists for the default loop. lexer.expect(self, 'keyword', 'loop') lexer.expect(self, 'whitespace') - if not lexer.current_is('keyword', 'while') and \ - not lexer.current_is('keyword', 'until') and \ - not lexer.current_is('keyword', 'from'): + if ( + not lexer.current_is('keyword', 'while') + and not lexer.current_is('keyword', 'until') + and not lexer.current_is('keyword', 'from') + ): self.list_variables = [Term(lexer, parser, parent)] lexer.next_if('whitespace') while lexer.next_if('comma'): diff --git a/src/Exscript/interpreter/number.py b/src/Exscript/interpreter/number.py index 5181f251..98651abe 100644 --- a/src/Exscript/interpreter/number.py +++ b/src/Exscript/interpreter/number.py @@ -21,8 +21,8 @@ # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -class Number: +class Number: def __init__(self, number): self.number = int(number) diff --git a/src/Exscript/interpreter/parser.py b/src/Exscript/interpreter/parser.py index 3a916175..e486943b 100644 --- a/src/Exscript/interpreter/parser.py +++ b/src/Exscript/interpreter/parser.py @@ -28,7 +28,6 @@ class Parser: - def __init__(self, **kwargs): self.no_escaping = kwargs.get('no_escaping', False) self.no_prompt = kwargs.get('no_prompt', False) @@ -39,7 +38,11 @@ def __init__(self, **kwargs): def define(self, **kwargs): for key, value in list(kwargs.items()): - if not isinstance(value, str) and isinstance(value, Iterable) or isinstance(value, Callable): + if ( + not isinstance(value, str) + and isinstance(value, Iterable) + or isinstance(value, Callable) + ): self.variables[key] = value else: self.variables[key] = [value] @@ -59,8 +62,10 @@ def parse_file(self, filename): lexer = self._create_lexer() return lexer.parse_file(filename) + if __name__ == "__main__": import sys + if len(sys.argv) == 1: filename = 'test.exscript' elif len(sys.argv) == 2: diff --git a/src/Exscript/interpreter/program.py b/src/Exscript/interpreter/program.py index d0894c8e..17de1b80 100644 --- a/src/Exscript/interpreter/program.py +++ b/src/Exscript/interpreter/program.py @@ -27,7 +27,6 @@ class Program(Scope): - def __init__(self, lexer, parser, variables, **kwargs): Scope.__init__(self, 'Program', lexer, parser, None, **kwargs) self.variables = variables diff --git a/src/Exscript/interpreter/regex.py b/src/Exscript/interpreter/regex.py index b044bbd3..bfd98db5 100644 --- a/src/Exscript/interpreter/regex.py +++ b/src/Exscript/interpreter/regex.py @@ -25,7 +25,7 @@ from .string import String modifier_grammar = ( - ('modifier', r'[a-zA-Z]'), + ('modifier', r'[a-zA-Z]'), ('invalid_char', r'.'), # used to implicitly exit the loop ) @@ -35,7 +35,6 @@ class Regex(String): - def __init__(self, lexer, parser, parent): # String parser collects the regex. String.__init__(self, lexer, parser, parent, purpose='regex') diff --git a/src/Exscript/interpreter/rpc.py b/src/Exscript/interpreter/rpc.py index d3a57cff..0ba9f546 100644 --- a/src/Exscript/interpreter/rpc.py +++ b/src/Exscript/interpreter/rpc.py @@ -26,12 +26,12 @@ from .scope import Scope grammar = ( - ('line_comment', r'(?:^|(?<=[\r\n]))[ \t]*#[^\r\n]*(?:[\r\n]|$)'), - ('open_rpc_tag', r''), - ('close_rpc_tag', r'<\/rpc>'), - ('newline', r'[\r\n]'), + ('line_comment', r'(?:^|(?<=[\r\n]))[ \t]*#[^\r\n]*(?:[\r\n]|$)'), + ('open_rpc_tag', r''), + ('close_rpc_tag', r'<\/rpc>'), + ('newline', r'[\r\n]'), ('raw_data_before_tag', r'[^\r\n]+?(?=<\/?rpc>)'), - ('raw_data', r'[^\r\n]+'), + ('raw_data', r'[^\r\n]+'), ) grammar_c = [] @@ -40,7 +40,6 @@ class RPCBody(Scope): - def __init__(self, lexer, parser, parent): Scope.__init__(self, self.__class__.__name__, lexer, parser, parent) lexer.set_grammar(grammar_c) @@ -54,9 +53,7 @@ def __init__(self, lexer, parser, parent): break # Line comments in multiline RPCs are handled later by the Execute # token, so we have to collect them too. - elif lexer.current_is( - ['line_comment', 'newline', 'raw_data_before_tag', 'raw_data'] - ): + elif lexer.current_is(['line_comment', 'newline', 'raw_data_before_tag', 'raw_data']): buffer += lexer.token()[1] next(lexer) elif lexer.current_is('EOF'): diff --git a/src/Exscript/interpreter/scope.py b/src/Exscript/interpreter/scope.py index 0a7b606f..400753a8 100644 --- a/src/Exscript/interpreter/scope.py +++ b/src/Exscript/interpreter/scope.py @@ -26,7 +26,6 @@ class Scope(Token): - def __init__(self, name, lexer, parser, parent=None, *args, **kwargs): Token.__init__(self, name, lexer, parser, parent) self.variables = kwargs.get('variables', {}) @@ -46,8 +45,7 @@ def define(self, **kwargs): if self.parent is not None: return self.parent.define(**kwargs) for key in kwargs: - if key.find('.') >= 0 or key.startswith('_') \ - or type(kwargs[key]) is type([]): + if key.find('.') >= 0 or key.startswith('_') or type(kwargs[key]) is type([]): self.variables[key] = kwargs[key] else: self.variables[key] = [kwargs[key]] diff --git a/src/Exscript/interpreter/string.py b/src/Exscript/interpreter/string.py index 3837edf7..e7b883a0 100644 --- a/src/Exscript/interpreter/string.py +++ b/src/Exscript/interpreter/string.py @@ -30,17 +30,17 @@ escaped_grammar = [ # Unicode range: 0x00000000 to 0x0010FFFF - ('escaped_unicode_32bit', r'\\U[0-9a-fA-F]{8}'), - ('escaped_unicode_16bit', r'\\u[0-9a-fA-F]{4}'), - ('escaped_hex', r'\\x[0-9a-fA-F]{2}'), - ('escaped_octal', r'\\[0-7]{3}'), + ('escaped_unicode_32bit', r'\\U[0-9a-fA-F]{8}'), + ('escaped_unicode_16bit', r'\\u[0-9a-fA-F]{4}'), + ('escaped_hex', r'\\x[0-9a-fA-F]{2}'), + ('escaped_octal', r'\\[0-7]{3}'), ('escaped_extended_controls', r'\\[0abefv]'), - ('escaped_common_controls', r'\\[nrt]'), + ('escaped_common_controls', r'\\[nrt]'), ] grammar = escaped_grammar + [ - ('escaped_data', r'\\.'), - ('newline', r'[\r\n]'), + ('escaped_data', r'\\.'), + ('newline', r'[\r\n]'), ] grammar_c = [] @@ -49,7 +49,6 @@ class String(Token): - def __init__(self, lexer, parser, parent, purpose='string'): Token.__init__(self, self.__class__.__name__, lexer, parser, parent) @@ -59,7 +58,7 @@ def __init__(self, lexer, parser, parent, purpose='string'): data_re = re.compile(r'[^\r\n\\' + escaped_delimiter + ']+') delimiter_re = re.compile(escaped_delimiter) grammar_with_delim = grammar_c[:] - grammar_with_delim.append((purpose + '_data', data_re)) + grammar_with_delim.append((purpose + '_data', data_re)) grammar_with_delim.append((purpose + '_delimiter', delimiter_re)) lexer.set_grammar(grammar_with_delim) @@ -70,15 +69,17 @@ def __init__(self, lexer, parser, parent, purpose='string'): if lexer.current_is(purpose + '_data'): self.string += lexer.token()[1] next(lexer) - elif lexer.current_is([ - 'escaped_unicode_32bit', - 'escaped_unicode_16bit', - 'escaped_hex', - 'escaped_octal', - 'escaped_extended_controls', - 'escaped_common_controls', - 'escaped_data', - ]): + elif lexer.current_is( + [ + 'escaped_unicode_32bit', + 'escaped_unicode_16bit', + 'escaped_hex', + 'escaped_octal', + 'escaped_extended_controls', + 'escaped_common_controls', + 'escaped_data', + ] + ): self.string += self.eval_escape(*lexer.token()) next(lexer) elif lexer.next_if(purpose + '_delimiter'): diff --git a/src/Exscript/interpreter/template.py b/src/Exscript/interpreter/template.py index cbf3c97c..81be3c13 100644 --- a/src/Exscript/interpreter/template.py +++ b/src/Exscript/interpreter/template.py @@ -30,24 +30,24 @@ escaped_grammar = [ # Unicode range: 0x00000000 to 0x0010FFFF - ('escaped_unicode_32bit', r'\\U[0-9a-fA-F]{8}'), - ('escaped_unicode_16bit', r'\\u[0-9a-fA-F]{4}'), - ('escaped_hex', r'\\x[0-9a-fA-F]{2}'), - ('escaped_octal', r'\\[0-7]{3}'), + ('escaped_unicode_32bit', r'\\U[0-9a-fA-F]{8}'), + ('escaped_unicode_16bit', r'\\u[0-9a-fA-F]{4}'), + ('escaped_hex', r'\\x[0-9a-fA-F]{2}'), + ('escaped_octal', r'\\[0-7]{3}'), ('escaped_extended_controls', r'\\[0abefv]'), - ('escaped_common_controls', r'\\[nrt]'), + ('escaped_common_controls', r'\\[nrt]'), ] grammar = escaped_grammar + [ - ('escaped_data', r'\\.'), - ('line_comment', r'(?:^|(?<=[\r\n}]))[ \t]*#[^\r\n]*(?:[\r\n]|$)'), - ('open_curly_bracket', '{'), + ('escaped_data', r'\\.'), + ('line_comment', r'(?:^|(?<=[\r\n}]))[ \t]*#[^\r\n]*(?:[\r\n]|$)'), + ('open_curly_bracket', '{'), ('close_curly_bracket', '}'), - ('open_rpc_tag', r''), - ('close_rpc_tag', r'<\/rpc>'), - ('newline', r'[\r\n]'), + ('open_rpc_tag', r''), + ('close_rpc_tag', r'<\/rpc>'), + ('newline', r'[\r\n]'), ('raw_data_before_tag', r'[^\r\n{}\\]+?(?=<\/?rpc>)'), - ('raw_data', r'[^\r\n{}\\]+'), + ('raw_data', r'[^\r\n{}\\]+'), ] grammar_c = [] @@ -56,7 +56,6 @@ class Template(Scope): - def __init__(self, lexer, parser, parent, *args, **kwargs): Scope.__init__(self, self.__class__.__name__, lexer, parser, parent, **kwargs) lexer.set_grammar(grammar_c) @@ -86,15 +85,17 @@ def __init__(self, lexer, parser, parent, *args, **kwargs): elif lexer.current_is(['raw_data_before_tag', 'raw_data']): buffer += lexer.token()[1] next(lexer) - elif lexer.current_is([ - 'escaped_unicode_32bit', - 'escaped_unicode_16bit', - 'escaped_hex', - 'escaped_octal', - 'escaped_extended_controls', - 'escaped_common_controls', - 'escaped_data', - ]): + elif lexer.current_is( + [ + 'escaped_unicode_32bit', + 'escaped_unicode_16bit', + 'escaped_hex', + 'escaped_octal', + 'escaped_extended_controls', + 'escaped_common_controls', + 'escaped_data', + ] + ): ttype, token = lexer.token() result = self.eval_escape(ttype, token) buffer += result diff --git a/src/Exscript/interpreter/term.py b/src/Exscript/interpreter/term.py index 5307886e..2e33ffef 100644 --- a/src/Exscript/interpreter/term.py +++ b/src/Exscript/interpreter/term.py @@ -29,7 +29,6 @@ class Term(Token): - def __init__(self, lexer, parser, parent): Token.__init__(self, self.__class__.__name__, lexer, parser, parent) self.term = None diff --git a/src/Exscript/interpreter/trying.py b/src/Exscript/interpreter/trying.py index 302dc5c7..3af15965 100644 --- a/src/Exscript/interpreter/trying.py +++ b/src/Exscript/interpreter/trying.py @@ -27,7 +27,6 @@ class Try(Scope): - def __init__(self, lexer, parser, parent): Scope.__init__(self, self.__class__.__name__, lexer, parser, parent) lexer.expect(self, 'keyword', 'try') diff --git a/src/Exscript/interpreter/variable.py b/src/Exscript/interpreter/variable.py index 3deed11a..d6444ab5 100644 --- a/src/Exscript/interpreter/variable.py +++ b/src/Exscript/interpreter/variable.py @@ -24,7 +24,6 @@ class Variable(Token): - def __init__(self, lexer, parser, parent): Token.__init__(self, self.__class__.__name__, lexer, parser, parent) self.varname = lexer.token()[1] diff --git a/src/Exscript/key.py b/src/Exscript/key.py index 47e99c97..844aecfc 100644 --- a/src/Exscript/key.py +++ b/src/Exscript/key.py @@ -23,16 +23,17 @@ """ Represents a private key. """ + from paramiko import ECDSAKey, Ed25519Key, RSAKey from paramiko.ssh_exception import SSHException class PrivateKey: - """ Represents a cryptographic key, and may be used to authenticate using :class:`Exscript.protocols`. """ + keytypes: set[str] = set() def __init__(self, keytype='rsa'): diff --git a/src/Exscript/logger.py b/src/Exscript/logger.py index d44ae8f0..0eaab1a9 100644 --- a/src/Exscript/logger.py +++ b/src/Exscript/logger.py @@ -77,7 +77,6 @@ def setupLogging(*, package=False, parser=False, protocol=False, init=getConsole """ class _LogFilter: - def __init__(self, name, enabled): self.name = name self.enabled = enabled @@ -117,10 +116,12 @@ def filter(self, record): # Exscript Logging -logger_registry: 'weakref.WeakValueDictionary[int, Logger]' = weakref.WeakValueDictionary() # Map id(logger) to Logger. +logger_registry: 'weakref.WeakValueDictionary[int, Logger]' = ( + weakref.WeakValueDictionary() +) # Map id(logger) to Logger. -class Log: +class Log: def __init__(self, name): self.name = name self.data = StringIO('') @@ -176,7 +177,6 @@ def has_ended(self): class Logfile(Log): - """ This class logs to two files: The raw log, and sometimes a separate log containing the error message with a traceback. @@ -241,7 +241,6 @@ def succeeded(self): class Logger: - """ Implements logging for the queue: keeps a log of each attempt of each job. Logs are kept in memory, and not written to the disk. @@ -316,7 +315,6 @@ def log_succeeded(self, job_id): class LoggerProxy: - """ An object that has a 1:1 relation to a Logger object in another process. @@ -355,16 +353,11 @@ def log_succeeded(self, job_id): class FileLogger(Logger): - """ A Logger that stores logs into files. """ - def __init__(self, - logdir, - mode='a', - delete=False, - clearmem=True): + def __init__(self, logdir, mode='a', delete=False, clearmem=True): """ The logdir argument specifies the location where the logs are stored. The mode specifies whether to append the existing logs diff --git a/src/Exscript/parselib/__init__.py b/src/Exscript/parselib/__init__.py index 48a7245e..e6b5529c 100644 --- a/src/Exscript/parselib/__init__.py +++ b/src/Exscript/parselib/__init__.py @@ -25,5 +25,8 @@ from .lexer import Lexer from .token import Token -__all__ = [name for name, obj in list(locals().items()) - if not (name.startswith('_') or inspect.ismodule(obj))] +__all__ = [ + name + for name, obj in list(locals().items()) + if not (name.startswith('_') or inspect.ismodule(obj)) +] diff --git a/src/Exscript/parselib/exception.py b/src/Exscript/parselib/exception.py index 6620ec0e..4357a79a 100644 --- a/src/Exscript/parselib/exception.py +++ b/src/Exscript/parselib/exception.py @@ -23,43 +23,43 @@ class LexerException(Exception): - """ Fallback exception that is called when the error type is not known. """ + pass class LexerWarning(UserWarning): - """ Base class for minor warnings. """ + pass class CompileError(LexerException): - """ Raised during the compilation procedure if the template contained a syntax error. """ + pass class CompileWarning(LexerWarning): - """ Raised during the compilation procedure if the template contained a syntax error, that does not prevent compilation but should be noted. """ + pass class ExecuteError(LexerException): - """ Raised during the execution of the compiled template whenever any error occurs. """ + pass diff --git a/src/Exscript/parselib/lexer.py b/src/Exscript/parselib/lexer.py index 9e4da359..76b1647d 100644 --- a/src/Exscript/parselib/lexer.py +++ b/src/Exscript/parselib/lexer.py @@ -28,7 +28,6 @@ class Lexer: - def __init__(self, parser_cls, *args, **kwargs): """ The given args are passed to the parser_cls constructor. @@ -113,7 +112,7 @@ def _error(self, exc_cls, error, sender=None): lineno_last = self._get_line_number_from_char(sender.end) # When the sender marks a range, that ends with a newline character, the # end mark would point to an additional irrelevant line, so we fix that. - if self.input[sender.start:sender.end].endswith('\n'): + if self.input[sender.start : sender.end].endswith('\n'): lineno_last -= 1 lines = self._get_lines(lineno_first, lineno_last) diff --git a/src/Exscript/parselib/token.py b/src/Exscript/parselib/token.py index 0de03d22..ac12d4fd 100644 --- a/src/Exscript/parselib/token.py +++ b/src/Exscript/parselib/token.py @@ -23,13 +23,11 @@ class Token: - """ Abstract base class for all tokens. """ class Iterator: - """ A tree iterator that walks through all tokens. """ @@ -117,7 +115,7 @@ def mark_end(self, char=None, distance=None, rstrip=False): isspace = lambda c: c in rstrip # noqa: E731 else: isspace = lambda c: c.isspace() # noqa: E731 - while char > 0 and isspace(self.lexer.input[char - 1:char]): + while char > 0 and isspace(self.lexer.input[char - 1 : char]): char -= 1 # EOF will point behind the last character index (length of input)! self.end = max(self.start, min(char, self.lexer.input_length)) diff --git a/src/Exscript/protocols/__init__.py b/src/Exscript/protocols/__init__.py index 7d85ed72..3f74cf94 100644 --- a/src/Exscript/protocols/__init__.py +++ b/src/Exscript/protocols/__init__.py @@ -30,11 +30,11 @@ from .telnet import Telnet protocol_map = { - 'dummy': Dummy, + 'dummy': Dummy, 'pseudo': Dummy, 'telnet': Telnet, - 'ssh': SSH2, - 'ssh2': SSH2, + 'ssh': SSH2, + 'ssh2': SSH2, 'netconf': Netconf, } @@ -116,7 +116,11 @@ def connect(host, default_protocol='ssh', **kwargs): conn.login(account) return conn + import inspect -__all__ = [name for name, obj in list(locals().items()) - if not (name.startswith('_') or inspect.ismodule(obj))] +__all__ = [ + name + for name, obj in list(locals().items()) + if not (name.startswith('_') or inspect.ismodule(obj)) +] diff --git a/src/Exscript/protocols/drivers/__init__.py b/src/Exscript/protocols/drivers/__init__.py index a170b9dc..62015d6d 100644 --- a/src/Exscript/protocols/drivers/__init__.py +++ b/src/Exscript/protocols/drivers/__init__.py @@ -66,17 +66,20 @@ def isdriver(o): return inspect.isclass(o) and issubclass(o, Driver) and o is not Driver + def add_driver(cls): driver = cls() driver_classes.append(cls) drivers.append(driver) driver_map[driver.name] = driver + def disable_driver(name): driver = driver_map.pop(name) drivers.pop(driver) driver_classes.pop(driver.__class__) + # Load built-in drivers. for name, obj in list(locals().items()): if isdriver(obj): diff --git a/src/Exscript/protocols/drivers/ace.py b/src/Exscript/protocols/drivers/ace.py index 0030342e..638fd75c 100644 --- a/src/Exscript/protocols/drivers/ace.py +++ b/src/Exscript/protocols/drivers/ace.py @@ -23,6 +23,7 @@ """ A driver for Cisco Application Control Engine (ACE). """ + import re from .driver import Driver @@ -40,7 +41,6 @@ class ACEDriver(Driver): - def __init__(self): Driver.__init__(self, 'ace') self.user_re = _user_re diff --git a/src/Exscript/protocols/drivers/adtran.py b/src/Exscript/protocols/drivers/adtran.py index b3a7c59b..62b93020 100644 --- a/src/Exscript/protocols/drivers/adtran.py +++ b/src/Exscript/protocols/drivers/adtran.py @@ -26,11 +26,11 @@ The DSLAMs only report "OpenSSH" in their SSH remote protocol id and have no SSH banner so no possibility for check_*_for_os(). """ + from .driver import Driver class AdtranDriver(Driver): - def __init__(self): Driver.__init__(self, 'adtran') diff --git a/src/Exscript/protocols/drivers/aironet.py b/src/Exscript/protocols/drivers/aironet.py index a4ad3de9..8ef8dbf6 100644 --- a/src/Exscript/protocols/drivers/aironet.py +++ b/src/Exscript/protocols/drivers/aironet.py @@ -23,6 +23,7 @@ """ A driver for Cisco Aironet Wireless Controllers. """ + import re from .driver import Driver @@ -32,7 +33,6 @@ class AironetDriver(Driver): - def __init__(self): Driver.__init__(self, 'aironet') self.user_re = _user_re diff --git a/src/Exscript/protocols/drivers/aix.py b/src/Exscript/protocols/drivers/aix.py index 7eeacacc..07865d53 100644 --- a/src/Exscript/protocols/drivers/aix.py +++ b/src/Exscript/protocols/drivers/aix.py @@ -23,6 +23,7 @@ """ A driver for AIX. """ + import re from .driver import Driver @@ -33,7 +34,6 @@ class AIXDriver(Driver): - def __init__(self): Driver.__init__(self, 'aix') self.user_re = _user_re diff --git a/src/Exscript/protocols/drivers/arbor_peakflow.py b/src/Exscript/protocols/drivers/arbor_peakflow.py index d7ab14f0..b5329fd8 100644 --- a/src/Exscript/protocols/drivers/arbor_peakflow.py +++ b/src/Exscript/protocols/drivers/arbor_peakflow.py @@ -23,6 +23,7 @@ """ A driver for Peakflow SP by Arbor Networks. """ + import re from .driver import Driver @@ -30,7 +31,7 @@ _user_re = [re.compile(r'(user|login): $', re.I)] _password_re = [re.compile(r'Password: $')] _prompt_re = [ - re.compile(r'(?:\r\n|[\r\n])[\-\w+\._]+@[\-\w+\._~]+:(?:/\w*)+[#%] $'), # arbos prompt + re.compile(r'(?:\r\n|[\r\n])[\-\w+\._]+@[\-\w+\._~]+:(?:/\w*)+[#%] $'), # arbos prompt re.compile(r'Exit anyway\? \[.\] $'), # confirm on exit re.compile(r'(?:\r\n|[\r\n])DIAG\>'), # up to 8.0 shell re.compile(r'[\-\w+\._~()]+:.*#'), # 8.3 shell @@ -39,7 +40,6 @@ class ArborPeakflowDriver(Driver): - def __init__(self): Driver.__init__(self, 'arbor_peakflow') self.user_re = _user_re diff --git a/src/Exscript/protocols/drivers/aruba.py b/src/Exscript/protocols/drivers/aruba.py index cfbda7ad..182c5cda 100644 --- a/src/Exscript/protocols/drivers/aruba.py +++ b/src/Exscript/protocols/drivers/aruba.py @@ -23,6 +23,7 @@ """ A driver for Aruba controllers. """ + import re from .driver import Driver @@ -40,7 +41,6 @@ class ArubaDriver(Driver): - def __init__(self): Driver.__init__(self, 'aruba') self.user_re = _user_re diff --git a/src/Exscript/protocols/drivers/bigip.py b/src/Exscript/protocols/drivers/bigip.py index f60d55ad..7322e07c 100644 --- a/src/Exscript/protocols/drivers/bigip.py +++ b/src/Exscript/protocols/drivers/bigip.py @@ -23,6 +23,7 @@ """ A driver for F5 Big-IP system (TMSH SHELL). """ + import re from .driver import Driver @@ -37,7 +38,6 @@ class BigIPDriver(Driver): - def __init__(self): Driver.__init__(self, 'bigip') self.user_re = _user_re diff --git a/src/Exscript/protocols/drivers/brocade.py b/src/Exscript/protocols/drivers/brocade.py index 514196ac..c86f45c0 100644 --- a/src/Exscript/protocols/drivers/brocade.py +++ b/src/Exscript/protocols/drivers/brocade.py @@ -43,7 +43,6 @@ class BrocadeDriver(Driver): - def __init__(self): Driver.__init__(self, 'brocade') self.user_re = _user_re diff --git a/src/Exscript/protocols/drivers/cienasaos.py b/src/Exscript/protocols/drivers/cienasaos.py index caad173a..944c7e8c 100644 --- a/src/Exscript/protocols/drivers/cienasaos.py +++ b/src/Exscript/protocols/drivers/cienasaos.py @@ -23,6 +23,7 @@ """ A driver for Ciena SAOS carrier Ethernet devices. """ + import re from .driver import Driver @@ -40,7 +41,6 @@ class CienaSAOSDriver(Driver): - def __init__(self): Driver.__init__(self, 'cienasaos') self.user_re = _user_re diff --git a/src/Exscript/protocols/drivers/driver.py b/src/Exscript/protocols/drivers/driver.py index ba926532..40536481 100644 --- a/src/Exscript/protocols/drivers/driver.py +++ b/src/Exscript/protocols/drivers/driver.py @@ -23,6 +23,7 @@ """ Base class for all drivers. """ + import re import string @@ -39,23 +40,32 @@ _user = r'(?:[\w+\-]+)' _user_host = r'(?:(?:' + _user + r'\@)?' + _host + r')' _prompt_re = [ - re.compile(_prompt_start - + r'[\[\<]?' - + r'\w+' - + _user_host + r'?' - + r':?' - + _any_path + r'?' - + r'[: ]?' - + _any_path + r'?' - + r'(?:\(' + _filename + r'\))?' - + r'[\]\-]?' - + r'[#>%\$\]] ?' - + _unprintable + r'*' - + r'\Z', _flags), + re.compile( + _prompt_start + + r'[\[\<]?' + + r'\w+' + + _user_host + + r'?' + + r':?' + + _any_path + + r'?' + + r'[: ]?' + + _any_path + + r'?' + + r'(?:\(' + + _filename + + r'\))?' + + r'[\]\-]?' + + r'[#>%\$\]] ?' + + _unprintable + + r'*' + + r'\Z', + _flags, + ), ] _user_re = [re.compile(r'(user ?name|user|login): *$', _flags)] -_pass_re = [re.compile(r'password:? *$', _flags)] +_pass_re = [re.compile(r'password:? *$', _flags)] _errors = [ r'error', r'invalid', @@ -77,23 +87,24 @@ r'failure', ] _login_fail_re = [ - re.compile(_nl - + r'[^\r\n]*' - + r'(?:' + '|'.join(_login_fail) + r')', _flags), + re.compile(_nl + r'[^\r\n]*' + r'(?:' + '|'.join(_login_fail) + r')', _flags), ] _netconf_prompt_re = [ - re.compile(r'(?<=>)' # any XML closing tag (generic) - + r'[ \t\r\n]*' - + r'(?:]]>]]>|\n##\n)' # 1.0 EOM or 1.1 EOC sequence - + r'[ \t\r\n]*' - + _unprintable + r'*' - + r'\Z', _flags), + re.compile( + r'(?<=>)' # any XML closing tag (generic) + + r'[ \t\r\n]*' + + r'(?:]]>]]>|\n##\n)' # 1.0 EOM or 1.1 EOC sequence + + r'[ \t\r\n]*' + + _unprintable + + r'*' + + r'\Z', + _flags, + ), ] _netconf_error_re = [re.compile(r'<(?:[\w.\-]+:)?rpc-error[\s/>]', _flags)] class Driver: - def __init__(self, name): self.name = name self.user_re = _user_re @@ -114,9 +125,9 @@ def __repr__(self): def supports_os_guesser(self): return ( - self.check_protocol_for_os.__code__ is not Driver.check_protocol_for_os.__code__ or - self.check_head_for_os.__code__ is not Driver.check_head_for_os.__code__ or - self.check_response_for_os.__code__ is not Driver.check_response_for_os.__code__ + self.check_protocol_for_os.__code__ is not Driver.check_protocol_for_os.__code__ + or self.check_head_for_os.__code__ is not Driver.check_head_for_os.__code__ + or self.check_response_for_os.__code__ is not Driver.check_response_for_os.__code__ ) def check_protocol_for_os(self, string): diff --git a/src/Exscript/protocols/drivers/enterasys.py b/src/Exscript/protocols/drivers/enterasys.py index d9775aa0..af5ff4b5 100644 --- a/src/Exscript/protocols/drivers/enterasys.py +++ b/src/Exscript/protocols/drivers/enterasys.py @@ -23,6 +23,7 @@ """ A driver for Enterasys devices. """ + import re from .driver import Driver @@ -34,7 +35,6 @@ class EnterasysDriver(Driver): - def __init__(self): Driver.__init__(self, 'enterasys') self.user_re = _user_re diff --git a/src/Exscript/protocols/drivers/enterasys_wc.py b/src/Exscript/protocols/drivers/enterasys_wc.py index 9f33b637..b954eea8 100644 --- a/src/Exscript/protocols/drivers/enterasys_wc.py +++ b/src/Exscript/protocols/drivers/enterasys_wc.py @@ -25,6 +25,7 @@ Created using a C5110 device. """ + import re from .driver import Driver @@ -39,7 +40,6 @@ class EnterasysWCDriver(Driver): - def __init__(self): Driver.__init__(self, 'enterasys_wc') self.user_re = _user_re diff --git a/src/Exscript/protocols/drivers/eos.py b/src/Exscript/protocols/drivers/eos.py index 9fc290a0..2bd1bbf8 100644 --- a/src/Exscript/protocols/drivers/eos.py +++ b/src/Exscript/protocols/drivers/eos.py @@ -23,6 +23,7 @@ """ A driver for Arista EOS. """ + import re from .driver import Driver @@ -45,8 +46,7 @@ # % Incomplete command # % Ambiguous command -_user_re = [re.compile(r'user ?name: ?$', re.I), - re.compile(r' login: ?$', re.I)] +_user_re = [re.compile(r'user ?name: ?$', re.I), re.compile(r' login: ?$', re.I)] _password_re = [re.compile(r'(?:\r\n|[\r\n])Password: ?$', re.I)] _prompt_re = [re.compile(r'(?:\r\n|[\r\n])?[\-\w+\.:/]+(?:\([^\)]+\)){,3}[>#] ?$')] _error_re = [ @@ -71,7 +71,6 @@ class EOSDriver(Driver): - def __init__(self): Driver.__init__(self, 'eos') self.user_re = _user_re diff --git a/src/Exscript/protocols/drivers/ericsson_ban.py b/src/Exscript/protocols/drivers/ericsson_ban.py index 57d243fa..6b6bfae5 100644 --- a/src/Exscript/protocols/drivers/ericsson_ban.py +++ b/src/Exscript/protocols/drivers/ericsson_ban.py @@ -23,6 +23,7 @@ """ A driver for devices running Ericsson's Broadband Access Nodes (BAN) OS """ + import re from .driver import Driver @@ -42,7 +43,6 @@ class EricssonBanDriver(Driver): - def __init__(self): Driver.__init__(self, 'ericsson_ban') diff --git a/src/Exscript/protocols/drivers/fortios.py b/src/Exscript/protocols/drivers/fortios.py index ea72e487..21b00a54 100644 --- a/src/Exscript/protocols/drivers/fortios.py +++ b/src/Exscript/protocols/drivers/fortios.py @@ -25,6 +25,7 @@ Created using a Fortigate device and FortiOS 5.0. """ + import re from .driver import Driver @@ -54,7 +55,6 @@ class FortiOSDriver(Driver): - def __init__(self): Driver.__init__(self, 'fortios') self.user_re = _user_re diff --git a/src/Exscript/protocols/drivers/generic.py b/src/Exscript/protocols/drivers/generic.py index a7b54de1..969f0694 100644 --- a/src/Exscript/protocols/drivers/generic.py +++ b/src/Exscript/protocols/drivers/generic.py @@ -23,10 +23,10 @@ """ The default driver that is used when the OS is not recognized. """ + from .driver import Driver class GenericDriver(Driver): - def __init__(self): Driver.__init__(self, 'generic') diff --git a/src/Exscript/protocols/drivers/hp_pro_curve.py b/src/Exscript/protocols/drivers/hp_pro_curve.py index 42a52997..e439cfaa 100644 --- a/src/Exscript/protocols/drivers/hp_pro_curve.py +++ b/src/Exscript/protocols/drivers/hp_pro_curve.py @@ -23,6 +23,7 @@ """ A driver for HP ProCurve switches. """ + import re from .driver import Driver @@ -43,7 +44,6 @@ class HPProCurveDriver(Driver): - def __init__(self): Driver.__init__(self, 'hp_pro_curve') self.user_re = _user_re diff --git a/src/Exscript/protocols/drivers/icotera.py b/src/Exscript/protocols/drivers/icotera.py index 9e0e0449..a20aa1b3 100644 --- a/src/Exscript/protocols/drivers/icotera.py +++ b/src/Exscript/protocols/drivers/icotera.py @@ -23,13 +23,13 @@ """ A driver for Icotera CPE """ + import re from .driver import Driver class IcoteraDriver(Driver): - def __init__(self): """ Constructor of the IcoteraDriver. diff --git a/src/Exscript/protocols/drivers/ios.py b/src/Exscript/protocols/drivers/ios.py index e7badcbc..e7fdb92d 100644 --- a/src/Exscript/protocols/drivers/ios.py +++ b/src/Exscript/protocols/drivers/ios.py @@ -23,6 +23,7 @@ """ A driver for Cisco IOS (not IOS XR). """ + import re from ..exception import InvalidCommandException @@ -42,7 +43,6 @@ class IOSDriver(Driver): - def __init__(self): Driver.__init__(self, 'ios') self.user_re = _user_re diff --git a/src/Exscript/protocols/drivers/ios_xr.py b/src/Exscript/protocols/drivers/ios_xr.py index 0d519950..03b11727 100644 --- a/src/Exscript/protocols/drivers/ios_xr.py +++ b/src/Exscript/protocols/drivers/ios_xr.py @@ -23,6 +23,7 @@ """ A driver for Cisco IOS XR. """ + import re from .driver import Driver @@ -36,7 +37,6 @@ class IOSXRDriver(Driver): - def __init__(self): Driver.__init__(self, 'ios_xr') self.user_re = _user_re diff --git a/src/Exscript/protocols/drivers/isam.py b/src/Exscript/protocols/drivers/isam.py index 783dff07..e927d5ca 100644 --- a/src/Exscript/protocols/drivers/isam.py +++ b/src/Exscript/protocols/drivers/isam.py @@ -23,6 +23,7 @@ """ A driver for devices running ISAM (runs on Alcatel ISAM). """ + import re from .driver import Driver @@ -41,7 +42,6 @@ class IsamDriver(Driver): - def __init__(self): Driver.__init__(self, 'isam') self.user_re = _user_re diff --git a/src/Exscript/protocols/drivers/junos.py b/src/Exscript/protocols/drivers/junos.py index 42447d50..91203aee 100644 --- a/src/Exscript/protocols/drivers/junos.py +++ b/src/Exscript/protocols/drivers/junos.py @@ -23,6 +23,7 @@ """ A driver for devices running JunOS (by Juniper). """ + import re from .driver import Driver @@ -64,7 +65,6 @@ class JunOSDriver(Driver): - def __init__(self): Driver.__init__(self, 'junos') self.user_re = _user_re diff --git a/src/Exscript/protocols/drivers/junos_erx.py b/src/Exscript/protocols/drivers/junos_erx.py index f8241a38..aca78263 100644 --- a/src/Exscript/protocols/drivers/junos_erx.py +++ b/src/Exscript/protocols/drivers/junos_erx.py @@ -23,6 +23,7 @@ """ A driver for devices running Juniper ERX OS. """ + import re from .driver import Driver @@ -34,7 +35,6 @@ class JunOSERXDriver(Driver): - def __init__(self): Driver.__init__(self, 'junos_erx') self.user_re = _user_re diff --git a/src/Exscript/protocols/drivers/mrv.py b/src/Exscript/protocols/drivers/mrv.py index 3a7e418b..3b978a99 100644 --- a/src/Exscript/protocols/drivers/mrv.py +++ b/src/Exscript/protocols/drivers/mrv.py @@ -23,6 +23,7 @@ """ A driver for MRV Linux. """ + import re from .driver import Driver @@ -34,7 +35,6 @@ class MRVDriver(Driver): - def __init__(self): Driver.__init__(self, 'mrv') self.user_re = _user_re diff --git a/src/Exscript/protocols/drivers/nxos.py b/src/Exscript/protocols/drivers/nxos.py index d7792eac..ec925dfa 100644 --- a/src/Exscript/protocols/drivers/nxos.py +++ b/src/Exscript/protocols/drivers/nxos.py @@ -23,6 +23,7 @@ """ A driver for Cisco Nexus OS (NXOS). """ + import re from .driver import Driver @@ -40,7 +41,6 @@ class NXOSDriver(Driver): - def __init__(self): Driver.__init__(self, 'nxos') self.user_re = _user_re diff --git a/src/Exscript/protocols/drivers/one_os.py b/src/Exscript/protocols/drivers/one_os.py index 8cc9c0ba..157d644e 100644 --- a/src/Exscript/protocols/drivers/one_os.py +++ b/src/Exscript/protocols/drivers/one_os.py @@ -23,6 +23,7 @@ """ A driver for OneOS (OneAccess). """ + import re from .driver import Driver @@ -34,7 +35,6 @@ class OneOSDriver(Driver): - def __init__(self): Driver.__init__(self, 'one_os') self.user_re = _user_re diff --git a/src/Exscript/protocols/drivers/rios.py b/src/Exscript/protocols/drivers/rios.py index 1c53be1c..a6a3e150 100644 --- a/src/Exscript/protocols/drivers/rios.py +++ b/src/Exscript/protocols/drivers/rios.py @@ -23,6 +23,7 @@ """ A driver for Riverbed RIOS. """ + import re from .driver import Driver @@ -38,7 +39,6 @@ class RIOSDriver(Driver): - def __init__(self): Driver.__init__(self, 'rios') self.user_re = _user_re diff --git a/src/Exscript/protocols/drivers/shell.py b/src/Exscript/protocols/drivers/shell.py index 0fb9926a..5a8f6b6c 100644 --- a/src/Exscript/protocols/drivers/shell.py +++ b/src/Exscript/protocols/drivers/shell.py @@ -23,6 +23,7 @@ """ A generic shell driver that handles unknown unix shells. """ + import re from .driver import Driver @@ -33,7 +34,6 @@ class ShellDriver(Driver): - def __init__(self): Driver.__init__(self, 'shell') self.user_re = _user_re diff --git a/src/Exscript/protocols/drivers/smart_edge_os.py b/src/Exscript/protocols/drivers/smart_edge_os.py index 5065ed1d..dde18c30 100644 --- a/src/Exscript/protocols/drivers/smart_edge_os.py +++ b/src/Exscript/protocols/drivers/smart_edge_os.py @@ -23,6 +23,7 @@ """ A driver for Redback Smart Edge OS. """ + import re from .driver import Driver @@ -36,7 +37,6 @@ class SmartEdgeOSDriver(Driver): - def __init__(self): Driver.__init__(self, 'smart_edge_os') self.user_re = _user_re diff --git a/src/Exscript/protocols/drivers/sp_vxoa.py b/src/Exscript/protocols/drivers/sp_vxoa.py index d2eb986c..3dd92b8d 100644 --- a/src/Exscript/protocols/drivers/sp_vxoa.py +++ b/src/Exscript/protocols/drivers/sp_vxoa.py @@ -23,6 +23,7 @@ """ A driver for devices running VXOA (by Silver Peak). """ + import re from .driver import Driver @@ -36,7 +37,6 @@ class SilverPeakVXOADriver(Driver): - def __init__(self): Driver.__init__(self, 'sp_vxoa') self.user_re = _user_re diff --git a/src/Exscript/protocols/drivers/sros.py b/src/Exscript/protocols/drivers/sros.py index cc56095b..8551901f 100644 --- a/src/Exscript/protocols/drivers/sros.py +++ b/src/Exscript/protocols/drivers/sros.py @@ -23,6 +23,7 @@ """ A driver for Alcatel SROS. """ + import re from .driver import Driver @@ -31,7 +32,6 @@ class SROSDriver(Driver): - def __init__(self): Driver.__init__(self, 'sros') self.prompt_re = _prompt_re diff --git a/src/Exscript/protocols/drivers/vrp.py b/src/Exscript/protocols/drivers/vrp.py index 9365cb18..e20ca144 100644 --- a/src/Exscript/protocols/drivers/vrp.py +++ b/src/Exscript/protocols/drivers/vrp.py @@ -23,6 +23,7 @@ """ A driver for devices running VRP (by Huawei). """ + import re from .driver import Driver @@ -37,7 +38,6 @@ class VRPDriver(Driver): - def __init__(self): Driver.__init__(self, 'vrp') self.user_re = _user_re diff --git a/src/Exscript/protocols/drivers/vxworks.py b/src/Exscript/protocols/drivers/vxworks.py index c8789704..f0e0a92a 100644 --- a/src/Exscript/protocols/drivers/vxworks.py +++ b/src/Exscript/protocols/drivers/vxworks.py @@ -23,6 +23,7 @@ """ A driver for devices running Vxworks, which can be found on huawei5600T """ + import re from .driver import Driver @@ -43,7 +44,6 @@ class VxworksDriver(Driver): - def __init__(self): Driver.__init__(self, 'vxworks') self.user_re = _user_re diff --git a/src/Exscript/protocols/drivers/zte.py b/src/Exscript/protocols/drivers/zte.py index b651200d..5f2992ec 100644 --- a/src/Exscript/protocols/drivers/zte.py +++ b/src/Exscript/protocols/drivers/zte.py @@ -23,6 +23,7 @@ """ A driver for devices running Zte operating system. """ + import re from .driver import Driver @@ -43,7 +44,6 @@ class ZteDriver(Driver): - def __init__(self): Driver.__init__(self, 'zte') self.user_re = _user_re diff --git a/src/Exscript/protocols/drivers/zyxel.py b/src/Exscript/protocols/drivers/zyxel.py index 894e608e..4f62e40a 100644 --- a/src/Exscript/protocols/drivers/zyxel.py +++ b/src/Exscript/protocols/drivers/zyxel.py @@ -23,6 +23,7 @@ """ A driver for devices running Zyxel operating system. """ + import re from .driver import Driver @@ -34,8 +35,8 @@ _zyxel_re = re.compile(r"Copyright .* ZyXEL Communications Corp.", re.I) -class ZyxelDriver(Driver): +class ZyxelDriver(Driver): def __init__(self): Driver.__init__(self, 'zyxel') self.prompt_re = _prompt_re @@ -44,4 +45,3 @@ def check_response_for_os(self, string): if _zyxel_re.search(string): return 90 return 0 - diff --git a/src/Exscript/protocols/dummy.py b/src/Exscript/protocols/dummy.py index 6679b1ef..fd6c5269 100644 --- a/src/Exscript/protocols/dummy.py +++ b/src/Exscript/protocols/dummy.py @@ -34,7 +34,6 @@ class Dummy(Protocol): - """ This protocol adapter does not open a network connection, but talks to a :class:`Exscript.emulators.VirtualDevice` internally. diff --git a/src/Exscript/protocols/exception.py b/src/Exscript/protocols/exception.py index 4ce79f9e..d984d545 100644 --- a/src/Exscript/protocols/exception.py +++ b/src/Exscript/protocols/exception.py @@ -26,69 +26,66 @@ class ProtocolException(Exception): - """ Default exception that is thrown on most protocol related errors. """ + pass class TimeoutException(ProtocolException): - """ An exception that is thrown if the connected host did not respond for too long. """ + pass class ExpectCancelledException(ProtocolException): - """ An exception that is thrown if Protocol.cancel_expect() was called. """ + pass class DriverReplacedException(ProtocolException): - """ An exception that is thrown if the protocol driver was switched during a call to expect(). """ + pass class LoginFailure(ProtocolException): - """ An exception that is thrown if the response of a connected host looked like it was trying to signal a login error during the authentication procedure. """ + pass class InvalidCommandException(ProtocolException): - """ An exception that is thrown if the response of a connected host contained a string that looked like an error. """ + pass class RPCError(InvalidCommandException): - """ An exception that is raised when a NETCONF rpc-reply contains one or more rpc-error elements with error severity. """ - def __init__( - self, message, *, element, type, tag, severity, app_tag='', path='', info='' - ): + def __init__(self, message, *, element, type, tag, severity, app_tag='', path='', info=''): """ :param message: The primary rpc-error message (error-message). :param element: The XML element of the rpc-error node. diff --git a/src/Exscript/protocols/netconf.py b/src/Exscript/protocols/netconf.py index 9eaeac6a..eb84a679 100644 --- a/src/Exscript/protocols/netconf.py +++ b/src/Exscript/protocols/netconf.py @@ -23,6 +23,7 @@ """ SSH2 Netconf support, based on ncclient and paramiko. """ + import functools import inspect import logging @@ -55,7 +56,6 @@ class Netconf(SSH2): - """ The SSH Netconf protocol adapter, based on the SSH2 adapter and ncclient. """ @@ -169,9 +169,7 @@ def _paramiko_channel(self): try: self._channel = self._transport.open_session() self._channel.invoke_subsystem(subname) - self._channel.set_name( - "%s-subsystem-%s" % (subname, self._channel.get_id()) - ) + self._channel.set_name("%s-subsystem-%s" % (subname, self._channel.get_id())) except SSHException as e: msg = 'Failed to open a channel with %s subsystem. (%s)' self._dbg(1, msg % (repr(subname), e)) @@ -198,7 +196,7 @@ def _init_ncclient(self): session = SSHSession(device_handler) session._host = self.host session._transport = self._transport - session._channel = self._channel + session._channel = self._channel session._channel_id = self._channel.get_id() session._channel_name = self._channel.get_name() session._connected = True @@ -232,7 +230,7 @@ def recv_wrapper(*args, **kwargs): # Setup the channel send wrapper. This is needed as Netconf does not # echo the sent data back to the client known from CLI shells. send = self._channel.send - param = next(iter(inspect.signature(send).parameters)) # first param + param = next(iter(inspect.signature(send).parameters)) # first param @functools.wraps(send) def send_wrapper(*args, **kwargs): @@ -341,15 +339,15 @@ def reassemble_chunks(response): assembled = '' while match: size = int(match.group(1)) - response = response[match.end():] # consume the chunk header - assembled += response[:size] # extract exactly size chars of content - response = response[size:] # consume the chunk content + response = response[match.end() :] # consume the chunk header + assembled += response[:size] # extract exactly size chars of content + response = response[size:] # consume the chunk content match = _chunk_header_re.match(response) if response: # Internally this would be already consumed as the prompt. match = _end_of_chunks_re.match(response) if match: - response = response[match.end():] # consume the end-of-chunks marker + response = response[match.end() :] # consume the end-of-chunks marker if response: # This should only happen if we have internal issues with the prompt # or buffer handling before this reassembly is called. ncclient diff --git a/src/Exscript/protocols/osguesser.py b/src/Exscript/protocols/osguesser.py index f97c8348..c484f133 100644 --- a/src/Exscript/protocols/osguesser.py +++ b/src/Exscript/protocols/osguesser.py @@ -33,7 +33,6 @@ class OsGuesser: - """ The OsGuesser monitors everything that happens on a Protocol, and attempts to collect data out of the network activity. It watches for specific patterns in the @@ -85,9 +84,11 @@ def set(self, key, value, confidence=100): old_value, old_confidence = self._info.get(key, (None, None)) if old_confidence is not None and old_confidence >= confidence: return - self._dbg(2, "Higher confidence for %s: %s with %s -> %s with %s" % ( - key, old_value, old_confidence, value, confidence - )) + self._dbg( + 2, + "Higher confidence for %s: %s with %s -> %s with %s" + % (key, old_value, old_confidence, value, confidence), + ) self._info[key] = (value, confidence) def set_from_match(self, key, regex_list, string): @@ -141,14 +142,14 @@ def check_received_data(self, data, authenticated): if self.get('os', 80) in ('unknown', None): self._dbg(3, "Matching received data: %r" % self._response_buffer) self.set_from_match('os', self._response_os_map, self._response_buffer) - self._response_buffer = self._response_buffer[-self._buffer_limit:] + self._response_buffer = self._response_buffer[-self._buffer_limit :] return self._auth_buffer += data self._dbg(3, "Matching authentication buffer: %r" % self._auth_buffer) self.set_from_match('os', self._auth_os_map, self._auth_buffer) self.set_from_match('os', self._response_os_map, self._auth_buffer) - self._auth_buffer = self._auth_buffer[-self._buffer_limit:] + self._auth_buffer = self._auth_buffer[-self._buffer_limit :] def check_protocol_info(self, data): """ @@ -159,7 +160,7 @@ def check_protocol_info(self, data): self._protocol_buffer += data self._dbg(3, "Matching protocol info: %r" % self._protocol_buffer) self.set_from_match('os', self._protocol_os_map, self._protocol_buffer) - self._protocol_buffer = self._protocol_buffer[-self._buffer_limit:] + self._protocol_buffer = self._protocol_buffer[-self._buffer_limit :] def check_netconf_capabilities(self, capabilities): """ diff --git a/src/Exscript/protocols/protocol.py b/src/Exscript/protocols/protocol.py index e34de863..39b8384e 100644 --- a/src/Exscript/protocols/protocol.py +++ b/src/Exscript/protocols/protocol.py @@ -59,6 +59,7 @@ try: import termios import tty + _have_termios = True except ImportError: _have_termios = False @@ -67,11 +68,11 @@ _logger.addHandler(logging.NullHandler()) _ansi_escape_re = re.compile( - r'\x1b\[' # CSI: ESC[ + r'\x1b\[' # CSI: ESC[ + r'[\x30-\x3f]*' # PARAMETER: 0–9:;<=>? + r'[\x20-\x2f]*' # INTERMEDIATE: !"#$%&'( )*+,-./ - + r'[\x40-\x7e]' # FINAL: @A–Z[\]^_`a–z{|}~ - + r'|\x1b' # alternate lone ESC + + r'[\x40-\x7e]' # FINAL: @A–Z[\]^_`a–z{|}~ + + r'|\x1b' # alternate lone ESC ) _skey_re = re.compile(r'(?:s\/key|otp-md4) (\d+) (\S+)(?=\s|[\r\n])') @@ -88,7 +89,6 @@ class Protocol: - """ This is the base class for all protocols; it defines the common portions of the API. @@ -154,22 +154,24 @@ class Protocol: _log: IO[str] | None # managed by the logfile property - def __init__(self, - driver=None, - stdout=None, - stderr=None, - debug=0, - connect_rate_limit=60, - connect_timeout=30, - timeout=30, - logfile=None, - termtype='dumb', - disabled_algorithms=None, # SSH2 only - preferred_authentications=None, # SSH2 only - verify_fingerprint=True, # SSH2 only - account_factory=None, - banner_timeout=20, # SSH2 only - encoding='latin-1'): + def __init__( + self, + driver=None, + stdout=None, + stderr=None, + debug=0, + connect_rate_limit=60, + connect_timeout=30, + timeout=30, + logfile=None, + termtype='dumb', + disabled_algorithms=None, # SSH2 only + preferred_authentications=None, # SSH2 only + verify_fingerprint=True, # SSH2 only + account_factory=None, + banner_timeout=20, # SSH2 only + encoding='latin-1', + ): """ Constructor. @@ -781,9 +783,7 @@ def get_protocol(self): """ return self._name - def get_response( - self, *, strip_command=True, strip_chars='\x00\x07\x08', strip_ansi=True - ): + def get_response(self, *, strip_command=True, strip_chars='\x00\x07\x08', strip_ansi=True): """ Returns a clean copy of the latest plaintext response. @@ -926,9 +926,7 @@ def connect(self, hostname=None, port=None): # Check whether a better driver is found based on the incoming data. self._os_guesser.check_protocol_info(self.get_remote_version() or '') - self._os_guesser.check_received_data( - self.get_banner() or '', self.is_app_authenticated() - ) + self._os_guesser.check_received_data(self.get_banner() or '', self.is_app_authenticated()) self._update_auto_driver() return self._connected @@ -1049,11 +1047,7 @@ def is_protocol_authenticated(self): """ return self._proto_authenticated - def _app_authenticate(self, - account, - password, - flush=True, - bailout=False): + def _app_authenticate(self, account, password, flush=True, bailout=False): """ Generic implementation for app-level authentications or authorizations. """ @@ -1064,11 +1058,13 @@ def _app_authenticate(self, # to find out which type of prompt was matched, so we build a # structure to allow for mapping the match index back to the # prompt type. - prompts = (('login-error', self.get_login_error_prompt()), - ('username', self.get_username_prompt()), - ('skey', [_skey_re]), - ('password', self.get_password_prompt()), - ('cli', self.get_prompt())) + prompts = ( + ('login-error', self.get_login_error_prompt()), + ('username', self.get_username_prompt()), + ('skey', [_skey_re]), + ('password', self.get_password_prompt()), + ('cli', self.get_prompt()), + ) prompt_map = [] prompt_list = [] for section, section_prompts in prompts: @@ -1541,10 +1537,12 @@ def _open_posix_shell(self, channel, key_handlers, handle_window_size): # Update the terminal size whenever the size changes. if handle_window_size: + def handle_sigwinch(signum, frame): rows, cols = get_terminal_size() self._dbg(2, "Trying to update terminal size: %dx%d" % (rows, cols)) self._set_terminal_size(rows, cols) + signal.signal(signal.SIGWINCH, handle_sigwinch) handle_sigwinch(None, None) @@ -1614,9 +1612,7 @@ def writeall(sock): # NOTE: Thread in windows might need a daemon=True to avoid hanging on exit. # Needs to be analyzed on next possible testing with a windows box. - writer = threading.Thread( - name="channel.recv", target=writeall, args=(channel,) - ) + writer = threading.Thread(name="channel.recv", target=writeall, args=(channel,)) writer.start() try: @@ -1678,23 +1674,23 @@ def _capture_device_context(self): """Captures all context-specific state into a plain dict snapshot.""" return { # Authentication - 'app_authenticated': self._app_authenticated, - 'app_authorized': self._app_authorized, - 'last_account': self._last_account, + 'app_authenticated': self._app_authenticated, + 'app_authorized': self._app_authorized, + 'last_account': self._last_account, # OS guesser - 'guesser_info': dict(self._os_guesser._info), - 'guesser_auth_buffer': self._os_guesser._auth_buffer, + 'guesser_info': dict(self._os_guesser._info), + 'guesser_auth_buffer': self._os_guesser._auth_buffer, 'guesser_protocol_buffer': self._os_guesser._protocol_buffer, 'guesser_response_buffer': self._os_guesser._response_buffer, # Driver - 'auto_driver': self._auto_driver, - 'manual_driver': self._manual_driver, + 'auto_driver': self._auto_driver, + 'manual_driver': self._manual_driver, # Prompts - 'manual_prompt_re': self._manual_prompt_re, - 'manual_error_re': self._manual_error_re, - 'manual_user_re': self._manual_user_re, - 'manual_password_re': self._manual_password_re, - 'manual_login_error_re': self._manual_login_error_re, + 'manual_prompt_re': self._manual_prompt_re, + 'manual_error_re': self._manual_error_re, + 'manual_user_re': self._manual_user_re, + 'manual_password_re': self._manual_password_re, + 'manual_login_error_re': self._manual_login_error_re, } def switch_device_context(self, *, reset_auth=True): @@ -1760,24 +1756,24 @@ def switch_device_context(self, *, reset_auth=True): def _restore_device_context(self, snapshot): """Restores all context-specific state from a snapshot dict.""" # Authentication - self._app_authenticated = snapshot['app_authenticated'] - self._app_authorized = snapshot['app_authorized'] - self._last_account = snapshot['last_account'] + self._app_authenticated = snapshot['app_authenticated'] + self._app_authorized = snapshot['app_authorized'] + self._last_account = snapshot['last_account'] # OS guesser - self._os_guesser._info = dict(snapshot['guesser_info']) - self._os_guesser._auth_buffer = snapshot['guesser_auth_buffer'] + self._os_guesser._info = dict(snapshot['guesser_info']) + self._os_guesser._auth_buffer = snapshot['guesser_auth_buffer'] self._os_guesser._protocol_buffer = snapshot['guesser_protocol_buffer'] self._os_guesser._response_buffer = snapshot['guesser_response_buffer'] # Driver - self._auto_driver = snapshot['auto_driver'] - self._manual_driver = snapshot['manual_driver'] + self._auto_driver = snapshot['auto_driver'] + self._manual_driver = snapshot['manual_driver'] self._update_auto_driver() # should only trigger notifications # Prompts - self._manual_prompt_re = snapshot['manual_prompt_re'] - self._manual_error_re = snapshot['manual_error_re'] - self._manual_user_re = snapshot['manual_user_re'] - self._manual_password_re = snapshot['manual_password_re'] - self._manual_login_error_re = snapshot['manual_login_error_re'] + self._manual_prompt_re = snapshot['manual_prompt_re'] + self._manual_error_re = snapshot['manual_error_re'] + self._manual_user_re = snapshot['manual_user_re'] + self._manual_password_re = snapshot['manual_password_re'] + self._manual_login_error_re = snapshot['manual_login_error_re'] def restore_device_context(self): """ diff --git a/src/Exscript/protocols/ssh2.py b/src/Exscript/protocols/ssh2.py index d3b94e77..72a84b7f 100644 --- a/src/Exscript/protocols/ssh2.py +++ b/src/Exscript/protocols/ssh2.py @@ -59,17 +59,19 @@ for key in keymap: PrivateKey.keytypes.add(key) -auth_types = {'publickey': ('_paramiko_auth_agent', '_paramiko_auth_autokey'), - 'keyboard-interactive': ('_paramiko_auth_interactive',), - 'password': ('_paramiko_auth_password',)} +auth_types = { + 'publickey': ('_paramiko_auth_agent', '_paramiko_auth_autokey'), + 'keyboard-interactive': ('_paramiko_auth_interactive',), + 'password': ('_paramiko_auth_password',), +} class SSH2(Protocol): - """ The SSH2 protocol adapter, based on Paramiko. """ - KEEPALIVE_INTERVAL = 2.5 * 60 # Two and a half minutes + + KEEPALIVE_INTERVAL = 2.5 * 60 # Two and a half minutes def __init__(self, **kwargs): Protocol.__init__(self, **kwargs) @@ -212,6 +214,7 @@ def _paramiko_auth_none(self, username, password=None): def _paramiko_auth_interactive(self, username, password=None): if password is None: return + def handler(title, instructions, prompt_list): if not prompt_list: return [] @@ -237,6 +240,7 @@ def handler(title, instructions, prompt_list): response.append(password) break return response + self._transport.auth_interactive(username, handler) def _paramiko_auth_password(self, username, password): @@ -248,8 +252,7 @@ def _paramiko_auth_agent(self, username, password=None): if not keys: raise AuthenticationException('auth agent found no keys') - saved_exception = AuthenticationException( - 'Failed to authenticate with given username') + saved_exception = AuthenticationException('Failed to authenticate with given username') for key in keys: try: @@ -266,7 +269,8 @@ def _paramiko_auth_key(self, username, keys, password): password = '' saved_exception = AuthenticationException( - 'Failed to authenticate with given username and password/key') + 'Failed to authenticate with given username and password/key' + ) for pkey_class, filename in keys: try: @@ -283,12 +287,14 @@ def _paramiko_auth_key(self, username, keys, password): def _paramiko_auth_autokey(self, username, password): keyfiles = [] - for cls, file in ((paramiko.RSAKey, '~/.ssh/id_rsa'), # Unix - (paramiko.ECDSAKey, '~/.ssh/id_ecdsa'), # Unix - (paramiko.Ed25519Key, '~/.ssh/id_ed25519'), # Unix - (paramiko.RSAKey, '~/ssh/id_rsa'), # Windows - (paramiko.ECDSAKey, '~/ssh/id_ecdsa'), # Windows - (paramiko.Ed25519Key, '~/ssh/id_ed25519')): # Windows + for cls, file in ( + (paramiko.RSAKey, '~/.ssh/id_rsa'), # Unix + (paramiko.ECDSAKey, '~/.ssh/id_ecdsa'), # Unix + (paramiko.Ed25519Key, '~/.ssh/id_ed25519'), # Unix + (paramiko.RSAKey, '~/ssh/id_rsa'), # Windows + (paramiko.ECDSAKey, '~/ssh/id_ecdsa'), # Windows + (paramiko.Ed25519Key, '~/ssh/id_ed25519'), + ): # Windows file = os.path.expanduser(file) if os.path.isfile(file): keyfiles.append((cls, file)) @@ -327,7 +333,9 @@ def _paramiko_auth(self, username, password): self._dbg(2, 'Authenticating with client.auth_none().') self._transport.auth_none(username) except BadAuthenticationType as err: - self._dbg(2, 'Authentication with client.auth_none() failed, allowed: %s' % err.allowed_types) + self._dbg( + 2, 'Authentication with client.auth_none() failed, allowed: %s' % err.allowed_types + ) auth_methods = self._get_auth_methods(err.allowed_types) else: return @@ -344,10 +352,7 @@ def _paramiko_auth(self, username, password): # Some OSes (e.g. JunOS ERX OS, Huawei) do not accept further login # attempts after failing one. So in this hack, we # re-connect after each attempt... - if ( - self.get_driver().reconnect_between_auth_methods - or not self._transport.active - ): + if self.get_driver().reconnect_between_auth_methods or not self._transport.active: if self.get_driver().reconnect_between_auth_methods: self._dbg(1, 'Forced reconnect by driver.') else: @@ -464,9 +469,7 @@ def _domatch_buffer(self, prompt): driver = self.get_driver() search_window_size = 150 search_window = self._buffer.tail(search_window_size) - search_window, incomplete_tail = driver.clean_response_for_re_match( - search_window - ) + search_window, incomplete_tail = driver.clean_response_for_re_match(search_window) n, match = -1, None for n, regex in enumerate(prompt): match = regex.search(search_window) diff --git a/src/Exscript/protocols/telnet.py b/src/Exscript/protocols/telnet.py index 3fdb8f1e..af1ae2fb 100644 --- a/src/Exscript/protocols/telnet.py +++ b/src/Exscript/protocols/telnet.py @@ -23,6 +23,7 @@ """ The Telnet protocol. """ + import logging from ..util.tty import get_terminal_size @@ -34,7 +35,6 @@ class Telnet(Protocol): - """ The Telnet protocol adapter. """ @@ -85,8 +85,7 @@ def _domatch(self, prompt, flush): # Wait for a prompt. cleanup_callback = self.get_driver().clean_response_for_re_match try: - result, match, self.response = func( - prompt, self.timeout, cleanup=cleanup_callback) + result, match, self.response = func(prompt, self.timeout, cleanup=cleanup_callback) except Exception: self._dbg(1, 'Error while waiting for ' + repr(prompt)) raise diff --git a/src/Exscript/protocols/telnetlib.py b/src/Exscript/protocols/telnetlib.py index cfbe9d25..dc55e4e6 100644 --- a/src/Exscript/protocols/telnetlib.py +++ b/src/Exscript/protocols/telnetlib.py @@ -158,8 +158,8 @@ else: _TelnetSelector = selectors.SelectSelector -class Telnet: +class Telnet: """Telnet interface class. An instance of this class represents a connection to a telnet @@ -311,7 +311,7 @@ def write(self, buffer): elif not isinstance(buffer, bytes): buffer = buffer.encode(self.encoding) if IAC in buffer: - buffer = buffer.replace(IAC, IAC+IAC) + buffer = buffer.replace(IAC, IAC + IAC) self.msg("Send: %s", repr(buffer)) self.sock.send(buffer) @@ -325,7 +325,7 @@ def read_until(self, match, timeout=None): self.process_rawq() i = self.rawq.find(match) if i >= 0: - i = i+n + i = i + n buf = self.rawq[:i] self.rawq = self.rawq[i:] return buf @@ -335,12 +335,12 @@ def read_until(self, match, timeout=None): selector.register(self, selectors.EVENT_READ) while not self.eof: if selector.select(timeout): - i = max(0, len(self.rawq)-n) + i = max(0, len(self.rawq) - n) self.fill_rawq() self.process_rawq() i = self.rawq.find(match, i) if i >= 0: - i = i+n + i = i + n buf = self.rawq[:i] self.rawq = self.rawq[i:] return buf @@ -482,14 +482,14 @@ def process_rawq(self): continue self.msg('IAC DO %s', ord(opt)) if opt == TTYPE: - self.sock.send(IAC+WILL+opt) + self.sock.send(IAC + WILL + opt) elif opt == NAWS: - self.sock.send(IAC+WILL+opt) + self.sock.send(IAC + WILL + opt) self.can_naws = True if self.window_size: self.set_window_size(*self.window_size) else: - self.sock.send(IAC+WONT+opt) + self.sock.send(IAC + WONT + opt) # DON'T: Indicates the demand that the other party stop # performing, or confirmation that you are no longer @@ -498,7 +498,7 @@ def process_rawq(self): elif command == DONT: opt = self.rawq_getchar() self.msg('IAC DONT %s', ord(opt)) - self.sock.send(IAC+WONT+opt) + self.sock.send(IAC + WONT + opt) # SB: Indicates that what follows is subnegotiation of the # indicated option. @@ -526,23 +526,20 @@ def process_rawq(self): end = self.rawq_getchar() if (iac, end) != (IAC, SE): # whoops, that's an unexpected response... - self.msg( - 'Expected IAC SE, but got %d %d.', ord(iac), ord(end) - ) + self.msg('Expected IAC SE, but got %d %d.', ord(iac), ord(end)) self.msg('IAC SUBCOMMAND_END') # Send the next supported terminal. ttype = self.termtype.encode('latin-1') self.msg('Indicating support for terminal type %s.', ttype) - self.sock.send(IAC+SB+TTYPE+theNULL+ttype+IAC+SE) + self.sock.send(IAC + SB + TTYPE + theNULL + ttype + IAC + SE) elif command in (WILL, WONT): opt = self.rawq_getchar() - self.msg('IAC %s %d', - command == WILL and 'WILL' or 'WONT', ord(opt)) + self.msg('IAC %s %d', command == WILL and 'WILL' or 'WONT', ord(opt)) if opt == ECHO: - self.sock.send(IAC+DO+opt) + self.sock.send(IAC + DO + opt) else: - self.sock.send(IAC+DONT+opt) + self.sock.send(IAC + DONT + opt) else: self.msg('IAC %d not recognized.' % ord(command)) except EOFError: # raised by self.rawq_getchar() @@ -565,7 +562,7 @@ def rawq_getchar(self): raise EOFError c = self.rawq[self.irawq] - c = c.to_bytes((c.bit_length()+7)//8, 'big') + c = c.to_bytes((c.bit_length() + 7) // 8, 'big') self.irawq += 1 if self.irawq >= len(self.rawq): @@ -587,7 +584,7 @@ def fill_rawq(self): # behavior in process_rawq() above. buf = self.sock.recv(64) self.msg("Recv: %s", repr(buf)) - self.eof = (not buf) + self.eof = not buf self.rawq = self.rawq + buf def sock_avail(self): @@ -619,6 +616,7 @@ def interact(self): def mt_interact(self): """Multithreaded version of interact().""" import _thread + _thread.start_new_thread(self.listener, ()) while 1: line = sys.stdin.readline() @@ -660,7 +658,7 @@ def _waitfor(self, relist, timeout=None, flush=False, cleanup=None): import re relist[i] = re.compile(relist[i]) self.msg("Expecting: %s" % [l.pattern for l in relist]) - incomplete_tail = u'' + incomplete_tail = '' clean_sw_size = search_window_size while True: self.process_rawq() @@ -671,7 +669,7 @@ def _waitfor(self, relist, timeout=None, flush=False, cleanup=None): qlen = self.cookedq.tell() if cleanup: while True: - pos = qlen-clean_sw_size-len(incomplete_tail)-head_loockback_size + pos = qlen - clean_sw_size - len(incomplete_tail) - head_loockback_size self.cookedq.seek(max(0, pos)) search_window = self.cookedq.read() search_window, incomplete_tail = cleanup(search_window) @@ -697,14 +695,16 @@ def _waitfor(self, relist, timeout=None, flush=False, cleanup=None): if flush: self.cookedq.seek(0) self.cookedq.truncate() - self.cookedq.write(search_window[m.end():]) + self.cookedq.write(search_window[m.end() :]) else: self.cookedq.seek(qlen) return i, m, text if self.eof: break if timeout is not None: - if not self._wait_for_data(timeout): # Workaround for the problem with select() below. + if not self._wait_for_data( + timeout + ): # Workaround for the problem with select() below. break # The following will sometimes lock even if data is available # and I have no idea why. Do NOT reverse this unless you are sure @@ -777,5 +777,6 @@ def test(): tn.interact() tn.close() + if __name__ == '__main__': test() diff --git a/src/Exscript/queue.py b/src/Exscript/queue.py index c3eaddcd..d9d3db8a 100644 --- a/src/Exscript/queue.py +++ b/src/Exscript/queue.py @@ -23,6 +23,7 @@ """ The heart of Exscript. """ + import gc import logging import os @@ -67,6 +68,7 @@ def _prepare_connection(func): A decorator that unpacks the host and connection from the job argument and passes them as separate arguments to the wrapped function. """ + def _wrapped(job, *args, **kwargs): job_id = id(job) to_parent = job.data['pipe'] @@ -74,8 +76,7 @@ def _wrapped(job, *args, **kwargs): # Create a protocol adapter. mkaccount = partial(_account_factory, to_parent, host) - pargs = {'account_factory': mkaccount, - 'stdout': job.data['stdout']} + pargs = {'account_factory': mkaccount, 'stdout': job.data['stdout']} pargs.update(host.get_options()) conn = prepare(host, **pargs) @@ -128,7 +129,6 @@ def _call_logger(funcname, logger_id, *args): class _PipeHandler(threading.Thread): - """ Each PipeHandler holds an open pipe to a subprocess, to allow the sub-process to access the accounts and communicate status information. @@ -144,11 +144,13 @@ def _send_account(self, account): if account is None: self.to_child.send(account) return - response = (account.__hash__(), - account.get_name(), - account.get_password(), - account.get_authorization_password(), - account.get_key()) + response = ( + account.__hash__(), + account.get_name(), + account.get_password(), + account.get_authorization_password(), + account.get_key(), + ) self.to_child.send(response) def _handle_request(self, request): @@ -195,20 +197,21 @@ def run(self): class Queue: - """ Manages hosts/tasks, accounts, connections, and threads. """ - def __init__(self, - domain='', - verbose=1, - mode='threading', - max_threads=1, - host_driver=None, - exc_cb=None, - stdout=sys.stdout, - stderr=sys.stderr): + def __init__( + self, + domain='', + verbose=1, + mode='threading', + max_threads=1, + host_driver=None, + exc_cb=None, + stdout=sys.stdout, + stderr=sys.stderr, + ): """ Constructor. All arguments should be passed as keyword arguments. Depending on the verbosity level, the following types @@ -655,10 +658,7 @@ def run_or_ignore(self, hosts, function, attempts=1): :rtype: object :return: A task object, or None if all hosts were duplicates. """ - return self._run(hosts, - function, - self.workqueue.enqueue_or_ignore, - attempts) + return self._run(hosts, function, self.workqueue.enqueue_or_ignore, attempts) def priority_run(self, hosts, function, attempts=1): """ @@ -673,11 +673,7 @@ def priority_run(self, hosts, function, attempts=1): :rtype: object :return: An object representing the task. """ - return self._run(hosts, - function, - self.workqueue.priority_enqueue, - False, - attempts) + return self._run(hosts, function, self.workqueue.priority_enqueue, False, attempts) def priority_run_or_raise(self, hosts, function, attempts=1): """ @@ -694,11 +690,7 @@ def priority_run_or_raise(self, hosts, function, attempts=1): :rtype: object :return: A task object, or None if all hosts were duplicates. """ - return self._run(hosts, - function, - self.workqueue.priority_enqueue_or_raise, - False, - attempts) + return self._run(hosts, function, self.workqueue.priority_enqueue_or_raise, False, attempts) def force_run(self, hosts, function, attempts=1): """ @@ -714,11 +706,7 @@ def force_run(self, hosts, function, attempts=1): :rtype: object :return: An object representing the task. """ - return self._run(hosts, - function, - self.workqueue.priority_enqueue, - True, - attempts) + return self._run(hosts, function, self.workqueue.priority_enqueue, True, attempts) def enqueue(self, function, name=None, attempts=1): """ diff --git a/src/Exscript/servers/__init__.py b/src/Exscript/servers/__init__.py index a5244dfa..577b1c18 100644 --- a/src/Exscript/servers/__init__.py +++ b/src/Exscript/servers/__init__.py @@ -23,11 +23,15 @@ """ Very simple servers, useful for emulating a device for testing. """ + import inspect from .httpd import HTTPd from .sshd import SSHd from .telnetd import Telnetd -__all__ = [name for name, obj in list(locals().items()) - if not (name.startswith('_') or inspect.ismodule(obj))] +__all__ = [ + name + for name, obj in list(locals().items()) + if not (name.startswith('_') or inspect.ismodule(obj)) +] diff --git a/src/Exscript/servers/httpd.py b/src/Exscript/servers/httpd.py index e704417a..cab08225 100644 --- a/src/Exscript/servers/httpd.py +++ b/src/Exscript/servers/httpd.py @@ -23,6 +23,7 @@ """ A threaded HTTP server with support for HTTP/Digest authentication. """ + import hashlib import logging import time @@ -40,11 +41,12 @@ def md5hex(x): return hashlib.md5(x).hexdigest() + default_realm = 'exscript' # This is convoluted because there's no way to tell 2to3 to insert a # byte literal. -_HEADER_NEWLINES = [x.encode('ascii') for x in (u'\r\n', u'\n', u'')] +_HEADER_NEWLINES = [x.encode('ascii') for x in ('\r\n', '\n', '')] def _parse_url(path): @@ -69,12 +71,11 @@ def _parse_url(path): def _error_401(handler, msg): handler.send_response(401) realm = handler.server.realm - nonce = (u"%d:%s" % (time.time(), realm)).encode('utf8') - handler.send_header('WWW-Authenticate', - 'Digest realm="%s",' - 'qop="auth",' - 'algorithm="MD5",' - 'nonce="%s"' % (realm, nonce)) + nonce = ("%d:%s" % (time.time(), realm)).encode('utf8') + handler.send_header( + 'WWW-Authenticate', + 'Digest realm="%s",qop="auth",algorithm="MD5",nonce="%s"' % (realm, nonce), + ) handler.end_headers() handler.rfile.read() handler.rfile.close() @@ -91,9 +92,9 @@ def wrapped(self): if self.authenticated: return func(self) - auth = self.headers.get(u'Authorization') + auth = self.headers.get('Authorization') if auth is None: - msg = u"You are not allowed to access this page. Please login first!" + msg = "You are not allowed to access this page. Please login first!" return _error_401(self, msg) token, fields = auth.split(' ', 1) @@ -103,7 +104,7 @@ def wrapped(self): # Check the header fields of the request. cred = parse_http_list(fields) cred = parse_keqv_list(cred) - keys = u'realm', u'username', u'nonce', u'uri', u'response' + keys = 'realm', 'username', 'nonce', 'uri', 'response' if not all(cred.get(key) for key in keys): return _error_401(self, 'Incomplete authentication header') if cred['realm'] != self.server.realm: @@ -118,20 +119,16 @@ def wrapped(self): return _error_401(self, 'Invalid username or password') # Check the digest string. - location = u'%s:%s' % (self.command, self.path) + location = '%s:%s' % (self.command, self.path) location = md5hex(location.encode('utf8')) pwhash = md5hex('%s:%s:%s' % (username, self.server.realm, password)) if 'qop' in cred: - info = (cred['nonce'], - cred['nc'], - cred['cnonce'], - cred['qop'], - location) + info = (cred['nonce'], cred['nc'], cred['cnonce'], cred['qop'], location) else: info = cred['nonce'], location - expect = u'%s:%s' % (pwhash, ':'.join(info)) + expect = '%s:%s' % (pwhash, ':'.join(info)) expect = md5hex(expect.encode('utf8')) if expect != cred['response']: return _error_401(self, 'Invalid username or password') @@ -144,7 +141,6 @@ def wrapped(self): class HTTPd(ThreadingMixIn, HTTPServer): - """ An HTTP server, derived from Python's HTTPServer but with added support for HTTP/Digest. Usage:: @@ -161,6 +157,7 @@ def handle_GET(self): print('started httpserver...') server.serve_forever() """ + daemon_threads = True def __init__(self, addr, handler_cls, user_data=None): @@ -208,7 +205,6 @@ def _dbg(self, msg, exc_info=None): class RequestHandler(BaseHTTPRequestHandler): - """ A drop-in replacement for Python's BaseHTTPRequestHandler that handles HTTP/Digest. @@ -224,7 +220,7 @@ def _do_POSTGET(self, handler): # Extract POST data, if any. Clumsy syntax due to Python 2 and # 2to3's lack of a byte literal. - self.data = u"".encode() + self.data = "".encode() length = self.headers.get('Content-Length') if length and length.isdigit(): self.data = self.rfile.read(int(length)) @@ -284,6 +280,7 @@ def send_response(self, code): BaseHTTPRequestHandler.send_response(self, code) self.send_header("Connection", "close") + if __name__ == '__main__': try: server = HTTPd(('', 8123), RequestHandler) diff --git a/src/Exscript/servers/server.py b/src/Exscript/servers/server.py index e4a18244..0e4571b9 100644 --- a/src/Exscript/servers/server.py +++ b/src/Exscript/servers/server.py @@ -23,6 +23,7 @@ """ Base class for all servers. """ + import logging import select import socket @@ -38,7 +39,6 @@ # switch to threaded servers too. This will hopefully add more stability to the # test suite. class Server(threading.Thread): - """ Base class of the Telnet and SSH servers. Servers are intended to be used for tests and attempt to emulate a device using the behavior of @@ -71,7 +71,7 @@ def __init__(self, host, port, device, encoding='utf8'): self.name = '%s-%s:%s' % (self._get_server_type(), host, port) self.host = host self.port = int(port) - self.timeout = .5 + self.timeout = 0.5 self.dbg = 0 self.running = False self.socket = None @@ -131,9 +131,7 @@ def _run(self): proc = threading.Thread( target=self._handle_connection, args=(conn,), - name='%s-handler-%s:%s' % ( - self._get_server_type(), addr[0], addr[1] - ), + name='%s-handler-%s:%s' % (self._get_server_type(), addr[0], addr[1]), daemon=True, ) self.processes.append(proc) diff --git a/src/Exscript/servers/sshd.py b/src/Exscript/servers/sshd.py index 5462a6d8..cd105a62 100644 --- a/src/Exscript/servers/sshd.py +++ b/src/Exscript/servers/sshd.py @@ -23,6 +23,7 @@ """ An SSH2 server. """ + import base64 import logging import os @@ -92,19 +93,13 @@ def check_channel_shell_request(self, channel): self.event.set() return True - def check_channel_pty_request(self, - channel, - term, - width, - height, - pixelwidth, - pixelheight, - modes): + def check_channel_pty_request( + self, channel, term, width, height, pixelwidth, pixelheight, modes + ): return True class SSHd(Server): - """ A SSH2 server. Usage:: diff --git a/src/Exscript/servers/telnetd.py b/src/Exscript/servers/telnetd.py index e88ba8e1..8dbaa2dd 100644 --- a/src/Exscript/servers/telnetd.py +++ b/src/Exscript/servers/telnetd.py @@ -23,6 +23,7 @@ """ A Telnet server. """ + import logging import select import socket @@ -34,7 +35,6 @@ class Telnetd(Server): - """ A Telnet server. Usage:: diff --git a/src/Exscript/stdlib/__init__.py b/src/Exscript/stdlib/__init__.py index 61ed369d..3268cf5c 100644 --- a/src/Exscript/stdlib/__init__.py +++ b/src/Exscript/stdlib/__init__.py @@ -23,54 +23,54 @@ from . import connection, crypt, file, ipv4, list, mysys, string functions = { - 'connection.add_error': connection.add_error, - 'connection.add_prompt': connection.add_prompt, - 'connection.authenticate': connection.authenticate, - 'connection.authenticate_user': connection.authenticate_user, - 'connection.authorize': connection.authorize, - 'connection.auto_authorize': connection.auto_authorize, - 'connection.autoinit': connection.autoinit, - 'connection.close': connection.close, - 'connection.exec': connection.exec_, - 'connection.execline': connection.execline, - 'connection.expect': connection.expect, - 'connection.get_protocol': connection.get_protocol, - 'connection.guess_os': connection.guess_os, - 'connection.send': connection.send, - 'connection.sendline': connection.sendline, - 'connection.set_error': connection.set_error, - 'connection.set_prompt': connection.set_prompt, - 'connection.set_timeout': connection.set_timeout, - 'connection.wait_for': connection.wait_for, - 'crypt.otp': crypt.otp, - 'file.chmod': file.chmod, - 'file.clear': file.clear, - 'file.exists': file.exists, - 'file.mkdir': file.mkdir, - 'file.read': file.read, - 'file.rm': file.rm, - 'file.write': file.write, - 'ipv4.in_network': ipv4.in_network, - 'ipv4.mask': ipv4.mask, - 'ipv4.mask2pfxlen': ipv4.mask2pfxlen, - 'ipv4.pfxlen2mask': ipv4.pfxlen2mask, - 'ipv4.pfxmask': ipv4.pfxmask, - 'ipv4.network': ipv4.network, - 'ipv4.broadcast': ipv4.broadcast, - 'ipv4.remote_ip': ipv4.remote_ip, - 'list.new': list.new, - 'list.get': list.get, - 'list.length': list.length, - 'list.sort': list.sort, - 'list.reverse': list.reverse, - 'list.unique': list.unique, - 'string.replace': string.replace, - 'string.split': string.split, - 'string.strip': string.strip, - 'string.tolower': string.tolower, - 'string.toupper': string.toupper, - 'sys.env': mysys.env, - 'sys.exec': mysys.execute, - 'sys.message': mysys.message, - 'sys.wait': mysys.wait, + 'connection.add_error': connection.add_error, + 'connection.add_prompt': connection.add_prompt, + 'connection.authenticate': connection.authenticate, + 'connection.authenticate_user': connection.authenticate_user, + 'connection.authorize': connection.authorize, + 'connection.auto_authorize': connection.auto_authorize, + 'connection.autoinit': connection.autoinit, + 'connection.close': connection.close, + 'connection.exec': connection.exec_, + 'connection.execline': connection.execline, + 'connection.expect': connection.expect, + 'connection.get_protocol': connection.get_protocol, + 'connection.guess_os': connection.guess_os, + 'connection.send': connection.send, + 'connection.sendline': connection.sendline, + 'connection.set_error': connection.set_error, + 'connection.set_prompt': connection.set_prompt, + 'connection.set_timeout': connection.set_timeout, + 'connection.wait_for': connection.wait_for, + 'crypt.otp': crypt.otp, + 'file.chmod': file.chmod, + 'file.clear': file.clear, + 'file.exists': file.exists, + 'file.mkdir': file.mkdir, + 'file.read': file.read, + 'file.rm': file.rm, + 'file.write': file.write, + 'ipv4.in_network': ipv4.in_network, + 'ipv4.mask': ipv4.mask, + 'ipv4.mask2pfxlen': ipv4.mask2pfxlen, + 'ipv4.pfxlen2mask': ipv4.pfxlen2mask, + 'ipv4.pfxmask': ipv4.pfxmask, + 'ipv4.network': ipv4.network, + 'ipv4.broadcast': ipv4.broadcast, + 'ipv4.remote_ip': ipv4.remote_ip, + 'list.new': list.new, + 'list.get': list.get, + 'list.length': list.length, + 'list.sort': list.sort, + 'list.reverse': list.reverse, + 'list.unique': list.unique, + 'string.replace': string.replace, + 'string.split': string.split, + 'string.strip': string.strip, + 'string.tolower': string.tolower, + 'string.toupper': string.toupper, + 'sys.env': mysys.env, + 'sys.exec': mysys.execute, + 'sys.message': mysys.message, + 'sys.wait': mysys.wait, } diff --git a/src/Exscript/stdlib/list.py b/src/Exscript/stdlib/list.py index ec01675b..edc55b55 100644 --- a/src/Exscript/stdlib/list.py +++ b/src/Exscript/stdlib/list.py @@ -71,6 +71,7 @@ def get(scope, source, index): except IndexError: raise ValueError('no such item in the list') + @secure_function def sort(scope, source): """ @@ -83,6 +84,7 @@ def sort(scope, source): """ return sorted(source) + @secure_function def reverse(scope, source): """ @@ -95,6 +97,7 @@ def reverse(scope, source): """ return list(reversed(source)) + @secure_function def unique(scope, source): """ diff --git a/src/Exscript/stdlib/mysys.py b/src/Exscript/stdlib/mysys.py index cde6d512..4ff19b71 100644 --- a/src/Exscript/stdlib/mysys.py +++ b/src/Exscript/stdlib/mysys.py @@ -46,13 +46,9 @@ def execute(scope, command): :type command: string :param command: A shell command. """ - process = Popen(command[0], - shell=True, - stdin=PIPE, - stdout=PIPE, - stderr=STDOUT, - text=True, - close_fds=True) + process = Popen( + command[0], shell=True, stdin=PIPE, stdout=PIPE, stderr=STDOUT, text=True, close_fds=True + ) if process.stdout: scope.define(__response__=process.stdout.readlines()) else: diff --git a/src/Exscript/stdlib/string.py b/src/Exscript/stdlib/string.py index 85d6606b..4f5bec1b 100644 --- a/src/Exscript/stdlib/string.py +++ b/src/Exscript/stdlib/string.py @@ -40,6 +40,7 @@ def replace(scope, strings, source, dest): """ return [s.replace(source[0], dest[0]) for s in strings] + @secure_function def split(scope, strings, separator): """ diff --git a/src/Exscript/stdlib/util.py b/src/Exscript/stdlib/util.py index eec04c4b..3b51a9cc 100644 --- a/src/Exscript/stdlib/util.py +++ b/src/Exscript/stdlib/util.py @@ -29,8 +29,10 @@ def secure_function(function): The function is secure, if it does not have any side effects on and does not read data from the system where the interpreter is running. """ + @wraps(function) def wrapper(*args, **kwargs): return function(*args, **kwargs) + setattr(wrapper, '_is_secure', True) return wrapper diff --git a/src/Exscript/util/__init__.py b/src/Exscript/util/__init__.py index 01aa96ab..25e87a0a 100644 --- a/src/Exscript/util/__init__.py +++ b/src/Exscript/util/__init__.py @@ -21,5 +21,9 @@ # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. import inspect -__all__ = [name for name, obj in list(locals().items()) - if not (name.startswith('_') or inspect.ismodule(obj))] + +__all__ = [ + name + for name, obj in list(locals().items()) + if not (name.startswith('_') or inspect.ismodule(obj)) +] diff --git a/src/Exscript/util/buffer.py b/src/Exscript/util/buffer.py index 04114462..129e6d27 100644 --- a/src/Exscript/util/buffer.py +++ b/src/Exscript/util/buffer.py @@ -23,13 +23,13 @@ """ A buffer object. """ + from io import StringIO from .cast import to_regexs class MonitoredBuffer: - """ A specialized string buffer that allows for monitoring the content using regular expression-triggered callbacks. @@ -45,7 +45,7 @@ def __init__(self, io=None): :param io: A file-like object that is used for storing the data. """ if io is None: - self.io = StringIO(u'') + self.io = StringIO('') else: self.io = io self.monitors = [] diff --git a/src/Exscript/util/cast.py b/src/Exscript/util/cast.py index bcf0d0c0..a6c4c4c5 100644 --- a/src/Exscript/util/cast.py +++ b/src/Exscript/util/cast.py @@ -23,6 +23,7 @@ """ Handy shortcuts for converting types. """ + import re import Exscript @@ -80,8 +81,7 @@ def to_hosts(hosts, default_protocol='ssh', default_domain=''): :rtype: list[Host] :return: A list of Host objects. """ - return [to_host(h, default_protocol, default_domain) - for h in to_list(hosts)] + return [to_host(h, default_protocol, default_domain) for h in to_list(hosts)] def to_regex(regex, flags=0): diff --git a/src/Exscript/util/collections.py b/src/Exscript/util/collections.py index 44c52d99..dc571746 100644 --- a/src/Exscript/util/collections.py +++ b/src/Exscript/util/collections.py @@ -4,13 +4,12 @@ class OrderedDefaultDict(OrderedDict): - """ A fusion of Python's defaultdict and Python's OrderedDict. """ + def __init__(self, default_factory=None, *a, **kw): - if (default_factory is not None and - not isinstance(default_factory, Callable)): + if default_factory is not None and not isinstance(default_factory, Callable): raise TypeError('first argument must be callable') OrderedDict.__init__(self, *a, **kw) self.default_factory = default_factory @@ -31,7 +30,7 @@ def __reduce__(self): if self.default_factory is None: args = tuple() else: - args = self.default_factory, + args = (self.default_factory,) return type(self), args, None, None, self.items() def copy(self): @@ -41,9 +40,7 @@ def __copy__(self): return type(self)(self.default_factory, self) def __deepcopy__(self, memo): - return type(self)(self.default_factory, - copy.deepcopy(self.items())) + return type(self)(self.default_factory, copy.deepcopy(self.items())) def __repr__(self): - return 'OrderedDefaultDict(%s, %s)' % (self.default_factory, - OrderedDict.__repr__(self)) + return 'OrderedDefaultDict(%s, %s)' % (self.default_factory, OrderedDict.__repr__(self)) diff --git a/src/Exscript/util/crypt.py b/src/Exscript/util/crypt.py index 3ab7c8e9..639c8b84 100644 --- a/src/Exscript/util/crypt.py +++ b/src/Exscript/util/crypt.py @@ -23,6 +23,7 @@ """ Encryption related utilities. """ + import random import string @@ -34,263 +35,2056 @@ _VALIDSEEDCHARACTERS = string.ascii_letters + string.digits _DICTIONARY = [ - "A", "ABE", "ACE", "ACT", "AD", "ADA", "ADD", -"AGO", "AID", "AIM", "AIR", "ALL", "ALP", "AM", "AMY", -"AN", "ANA", "AND", "ANN", "ANT", "ANY", "APE", "APS", -"APT", "ARC", "ARE", "ARK", "ARM", "ART", "AS", "ASH", -"ASK", "AT", "ATE", "AUG", "AUK", "AVE", "AWE", "AWK", -"AWL", "AWN", "AX", "AYE", "BAD", "BAG", "BAH", "BAM", -"BAN", "BAR", "BAT", "BAY", "BE", "BED", "BEE", "BEG", -"BEN", "BET", "BEY", "BIB", "BID", "BIG", "BIN", "BIT", -"BOB", "BOG", "BON", "BOO", "BOP", "BOW", "BOY", "BUB", -"BUD", "BUG", "BUM", "BUN", "BUS", "BUT", "BUY", "BY", -"BYE", "CAB", "CAL", "CAM", "CAN", "CAP", "CAR", "CAT", -"CAW", "COD", "COG", "COL", "CON", "COO", "COP", "COT", -"COW", "COY", "CRY", "CUB", "CUE", "CUP", "CUR", "CUT", -"DAB", "DAD", "DAM", "DAN", "DAR", "DAY", "DEE", "DEL", -"DEN", "DES", "DEW", "DID", "DIE", "DIG", "DIN", "DIP", -"DO", "DOE", "DOG", "DON", "DOT", "DOW", "DRY", "DUB", -"DUD", "DUE", "DUG", "DUN", "EAR", "EAT", "ED", "EEL", -"EGG", "EGO", "ELI", "ELK", "ELM", "ELY", "EM", "END", -"EST", "ETC", "EVA", "EVE", "EWE", "EYE", "FAD", "FAN", -"FAR", "FAT", "FAY", "FED", "FEE", "FEW", "FIB", "FIG", -"FIN", "FIR", "FIT", "FLO", "FLY", "FOE", "FOG", "FOR", -"FRY", "FUM", "FUN", "FUR", "GAB", "GAD", "GAG", "GAL", -"GAM", "GAP", "GAS", "GAY", "GEE", "GEL", "GEM", "GET", -"GIG", "GIL", "GIN", "GO", "GOT", "GUM", "GUN", "GUS", -"GUT", "GUY", "GYM", "GYP", "HA", "HAD", "HAL", "HAM", -"HAN", "HAP", "HAS", "HAT", "HAW", "HAY", "HE", "HEM", -"HEN", "HER", "HEW", "HEY", "HI", "HID", "HIM", "HIP", -"HIS", "HIT", "HO", "HOB", "HOC", "HOE", "HOG", "HOP", -"HOT", "HOW", "HUB", "HUE", "HUG", "HUH", "HUM", "HUT", -"I", "ICY", "IDA", "IF", "IKE", "ILL", "INK", "INN", -"IO", "ION", "IQ", "IRA", "IRE", "IRK", "IS", "IT", -"ITS", "IVY", "JAB", "JAG", "JAM", "JAN", "JAR", "JAW", -"JAY", "JET", "JIG", "JIM", "JO", "JOB", "JOE", "JOG", -"JOT", "JOY", "JUG", "JUT", "KAY", "KEG", "KEN", "KEY", -"KID", "KIM", "KIN", "KIT", "LA", "LAB", "LAC", "LAD", -"LAG", "LAM", "LAP", "LAW", "LAY", "LEA", "LED", "LEE", -"LEG", "LEN", "LEO", "LET", "LEW", "LID", "LIE", "LIN", -"LIP", "LIT", "LO", "LOB", "LOG", "LOP", "LOS", "LOT", -"LOU", "LOW", "LOY", "LUG", "LYE", "MA", "MAC", "MAD", -"MAE", "MAN", "MAO", "MAP", "MAT", "MAW", "MAY", "ME", -"MEG", "MEL", "MEN", "MET", "MEW", "MID", "MIN", "MIT", -"MOB", "MOD", "MOE", "MOO", "MOP", "MOS", "MOT", "MOW", -"MUD", "MUG", "MUM", "MY", "NAB", "NAG", "NAN", "NAP", -"NAT", "NAY", "NE", "NED", "NEE", "NET", "NEW", "NIB", -"NIL", "NIP", "NIT", "NO", "NOB", "NOD", "NON", "NOR", -"NOT", "NOV", "NOW", "NU", "NUN", "NUT", "O", "OAF", -"OAK", "OAR", "OAT", "ODD", "ODE", "OF", "OFF", "OFT", -"OH", "OIL", "OK", "OLD", "ON", "ONE", "OR", "ORB", -"ORE", "ORR", "OS", "OTT", "OUR", "OUT", "OVA", "OW", -"OWE", "OWL", "OWN", "OX", "PA", "PAD", "PAL", "PAM", -"PAN", "PAP", "PAR", "PAT", "PAW", "PAY", "PEA", "PEG", -"PEN", "PEP", "PER", "PET", "PEW", "PHI", "PI", "PIE", -"PIN", "PIT", "PLY", "PO", "POD", "POE", "POP", "POT", -"POW", "PRO", "PRY", "PUB", "PUG", "PUN", "PUP", "PUT", -"QUO", "RAG", "RAM", "RAN", "RAP", "RAT", "RAW", "RAY", -"REB", "RED", "REP", "RET", "RIB", "RID", "RIG", "RIM", -"RIO", "RIP", "ROB", "ROD", "ROE", "RON", "ROT", "ROW", -"ROY", "RUB", "RUE", "RUG", "RUM", "RUN", "RYE", "SAC", -"SAD", "SAG", "SAL", "SAM", "SAN", "SAP", "SAT", "SAW", -"SAY", "SEA", "SEC", "SEE", "SEN", "SET", "SEW", "SHE", -"SHY", "SIN", "SIP", "SIR", "SIS", "SIT", "SKI", "SKY", -"SLY", "SO", "SOB", "SOD", "SON", "SOP", "SOW", "SOY", -"SPA", "SPY", "SUB", "SUD", "SUE", "SUM", "SUN", "SUP", -"TAB", "TAD", "TAG", "TAN", "TAP", "TAR", "TEA", "TED", -"TEE", "TEN", "THE", "THY", "TIC", "TIE", "TIM", "TIN", -"TIP", "TO", "TOE", "TOG", "TOM", "TON", "TOO", "TOP", -"TOW", "TOY", "TRY", "TUB", "TUG", "TUM", "TUN", "TWO", -"UN", "UP", "US", "USE", "VAN", "VAT", "VET", "VIE", -"WAD", "WAG", "WAR", "WAS", "WAY", "WE", "WEB", "WED", -"WEE", "WET", "WHO", "WHY", "WIN", "WIT", "WOK", "WON", -"WOO", "WOW", "WRY", "WU", "YAM", "YAP", "YAW", "YE", -"YEA", "YES", "YET", "YOU", "ABED", "ABEL", "ABET", "ABLE", -"ABUT", "ACHE", "ACID", "ACME", "ACRE", "ACTA", "ACTS", "ADAM", -"ADDS", "ADEN", "AFAR", "AFRO", "AGEE", "AHEM", "AHOY", "AIDA", -"AIDE", "AIDS", "AIRY", "AJAR", "AKIN", "ALAN", "ALEC", "ALGA", -"ALIA", "ALLY", "ALMA", "ALOE", "ALSO", "ALTO", "ALUM", "ALVA", -"AMEN", "AMES", "AMID", "AMMO", "AMOK", "AMOS", "AMRA", "ANDY", -"ANEW", "ANNA", "ANNE", "ANTE", "ANTI", "AQUA", "ARAB", "ARCH", -"AREA", "ARGO", "ARID", "ARMY", "ARTS", "ARTY", "ASIA", "ASKS", -"ATOM", "AUNT", "AURA", "AUTO", "AVER", "AVID", "AVIS", "AVON", -"AVOW", "AWAY", "AWRY", "BABE", "BABY", "BACH", "BACK", "BADE", -"BAIL", "BAIT", "BAKE", "BALD", "BALE", "BALI", "BALK", "BALL", -"BALM", "BAND", "BANE", "BANG", "BANK", "BARB", "BARD", "BARE", -"BARK", "BARN", "BARR", "BASE", "BASH", "BASK", "BASS", "BATE", -"BATH", "BAWD", "BAWL", "BEAD", "BEAK", "BEAM", "BEAN", "BEAR", -"BEAT", "BEAU", "BECK", "BEEF", "BEEN", "BEER", "BEET", "BELA", -"BELL", "BELT", "BEND", "BENT", "BERG", "BERN", "BERT", "BESS", -"BEST", "BETA", "BETH", "BHOY", "BIAS", "BIDE", "BIEN", "BILE", -"BILK", "BILL", "BIND", "BING", "BIRD", "BITE", "BITS", "BLAB", -"BLAT", "BLED", "BLEW", "BLOB", "BLOC", "BLOT", "BLOW", "BLUE", -"BLUM", "BLUR", "BOAR", "BOAT", "BOCA", "BOCK", "BODE", "BODY", -"BOGY", "BOHR", "BOIL", "BOLD", "BOLO", "BOLT", "BOMB", "BONA", -"BOND", "BONE", "BONG", "BONN", "BONY", "BOOK", "BOOM", "BOON", -"BOOT", "BORE", "BORG", "BORN", "BOSE", "BOSS", "BOTH", "BOUT", -"BOWL", "BOYD", "BRAD", "BRAE", "BRAG", "BRAN", "BRAY", "BRED", -"BREW", "BRIG", "BRIM", "BROW", "BUCK", "BUDD", "BUFF", "BULB", -"BULK", "BULL", "BUNK", "BUNT", "BUOY", "BURG", "BURL", "BURN", -"BURR", "BURT", "BURY", "BUSH", "BUSS", "BUST", "BUSY", "BYTE", -"CADY", "CAFE", "CAGE", "CAIN", "CAKE", "CALF", "CALL", "CALM", -"CAME", "CANE", "CANT", "CARD", "CARE", "CARL", "CARR", "CART", -"CASE", "CASH", "CASK", "CAST", "CAVE", "CEIL", "CELL", "CENT", -"CERN", "CHAD", "CHAR", "CHAT", "CHAW", "CHEF", "CHEN", "CHEW", -"CHIC", "CHIN", "CHOU", "CHOW", "CHUB", "CHUG", "CHUM", "CITE", -"CITY", "CLAD", "CLAM", "CLAN", "CLAW", "CLAY", "CLOD", "CLOG", -"CLOT", "CLUB", "CLUE", "COAL", "COAT", "COCA", "COCK", "COCO", -"CODA", "CODE", "CODY", "COED", "COIL", "COIN", "COKE", "COLA", -"COLD", "COLT", "COMA", "COMB", "COME", "COOK", "COOL", "COON", -"COOT", "CORD", "CORE", "CORK", "CORN", "COST", "COVE", "COWL", -"CRAB", "CRAG", "CRAM", "CRAY", "CREW", "CRIB", "CROW", "CRUD", -"CUBA", "CUBE", "CUFF", "CULL", "CULT", "CUNY", "CURB", "CURD", -"CURE", "CURL", "CURT", "CUTS", "DADE", "DALE", "DAME", "DANA", -"DANE", "DANG", "DANK", "DARE", "DARK", "DARN", "DART", "DASH", -"DATA", "DATE", "DAVE", "DAVY", "DAWN", "DAYS", "DEAD", "DEAF", -"DEAL", "DEAN", "DEAR", "DEBT", "DECK", "DEED", "DEEM", "DEER", -"DEFT", "DEFY", "DELL", "DENT", "DENY", "DESK", "DIAL", "DICE", -"DIED", "DIET", "DIME", "DINE", "DING", "DINT", "DIRE", "DIRT", -"DISC", "DISH", "DISK", "DIVE", "DOCK", "DOES", "DOLE", "DOLL", -"DOLT", "DOME", "DONE", "DOOM", "DOOR", "DORA", "DOSE", "DOTE", -"DOUG", "DOUR", "DOVE", "DOWN", "DRAB", "DRAG", "DRAM", "DRAW", -"DREW", "DRUB", "DRUG", "DRUM", "DUAL", "DUCK", "DUCT", "DUEL", -"DUET", "DUKE", "DULL", "DUMB", "DUNE", "DUNK", "DUSK", "DUST", -"DUTY", "EACH", "EARL", "EARN", "EASE", "EAST", "EASY", "EBEN", -"ECHO", "EDDY", "EDEN", "EDGE", "EDGY", "EDIT", "EDNA", "EGAN", -"ELAN", "ELBA", "ELLA", "ELSE", "EMIL", "EMIT", "EMMA", "ENDS", -"ERIC", "EROS", "EVEN", "EVER", "EVIL", "EYED", "FACE", "FACT", -"FADE", "FAIL", "FAIN", "FAIR", "FAKE", "FALL", "FAME", "FANG", -"FARM", "FAST", "FATE", "FAWN", "FEAR", "FEAT", "FEED", "FEEL", -"FEET", "FELL", "FELT", "FEND", "FERN", "FEST", "FEUD", "FIEF", -"FIGS", "FILE", "FILL", "FILM", "FIND", "FINE", "FINK", "FIRE", -"FIRM", "FISH", "FISK", "FIST", "FITS", "FIVE", "FLAG", "FLAK", -"FLAM", "FLAT", "FLAW", "FLEA", "FLED", "FLEW", "FLIT", "FLOC", -"FLOG", "FLOW", "FLUB", "FLUE", "FOAL", "FOAM", "FOGY", "FOIL", -"FOLD", "FOLK", "FOND", "FONT", "FOOD", "FOOL", "FOOT", "FORD", -"FORE", "FORK", "FORM", "FORT", "FOSS", "FOUL", "FOUR", "FOWL", -"FRAU", "FRAY", "FRED", "FREE", "FRET", "FREY", "FROG", "FROM", -"FUEL", "FULL", "FUME", "FUND", "FUNK", "FURY", "FUSE", "FUSS", -"GAFF", "GAGE", "GAIL", "GAIN", "GAIT", "GALA", "GALE", "GALL", -"GALT", "GAME", "GANG", "GARB", "GARY", "GASH", "GATE", "GAUL", -"GAUR", "GAVE", "GAWK", "GEAR", "GELD", "GENE", "GENT", "GERM", -"GETS", "GIBE", "GIFT", "GILD", "GILL", "GILT", "GINA", "GIRD", -"GIRL", "GIST", "GIVE", "GLAD", "GLEE", "GLEN", "GLIB", "GLOB", -"GLOM", "GLOW", "GLUE", "GLUM", "GLUT", "GOAD", "GOAL", "GOAT", -"GOER", "GOES", "GOLD", "GOLF", "GONE", "GONG", "GOOD", "GOOF", -"GORE", "GORY", "GOSH", "GOUT", "GOWN", "GRAB", "GRAD", "GRAY", -"GREG", "GREW", "GREY", "GRID", "GRIM", "GRIN", "GRIT", "GROW", -"GRUB", "GULF", "GULL", "GUNK", "GURU", "GUSH", "GUST", "GWEN", -"GWYN", "HAAG", "HAAS", "HACK", "HAIL", "HAIR", "HALE", "HALF", -"HALL", "HALO", "HALT", "HAND", "HANG", "HANK", "HANS", "HARD", -"HARK", "HARM", "HART", "HASH", "HAST", "HATE", "HATH", "HAUL", -"HAVE", "HAWK", "HAYS", "HEAD", "HEAL", "HEAR", "HEAT", "HEBE", -"HECK", "HEED", "HEEL", "HEFT", "HELD", "HELL", "HELM", "HERB", -"HERD", "HERE", "HERO", "HERS", "HESS", "HEWN", "HICK", "HIDE", -"HIGH", "HIKE", "HILL", "HILT", "HIND", "HINT", "HIRE", "HISS", -"HIVE", "HOBO", "HOCK", "HOFF", "HOLD", "HOLE", "HOLM", "HOLT", -"HOME", "HONE", "HONK", "HOOD", "HOOF", "HOOK", "HOOT", "HORN", -"HOSE", "HOST", "HOUR", "HOVE", "HOWE", "HOWL", "HOYT", "HUCK", -"HUED", "HUFF", "HUGE", "HUGH", "HUGO", "HULK", "HULL", "HUNK", -"HUNT", "HURD", "HURL", "HURT", "HUSH", "HYDE", "HYMN", "IBIS", -"ICON", "IDEA", "IDLE", "IFFY", "INCA", "INCH", "INTO", "IONS", -"IOTA", "IOWA", "IRIS", "IRMA", "IRON", "ISLE", "ITCH", "ITEM", -"IVAN", "JACK", "JADE", "JAIL", "JAKE", "JANE", "JAVA", "JEAN", -"JEFF", "JERK", "JESS", "JEST", "JIBE", "JILL", "JILT", "JIVE", -"JOAN", "JOBS", "JOCK", "JOEL", "JOEY", "JOHN", "JOIN", "JOKE", -"JOLT", "JOVE", "JUDD", "JUDE", "JUDO", "JUDY", "JUJU", "JUKE", -"JULY", "JUNE", "JUNK", "JUNO", "JURY", "JUST", "JUTE", "KAHN", -"KALE", "KANE", "KANT", "KARL", "KATE", "KEEL", "KEEN", "KENO", -"KENT", "KERN", "KERR", "KEYS", "KICK", "KILL", "KIND", "KING", -"KIRK", "KISS", "KITE", "KLAN", "KNEE", "KNEW", "KNIT", "KNOB", -"KNOT", "KNOW", "KOCH", "KONG", "KUDO", "KURD", "KURT", "KYLE", -"LACE", "LACK", "LACY", "LADY", "LAID", "LAIN", "LAIR", "LAKE", -"LAMB", "LAME", "LAND", "LANE", "LANG", "LARD", "LARK", "LASS", -"LAST", "LATE", "LAUD", "LAVA", "LAWN", "LAWS", "LAYS", "LEAD", -"LEAF", "LEAK", "LEAN", "LEAR", "LEEK", "LEER", "LEFT", "LEND", -"LENS", "LENT", "LEON", "LESK", "LESS", "LEST", "LETS", "LIAR", -"LICE", "LICK", "LIED", "LIEN", "LIES", "LIEU", "LIFE", "LIFT", -"LIKE", "LILA", "LILT", "LILY", "LIMA", "LIMB", "LIME", "LIND", -"LINE", "LINK", "LINT", "LION", "LISA", "LIST", "LIVE", "LOAD", -"LOAF", "LOAM", "LOAN", "LOCK", "LOFT", "LOGE", "LOIS", "LOLA", -"LONE", "LONG", "LOOK", "LOON", "LOOT", "LORD", "LORE", "LOSE", -"LOSS", "LOST", "LOUD", "LOVE", "LOWE", "LUCK", "LUCY", "LUGE", -"LUKE", "LULU", "LUND", "LUNG", "LURA", "LURE", "LURK", "LUSH", -"LUST", "LYLE", "LYNN", "LYON", "LYRA", "MACE", "MADE", "MAGI", -"MAID", "MAIL", "MAIN", "MAKE", "MALE", "MALI", "MALL", "MALT", -"MANA", "MANN", "MANY", "MARC", "MARE", "MARK", "MARS", "MART", -"MARY", "MASH", "MASK", "MASS", "MAST", "MATE", "MATH", "MAUL", -"MAYO", "MEAD", "MEAL", "MEAN", "MEAT", "MEEK", "MEET", "MELD", -"MELT", "MEMO", "MEND", "MENU", "MERT", "MESH", "MESS", "MICE", -"MIKE", "MILD", "MILE", "MILK", "MILL", "MILT", "MIMI", "MIND", -"MINE", "MINI", "MINK", "MINT", "MIRE", "MISS", "MIST", "MITE", -"MITT", "MOAN", "MOAT", "MOCK", "MODE", "MOLD", "MOLE", "MOLL", -"MOLT", "MONA", "MONK", "MONT", "MOOD", "MOON", "MOOR", "MOOT", -"MORE", "MORN", "MORT", "MOSS", "MOST", "MOTH", "MOVE", "MUCH", -"MUCK", "MUDD", "MUFF", "MULE", "MULL", "MURK", "MUSH", "MUST", -"MUTE", "MUTT", "MYRA", "MYTH", "NAGY", "NAIL", "NAIR", "NAME", -"NARY", "NASH", "NAVE", "NAVY", "NEAL", "NEAR", "NEAT", "NECK", -"NEED", "NEIL", "NELL", "NEON", "NERO", "NESS", "NEST", "NEWS", -"NEWT", "NIBS", "NICE", "NICK", "NILE", "NINA", "NINE", "NOAH", -"NODE", "NOEL", "NOLL", "NONE", "NOOK", "NOON", "NORM", "NOSE", -"NOTE", "NOUN", "NOVA", "NUDE", "NULL", "NUMB", "OATH", "OBEY", -"OBOE", "ODIN", "OHIO", "OILY", "OINT", "OKAY", "OLAF", "OLDY", -"OLGA", "OLIN", "OMAN", "OMEN", "OMIT", "ONCE", "ONES", "ONLY", -"ONTO", "ONUS", "ORAL", "ORGY", "OSLO", "OTIS", "OTTO", "OUCH", -"OUST", "OUTS", "OVAL", "OVEN", "OVER", "OWLY", "OWNS", "QUAD", -"QUIT", "QUOD", "RACE", "RACK", "RACY", "RAFT", "RAGE", "RAID", -"RAIL", "RAIN", "RAKE", "RANK", "RANT", "RARE", "RASH", "RATE", -"RAVE", "RAYS", "READ", "REAL", "REAM", "REAR", "RECK", "REED", -"REEF", "REEK", "REEL", "REID", "REIN", "RENA", "REND", "RENT", -"REST", "RICE", "RICH", "RICK", "RIDE", "RIFT", "RILL", "RIME", -"RING", "RINK", "RISE", "RISK", "RITE", "ROAD", "ROAM", "ROAR", -"ROBE", "ROCK", "RODE", "ROIL", "ROLL", "ROME", "ROOD", "ROOF", -"ROOK", "ROOM", "ROOT", "ROSA", "ROSE", "ROSS", "ROSY", "ROTH", -"ROUT", "ROVE", "ROWE", "ROWS", "RUBE", "RUBY", "RUDE", "RUDY", -"RUIN", "RULE", "RUNG", "RUNS", "RUNT", "RUSE", "RUSH", "RUSK", -"RUSS", "RUST", "RUTH", "SACK", "SAFE", "SAGE", "SAID", "SAIL", -"SALE", "SALK", "SALT", "SAME", "SAND", "SANE", "SANG", "SANK", -"SARA", "SAUL", "SAVE", "SAYS", "SCAN", "SCAR", "SCAT", "SCOT", -"SEAL", "SEAM", "SEAR", "SEAT", "SEED", "SEEK", "SEEM", "SEEN", -"SEES", "SELF", "SELL", "SEND", "SENT", "SETS", "SEWN", "SHAG", -"SHAM", "SHAW", "SHAY", "SHED", "SHIM", "SHIN", "SHOD", "SHOE", -"SHOT", "SHOW", "SHUN", "SHUT", "SICK", "SIDE", "SIFT", "SIGH", -"SIGN", "SILK", "SILL", "SILO", "SILT", "SINE", "SING", "SINK", -"SIRE", "SITE", "SITS", "SITU", "SKAT", "SKEW", "SKID", "SKIM", -"SKIN", "SKIT", "SLAB", "SLAM", "SLAT", "SLAY", "SLED", "SLEW", -"SLID", "SLIM", "SLIT", "SLOB", "SLOG", "SLOT", "SLOW", "SLUG", -"SLUM", "SLUR", "SMOG", "SMUG", "SNAG", "SNOB", "SNOW", "SNUB", -"SNUG", "SOAK", "SOAR", "SOCK", "SODA", "SOFA", "SOFT", "SOIL", -"SOLD", "SOME", "SONG", "SOON", "SOOT", "SORE", "SORT", "SOUL", -"SOUR", "SOWN", "STAB", "STAG", "STAN", "STAR", "STAY", "STEM", -"STEW", "STIR", "STOW", "STUB", "STUN", "SUCH", "SUDS", "SUIT", -"SULK", "SUMS", "SUNG", "SUNK", "SURE", "SURF", "SWAB", "SWAG", -"SWAM", "SWAN", "SWAT", "SWAY", "SWIM", "SWUM", "TACK", "TACT", -"TAIL", "TAKE", "TALE", "TALK", "TALL", "TANK", "TASK", "TATE", -"TAUT", "TEAL", "TEAM", "TEAR", "TECH", "TEEM", "TEEN", "TEET", -"TELL", "TEND", "TENT", "TERM", "TERN", "TESS", "TEST", "THAN", -"THAT", "THEE", "THEM", "THEN", "THEY", "THIN", "THIS", "THUD", -"THUG", "TICK", "TIDE", "TIDY", "TIED", "TIER", "TILE", "TILL", -"TILT", "TIME", "TINA", "TINE", "TINT", "TINY", "TIRE", "TOAD", -"TOGO", "TOIL", "TOLD", "TOLL", "TONE", "TONG", "TONY", "TOOK", -"TOOL", "TOOT", "TORE", "TORN", "TOTE", "TOUR", "TOUT", "TOWN", -"TRAG", "TRAM", "TRAY", "TREE", "TREK", "TRIG", "TRIM", "TRIO", -"TROD", "TROT", "TROY", "TRUE", "TUBA", "TUBE", "TUCK", "TUFT", -"TUNA", "TUNE", "TUNG", "TURF", "TURN", "TUSK", "TWIG", "TWIN", -"TWIT", "ULAN", "UNIT", "URGE", "USED", "USER", "USES", "UTAH", -"VAIL", "VAIN", "VALE", "VARY", "VASE", "VAST", "VEAL", "VEDA", -"VEIL", "VEIN", "VEND", "VENT", "VERB", "VERY", "VETO", "VICE", -"VIEW", "VINE", "VISE", "VOID", "VOLT", "VOTE", "WACK", "WADE", -"WAGE", "WAIL", "WAIT", "WAKE", "WALE", "WALK", "WALL", "WALT", -"WAND", "WANE", "WANG", "WANT", "WARD", "WARM", "WARN", "WART", -"WASH", "WAST", "WATS", "WATT", "WAVE", "WAVY", "WAYS", "WEAK", -"WEAL", "WEAN", "WEAR", "WEED", "WEEK", "WEIR", "WELD", "WELL", -"WELT", "WENT", "WERE", "WERT", "WEST", "WHAM", "WHAT", "WHEE", -"WHEN", "WHET", "WHOA", "WHOM", "WICK", "WIFE", "WILD", "WILL", -"WIND", "WINE", "WING", "WINK", "WINO", "WIRE", "WISE", "WISH", -"WITH", "WOLF", "WONT", "WOOD", "WOOL", "WORD", "WORE", "WORK", -"WORM", "WORN", "WOVE", "WRIT", "WYNN", "YALE", "YANG", "YANK", -"YARD", "YARN", "YAWL", "YAWN", "YEAH", "YEAR", "YELL", "YOGA", -"YOKE"] + "A", + "ABE", + "ACE", + "ACT", + "AD", + "ADA", + "ADD", + "AGO", + "AID", + "AIM", + "AIR", + "ALL", + "ALP", + "AM", + "AMY", + "AN", + "ANA", + "AND", + "ANN", + "ANT", + "ANY", + "APE", + "APS", + "APT", + "ARC", + "ARE", + "ARK", + "ARM", + "ART", + "AS", + "ASH", + "ASK", + "AT", + "ATE", + "AUG", + "AUK", + "AVE", + "AWE", + "AWK", + "AWL", + "AWN", + "AX", + "AYE", + "BAD", + "BAG", + "BAH", + "BAM", + "BAN", + "BAR", + "BAT", + "BAY", + "BE", + "BED", + "BEE", + "BEG", + "BEN", + "BET", + "BEY", + "BIB", + "BID", + "BIG", + "BIN", + "BIT", + "BOB", + "BOG", + "BON", + "BOO", + "BOP", + "BOW", + "BOY", + "BUB", + "BUD", + "BUG", + "BUM", + "BUN", + "BUS", + "BUT", + "BUY", + "BY", + "BYE", + "CAB", + "CAL", + "CAM", + "CAN", + "CAP", + "CAR", + "CAT", + "CAW", + "COD", + "COG", + "COL", + "CON", + "COO", + "COP", + "COT", + "COW", + "COY", + "CRY", + "CUB", + "CUE", + "CUP", + "CUR", + "CUT", + "DAB", + "DAD", + "DAM", + "DAN", + "DAR", + "DAY", + "DEE", + "DEL", + "DEN", + "DES", + "DEW", + "DID", + "DIE", + "DIG", + "DIN", + "DIP", + "DO", + "DOE", + "DOG", + "DON", + "DOT", + "DOW", + "DRY", + "DUB", + "DUD", + "DUE", + "DUG", + "DUN", + "EAR", + "EAT", + "ED", + "EEL", + "EGG", + "EGO", + "ELI", + "ELK", + "ELM", + "ELY", + "EM", + "END", + "EST", + "ETC", + "EVA", + "EVE", + "EWE", + "EYE", + "FAD", + "FAN", + "FAR", + "FAT", + "FAY", + "FED", + "FEE", + "FEW", + "FIB", + "FIG", + "FIN", + "FIR", + "FIT", + "FLO", + "FLY", + "FOE", + "FOG", + "FOR", + "FRY", + "FUM", + "FUN", + "FUR", + "GAB", + "GAD", + "GAG", + "GAL", + "GAM", + "GAP", + "GAS", + "GAY", + "GEE", + "GEL", + "GEM", + "GET", + "GIG", + "GIL", + "GIN", + "GO", + "GOT", + "GUM", + "GUN", + "GUS", + "GUT", + "GUY", + "GYM", + "GYP", + "HA", + "HAD", + "HAL", + "HAM", + "HAN", + "HAP", + "HAS", + "HAT", + "HAW", + "HAY", + "HE", + "HEM", + "HEN", + "HER", + "HEW", + "HEY", + "HI", + "HID", + "HIM", + "HIP", + "HIS", + "HIT", + "HO", + "HOB", + "HOC", + "HOE", + "HOG", + "HOP", + "HOT", + "HOW", + "HUB", + "HUE", + "HUG", + "HUH", + "HUM", + "HUT", + "I", + "ICY", + "IDA", + "IF", + "IKE", + "ILL", + "INK", + "INN", + "IO", + "ION", + "IQ", + "IRA", + "IRE", + "IRK", + "IS", + "IT", + "ITS", + "IVY", + "JAB", + "JAG", + "JAM", + "JAN", + "JAR", + "JAW", + "JAY", + "JET", + "JIG", + "JIM", + "JO", + "JOB", + "JOE", + "JOG", + "JOT", + "JOY", + "JUG", + "JUT", + "KAY", + "KEG", + "KEN", + "KEY", + "KID", + "KIM", + "KIN", + "KIT", + "LA", + "LAB", + "LAC", + "LAD", + "LAG", + "LAM", + "LAP", + "LAW", + "LAY", + "LEA", + "LED", + "LEE", + "LEG", + "LEN", + "LEO", + "LET", + "LEW", + "LID", + "LIE", + "LIN", + "LIP", + "LIT", + "LO", + "LOB", + "LOG", + "LOP", + "LOS", + "LOT", + "LOU", + "LOW", + "LOY", + "LUG", + "LYE", + "MA", + "MAC", + "MAD", + "MAE", + "MAN", + "MAO", + "MAP", + "MAT", + "MAW", + "MAY", + "ME", + "MEG", + "MEL", + "MEN", + "MET", + "MEW", + "MID", + "MIN", + "MIT", + "MOB", + "MOD", + "MOE", + "MOO", + "MOP", + "MOS", + "MOT", + "MOW", + "MUD", + "MUG", + "MUM", + "MY", + "NAB", + "NAG", + "NAN", + "NAP", + "NAT", + "NAY", + "NE", + "NED", + "NEE", + "NET", + "NEW", + "NIB", + "NIL", + "NIP", + "NIT", + "NO", + "NOB", + "NOD", + "NON", + "NOR", + "NOT", + "NOV", + "NOW", + "NU", + "NUN", + "NUT", + "O", + "OAF", + "OAK", + "OAR", + "OAT", + "ODD", + "ODE", + "OF", + "OFF", + "OFT", + "OH", + "OIL", + "OK", + "OLD", + "ON", + "ONE", + "OR", + "ORB", + "ORE", + "ORR", + "OS", + "OTT", + "OUR", + "OUT", + "OVA", + "OW", + "OWE", + "OWL", + "OWN", + "OX", + "PA", + "PAD", + "PAL", + "PAM", + "PAN", + "PAP", + "PAR", + "PAT", + "PAW", + "PAY", + "PEA", + "PEG", + "PEN", + "PEP", + "PER", + "PET", + "PEW", + "PHI", + "PI", + "PIE", + "PIN", + "PIT", + "PLY", + "PO", + "POD", + "POE", + "POP", + "POT", + "POW", + "PRO", + "PRY", + "PUB", + "PUG", + "PUN", + "PUP", + "PUT", + "QUO", + "RAG", + "RAM", + "RAN", + "RAP", + "RAT", + "RAW", + "RAY", + "REB", + "RED", + "REP", + "RET", + "RIB", + "RID", + "RIG", + "RIM", + "RIO", + "RIP", + "ROB", + "ROD", + "ROE", + "RON", + "ROT", + "ROW", + "ROY", + "RUB", + "RUE", + "RUG", + "RUM", + "RUN", + "RYE", + "SAC", + "SAD", + "SAG", + "SAL", + "SAM", + "SAN", + "SAP", + "SAT", + "SAW", + "SAY", + "SEA", + "SEC", + "SEE", + "SEN", + "SET", + "SEW", + "SHE", + "SHY", + "SIN", + "SIP", + "SIR", + "SIS", + "SIT", + "SKI", + "SKY", + "SLY", + "SO", + "SOB", + "SOD", + "SON", + "SOP", + "SOW", + "SOY", + "SPA", + "SPY", + "SUB", + "SUD", + "SUE", + "SUM", + "SUN", + "SUP", + "TAB", + "TAD", + "TAG", + "TAN", + "TAP", + "TAR", + "TEA", + "TED", + "TEE", + "TEN", + "THE", + "THY", + "TIC", + "TIE", + "TIM", + "TIN", + "TIP", + "TO", + "TOE", + "TOG", + "TOM", + "TON", + "TOO", + "TOP", + "TOW", + "TOY", + "TRY", + "TUB", + "TUG", + "TUM", + "TUN", + "TWO", + "UN", + "UP", + "US", + "USE", + "VAN", + "VAT", + "VET", + "VIE", + "WAD", + "WAG", + "WAR", + "WAS", + "WAY", + "WE", + "WEB", + "WED", + "WEE", + "WET", + "WHO", + "WHY", + "WIN", + "WIT", + "WOK", + "WON", + "WOO", + "WOW", + "WRY", + "WU", + "YAM", + "YAP", + "YAW", + "YE", + "YEA", + "YES", + "YET", + "YOU", + "ABED", + "ABEL", + "ABET", + "ABLE", + "ABUT", + "ACHE", + "ACID", + "ACME", + "ACRE", + "ACTA", + "ACTS", + "ADAM", + "ADDS", + "ADEN", + "AFAR", + "AFRO", + "AGEE", + "AHEM", + "AHOY", + "AIDA", + "AIDE", + "AIDS", + "AIRY", + "AJAR", + "AKIN", + "ALAN", + "ALEC", + "ALGA", + "ALIA", + "ALLY", + "ALMA", + "ALOE", + "ALSO", + "ALTO", + "ALUM", + "ALVA", + "AMEN", + "AMES", + "AMID", + "AMMO", + "AMOK", + "AMOS", + "AMRA", + "ANDY", + "ANEW", + "ANNA", + "ANNE", + "ANTE", + "ANTI", + "AQUA", + "ARAB", + "ARCH", + "AREA", + "ARGO", + "ARID", + "ARMY", + "ARTS", + "ARTY", + "ASIA", + "ASKS", + "ATOM", + "AUNT", + "AURA", + "AUTO", + "AVER", + "AVID", + "AVIS", + "AVON", + "AVOW", + "AWAY", + "AWRY", + "BABE", + "BABY", + "BACH", + "BACK", + "BADE", + "BAIL", + "BAIT", + "BAKE", + "BALD", + "BALE", + "BALI", + "BALK", + "BALL", + "BALM", + "BAND", + "BANE", + "BANG", + "BANK", + "BARB", + "BARD", + "BARE", + "BARK", + "BARN", + "BARR", + "BASE", + "BASH", + "BASK", + "BASS", + "BATE", + "BATH", + "BAWD", + "BAWL", + "BEAD", + "BEAK", + "BEAM", + "BEAN", + "BEAR", + "BEAT", + "BEAU", + "BECK", + "BEEF", + "BEEN", + "BEER", + "BEET", + "BELA", + "BELL", + "BELT", + "BEND", + "BENT", + "BERG", + "BERN", + "BERT", + "BESS", + "BEST", + "BETA", + "BETH", + "BHOY", + "BIAS", + "BIDE", + "BIEN", + "BILE", + "BILK", + "BILL", + "BIND", + "BING", + "BIRD", + "BITE", + "BITS", + "BLAB", + "BLAT", + "BLED", + "BLEW", + "BLOB", + "BLOC", + "BLOT", + "BLOW", + "BLUE", + "BLUM", + "BLUR", + "BOAR", + "BOAT", + "BOCA", + "BOCK", + "BODE", + "BODY", + "BOGY", + "BOHR", + "BOIL", + "BOLD", + "BOLO", + "BOLT", + "BOMB", + "BONA", + "BOND", + "BONE", + "BONG", + "BONN", + "BONY", + "BOOK", + "BOOM", + "BOON", + "BOOT", + "BORE", + "BORG", + "BORN", + "BOSE", + "BOSS", + "BOTH", + "BOUT", + "BOWL", + "BOYD", + "BRAD", + "BRAE", + "BRAG", + "BRAN", + "BRAY", + "BRED", + "BREW", + "BRIG", + "BRIM", + "BROW", + "BUCK", + "BUDD", + "BUFF", + "BULB", + "BULK", + "BULL", + "BUNK", + "BUNT", + "BUOY", + "BURG", + "BURL", + "BURN", + "BURR", + "BURT", + "BURY", + "BUSH", + "BUSS", + "BUST", + "BUSY", + "BYTE", + "CADY", + "CAFE", + "CAGE", + "CAIN", + "CAKE", + "CALF", + "CALL", + "CALM", + "CAME", + "CANE", + "CANT", + "CARD", + "CARE", + "CARL", + "CARR", + "CART", + "CASE", + "CASH", + "CASK", + "CAST", + "CAVE", + "CEIL", + "CELL", + "CENT", + "CERN", + "CHAD", + "CHAR", + "CHAT", + "CHAW", + "CHEF", + "CHEN", + "CHEW", + "CHIC", + "CHIN", + "CHOU", + "CHOW", + "CHUB", + "CHUG", + "CHUM", + "CITE", + "CITY", + "CLAD", + "CLAM", + "CLAN", + "CLAW", + "CLAY", + "CLOD", + "CLOG", + "CLOT", + "CLUB", + "CLUE", + "COAL", + "COAT", + "COCA", + "COCK", + "COCO", + "CODA", + "CODE", + "CODY", + "COED", + "COIL", + "COIN", + "COKE", + "COLA", + "COLD", + "COLT", + "COMA", + "COMB", + "COME", + "COOK", + "COOL", + "COON", + "COOT", + "CORD", + "CORE", + "CORK", + "CORN", + "COST", + "COVE", + "COWL", + "CRAB", + "CRAG", + "CRAM", + "CRAY", + "CREW", + "CRIB", + "CROW", + "CRUD", + "CUBA", + "CUBE", + "CUFF", + "CULL", + "CULT", + "CUNY", + "CURB", + "CURD", + "CURE", + "CURL", + "CURT", + "CUTS", + "DADE", + "DALE", + "DAME", + "DANA", + "DANE", + "DANG", + "DANK", + "DARE", + "DARK", + "DARN", + "DART", + "DASH", + "DATA", + "DATE", + "DAVE", + "DAVY", + "DAWN", + "DAYS", + "DEAD", + "DEAF", + "DEAL", + "DEAN", + "DEAR", + "DEBT", + "DECK", + "DEED", + "DEEM", + "DEER", + "DEFT", + "DEFY", + "DELL", + "DENT", + "DENY", + "DESK", + "DIAL", + "DICE", + "DIED", + "DIET", + "DIME", + "DINE", + "DING", + "DINT", + "DIRE", + "DIRT", + "DISC", + "DISH", + "DISK", + "DIVE", + "DOCK", + "DOES", + "DOLE", + "DOLL", + "DOLT", + "DOME", + "DONE", + "DOOM", + "DOOR", + "DORA", + "DOSE", + "DOTE", + "DOUG", + "DOUR", + "DOVE", + "DOWN", + "DRAB", + "DRAG", + "DRAM", + "DRAW", + "DREW", + "DRUB", + "DRUG", + "DRUM", + "DUAL", + "DUCK", + "DUCT", + "DUEL", + "DUET", + "DUKE", + "DULL", + "DUMB", + "DUNE", + "DUNK", + "DUSK", + "DUST", + "DUTY", + "EACH", + "EARL", + "EARN", + "EASE", + "EAST", + "EASY", + "EBEN", + "ECHO", + "EDDY", + "EDEN", + "EDGE", + "EDGY", + "EDIT", + "EDNA", + "EGAN", + "ELAN", + "ELBA", + "ELLA", + "ELSE", + "EMIL", + "EMIT", + "EMMA", + "ENDS", + "ERIC", + "EROS", + "EVEN", + "EVER", + "EVIL", + "EYED", + "FACE", + "FACT", + "FADE", + "FAIL", + "FAIN", + "FAIR", + "FAKE", + "FALL", + "FAME", + "FANG", + "FARM", + "FAST", + "FATE", + "FAWN", + "FEAR", + "FEAT", + "FEED", + "FEEL", + "FEET", + "FELL", + "FELT", + "FEND", + "FERN", + "FEST", + "FEUD", + "FIEF", + "FIGS", + "FILE", + "FILL", + "FILM", + "FIND", + "FINE", + "FINK", + "FIRE", + "FIRM", + "FISH", + "FISK", + "FIST", + "FITS", + "FIVE", + "FLAG", + "FLAK", + "FLAM", + "FLAT", + "FLAW", + "FLEA", + "FLED", + "FLEW", + "FLIT", + "FLOC", + "FLOG", + "FLOW", + "FLUB", + "FLUE", + "FOAL", + "FOAM", + "FOGY", + "FOIL", + "FOLD", + "FOLK", + "FOND", + "FONT", + "FOOD", + "FOOL", + "FOOT", + "FORD", + "FORE", + "FORK", + "FORM", + "FORT", + "FOSS", + "FOUL", + "FOUR", + "FOWL", + "FRAU", + "FRAY", + "FRED", + "FREE", + "FRET", + "FREY", + "FROG", + "FROM", + "FUEL", + "FULL", + "FUME", + "FUND", + "FUNK", + "FURY", + "FUSE", + "FUSS", + "GAFF", + "GAGE", + "GAIL", + "GAIN", + "GAIT", + "GALA", + "GALE", + "GALL", + "GALT", + "GAME", + "GANG", + "GARB", + "GARY", + "GASH", + "GATE", + "GAUL", + "GAUR", + "GAVE", + "GAWK", + "GEAR", + "GELD", + "GENE", + "GENT", + "GERM", + "GETS", + "GIBE", + "GIFT", + "GILD", + "GILL", + "GILT", + "GINA", + "GIRD", + "GIRL", + "GIST", + "GIVE", + "GLAD", + "GLEE", + "GLEN", + "GLIB", + "GLOB", + "GLOM", + "GLOW", + "GLUE", + "GLUM", + "GLUT", + "GOAD", + "GOAL", + "GOAT", + "GOER", + "GOES", + "GOLD", + "GOLF", + "GONE", + "GONG", + "GOOD", + "GOOF", + "GORE", + "GORY", + "GOSH", + "GOUT", + "GOWN", + "GRAB", + "GRAD", + "GRAY", + "GREG", + "GREW", + "GREY", + "GRID", + "GRIM", + "GRIN", + "GRIT", + "GROW", + "GRUB", + "GULF", + "GULL", + "GUNK", + "GURU", + "GUSH", + "GUST", + "GWEN", + "GWYN", + "HAAG", + "HAAS", + "HACK", + "HAIL", + "HAIR", + "HALE", + "HALF", + "HALL", + "HALO", + "HALT", + "HAND", + "HANG", + "HANK", + "HANS", + "HARD", + "HARK", + "HARM", + "HART", + "HASH", + "HAST", + "HATE", + "HATH", + "HAUL", + "HAVE", + "HAWK", + "HAYS", + "HEAD", + "HEAL", + "HEAR", + "HEAT", + "HEBE", + "HECK", + "HEED", + "HEEL", + "HEFT", + "HELD", + "HELL", + "HELM", + "HERB", + "HERD", + "HERE", + "HERO", + "HERS", + "HESS", + "HEWN", + "HICK", + "HIDE", + "HIGH", + "HIKE", + "HILL", + "HILT", + "HIND", + "HINT", + "HIRE", + "HISS", + "HIVE", + "HOBO", + "HOCK", + "HOFF", + "HOLD", + "HOLE", + "HOLM", + "HOLT", + "HOME", + "HONE", + "HONK", + "HOOD", + "HOOF", + "HOOK", + "HOOT", + "HORN", + "HOSE", + "HOST", + "HOUR", + "HOVE", + "HOWE", + "HOWL", + "HOYT", + "HUCK", + "HUED", + "HUFF", + "HUGE", + "HUGH", + "HUGO", + "HULK", + "HULL", + "HUNK", + "HUNT", + "HURD", + "HURL", + "HURT", + "HUSH", + "HYDE", + "HYMN", + "IBIS", + "ICON", + "IDEA", + "IDLE", + "IFFY", + "INCA", + "INCH", + "INTO", + "IONS", + "IOTA", + "IOWA", + "IRIS", + "IRMA", + "IRON", + "ISLE", + "ITCH", + "ITEM", + "IVAN", + "JACK", + "JADE", + "JAIL", + "JAKE", + "JANE", + "JAVA", + "JEAN", + "JEFF", + "JERK", + "JESS", + "JEST", + "JIBE", + "JILL", + "JILT", + "JIVE", + "JOAN", + "JOBS", + "JOCK", + "JOEL", + "JOEY", + "JOHN", + "JOIN", + "JOKE", + "JOLT", + "JOVE", + "JUDD", + "JUDE", + "JUDO", + "JUDY", + "JUJU", + "JUKE", + "JULY", + "JUNE", + "JUNK", + "JUNO", + "JURY", + "JUST", + "JUTE", + "KAHN", + "KALE", + "KANE", + "KANT", + "KARL", + "KATE", + "KEEL", + "KEEN", + "KENO", + "KENT", + "KERN", + "KERR", + "KEYS", + "KICK", + "KILL", + "KIND", + "KING", + "KIRK", + "KISS", + "KITE", + "KLAN", + "KNEE", + "KNEW", + "KNIT", + "KNOB", + "KNOT", + "KNOW", + "KOCH", + "KONG", + "KUDO", + "KURD", + "KURT", + "KYLE", + "LACE", + "LACK", + "LACY", + "LADY", + "LAID", + "LAIN", + "LAIR", + "LAKE", + "LAMB", + "LAME", + "LAND", + "LANE", + "LANG", + "LARD", + "LARK", + "LASS", + "LAST", + "LATE", + "LAUD", + "LAVA", + "LAWN", + "LAWS", + "LAYS", + "LEAD", + "LEAF", + "LEAK", + "LEAN", + "LEAR", + "LEEK", + "LEER", + "LEFT", + "LEND", + "LENS", + "LENT", + "LEON", + "LESK", + "LESS", + "LEST", + "LETS", + "LIAR", + "LICE", + "LICK", + "LIED", + "LIEN", + "LIES", + "LIEU", + "LIFE", + "LIFT", + "LIKE", + "LILA", + "LILT", + "LILY", + "LIMA", + "LIMB", + "LIME", + "LIND", + "LINE", + "LINK", + "LINT", + "LION", + "LISA", + "LIST", + "LIVE", + "LOAD", + "LOAF", + "LOAM", + "LOAN", + "LOCK", + "LOFT", + "LOGE", + "LOIS", + "LOLA", + "LONE", + "LONG", + "LOOK", + "LOON", + "LOOT", + "LORD", + "LORE", + "LOSE", + "LOSS", + "LOST", + "LOUD", + "LOVE", + "LOWE", + "LUCK", + "LUCY", + "LUGE", + "LUKE", + "LULU", + "LUND", + "LUNG", + "LURA", + "LURE", + "LURK", + "LUSH", + "LUST", + "LYLE", + "LYNN", + "LYON", + "LYRA", + "MACE", + "MADE", + "MAGI", + "MAID", + "MAIL", + "MAIN", + "MAKE", + "MALE", + "MALI", + "MALL", + "MALT", + "MANA", + "MANN", + "MANY", + "MARC", + "MARE", + "MARK", + "MARS", + "MART", + "MARY", + "MASH", + "MASK", + "MASS", + "MAST", + "MATE", + "MATH", + "MAUL", + "MAYO", + "MEAD", + "MEAL", + "MEAN", + "MEAT", + "MEEK", + "MEET", + "MELD", + "MELT", + "MEMO", + "MEND", + "MENU", + "MERT", + "MESH", + "MESS", + "MICE", + "MIKE", + "MILD", + "MILE", + "MILK", + "MILL", + "MILT", + "MIMI", + "MIND", + "MINE", + "MINI", + "MINK", + "MINT", + "MIRE", + "MISS", + "MIST", + "MITE", + "MITT", + "MOAN", + "MOAT", + "MOCK", + "MODE", + "MOLD", + "MOLE", + "MOLL", + "MOLT", + "MONA", + "MONK", + "MONT", + "MOOD", + "MOON", + "MOOR", + "MOOT", + "MORE", + "MORN", + "MORT", + "MOSS", + "MOST", + "MOTH", + "MOVE", + "MUCH", + "MUCK", + "MUDD", + "MUFF", + "MULE", + "MULL", + "MURK", + "MUSH", + "MUST", + "MUTE", + "MUTT", + "MYRA", + "MYTH", + "NAGY", + "NAIL", + "NAIR", + "NAME", + "NARY", + "NASH", + "NAVE", + "NAVY", + "NEAL", + "NEAR", + "NEAT", + "NECK", + "NEED", + "NEIL", + "NELL", + "NEON", + "NERO", + "NESS", + "NEST", + "NEWS", + "NEWT", + "NIBS", + "NICE", + "NICK", + "NILE", + "NINA", + "NINE", + "NOAH", + "NODE", + "NOEL", + "NOLL", + "NONE", + "NOOK", + "NOON", + "NORM", + "NOSE", + "NOTE", + "NOUN", + "NOVA", + "NUDE", + "NULL", + "NUMB", + "OATH", + "OBEY", + "OBOE", + "ODIN", + "OHIO", + "OILY", + "OINT", + "OKAY", + "OLAF", + "OLDY", + "OLGA", + "OLIN", + "OMAN", + "OMEN", + "OMIT", + "ONCE", + "ONES", + "ONLY", + "ONTO", + "ONUS", + "ORAL", + "ORGY", + "OSLO", + "OTIS", + "OTTO", + "OUCH", + "OUST", + "OUTS", + "OVAL", + "OVEN", + "OVER", + "OWLY", + "OWNS", + "QUAD", + "QUIT", + "QUOD", + "RACE", + "RACK", + "RACY", + "RAFT", + "RAGE", + "RAID", + "RAIL", + "RAIN", + "RAKE", + "RANK", + "RANT", + "RARE", + "RASH", + "RATE", + "RAVE", + "RAYS", + "READ", + "REAL", + "REAM", + "REAR", + "RECK", + "REED", + "REEF", + "REEK", + "REEL", + "REID", + "REIN", + "RENA", + "REND", + "RENT", + "REST", + "RICE", + "RICH", + "RICK", + "RIDE", + "RIFT", + "RILL", + "RIME", + "RING", + "RINK", + "RISE", + "RISK", + "RITE", + "ROAD", + "ROAM", + "ROAR", + "ROBE", + "ROCK", + "RODE", + "ROIL", + "ROLL", + "ROME", + "ROOD", + "ROOF", + "ROOK", + "ROOM", + "ROOT", + "ROSA", + "ROSE", + "ROSS", + "ROSY", + "ROTH", + "ROUT", + "ROVE", + "ROWE", + "ROWS", + "RUBE", + "RUBY", + "RUDE", + "RUDY", + "RUIN", + "RULE", + "RUNG", + "RUNS", + "RUNT", + "RUSE", + "RUSH", + "RUSK", + "RUSS", + "RUST", + "RUTH", + "SACK", + "SAFE", + "SAGE", + "SAID", + "SAIL", + "SALE", + "SALK", + "SALT", + "SAME", + "SAND", + "SANE", + "SANG", + "SANK", + "SARA", + "SAUL", + "SAVE", + "SAYS", + "SCAN", + "SCAR", + "SCAT", + "SCOT", + "SEAL", + "SEAM", + "SEAR", + "SEAT", + "SEED", + "SEEK", + "SEEM", + "SEEN", + "SEES", + "SELF", + "SELL", + "SEND", + "SENT", + "SETS", + "SEWN", + "SHAG", + "SHAM", + "SHAW", + "SHAY", + "SHED", + "SHIM", + "SHIN", + "SHOD", + "SHOE", + "SHOT", + "SHOW", + "SHUN", + "SHUT", + "SICK", + "SIDE", + "SIFT", + "SIGH", + "SIGN", + "SILK", + "SILL", + "SILO", + "SILT", + "SINE", + "SING", + "SINK", + "SIRE", + "SITE", + "SITS", + "SITU", + "SKAT", + "SKEW", + "SKID", + "SKIM", + "SKIN", + "SKIT", + "SLAB", + "SLAM", + "SLAT", + "SLAY", + "SLED", + "SLEW", + "SLID", + "SLIM", + "SLIT", + "SLOB", + "SLOG", + "SLOT", + "SLOW", + "SLUG", + "SLUM", + "SLUR", + "SMOG", + "SMUG", + "SNAG", + "SNOB", + "SNOW", + "SNUB", + "SNUG", + "SOAK", + "SOAR", + "SOCK", + "SODA", + "SOFA", + "SOFT", + "SOIL", + "SOLD", + "SOME", + "SONG", + "SOON", + "SOOT", + "SORE", + "SORT", + "SOUL", + "SOUR", + "SOWN", + "STAB", + "STAG", + "STAN", + "STAR", + "STAY", + "STEM", + "STEW", + "STIR", + "STOW", + "STUB", + "STUN", + "SUCH", + "SUDS", + "SUIT", + "SULK", + "SUMS", + "SUNG", + "SUNK", + "SURE", + "SURF", + "SWAB", + "SWAG", + "SWAM", + "SWAN", + "SWAT", + "SWAY", + "SWIM", + "SWUM", + "TACK", + "TACT", + "TAIL", + "TAKE", + "TALE", + "TALK", + "TALL", + "TANK", + "TASK", + "TATE", + "TAUT", + "TEAL", + "TEAM", + "TEAR", + "TECH", + "TEEM", + "TEEN", + "TEET", + "TELL", + "TEND", + "TENT", + "TERM", + "TERN", + "TESS", + "TEST", + "THAN", + "THAT", + "THEE", + "THEM", + "THEN", + "THEY", + "THIN", + "THIS", + "THUD", + "THUG", + "TICK", + "TIDE", + "TIDY", + "TIED", + "TIER", + "TILE", + "TILL", + "TILT", + "TIME", + "TINA", + "TINE", + "TINT", + "TINY", + "TIRE", + "TOAD", + "TOGO", + "TOIL", + "TOLD", + "TOLL", + "TONE", + "TONG", + "TONY", + "TOOK", + "TOOL", + "TOOT", + "TORE", + "TORN", + "TOTE", + "TOUR", + "TOUT", + "TOWN", + "TRAG", + "TRAM", + "TRAY", + "TREE", + "TREK", + "TRIG", + "TRIM", + "TRIO", + "TROD", + "TROT", + "TROY", + "TRUE", + "TUBA", + "TUBE", + "TUCK", + "TUFT", + "TUNA", + "TUNE", + "TUNG", + "TURF", + "TURN", + "TUSK", + "TWIG", + "TWIN", + "TWIT", + "ULAN", + "UNIT", + "URGE", + "USED", + "USER", + "USES", + "UTAH", + "VAIL", + "VAIN", + "VALE", + "VARY", + "VASE", + "VAST", + "VEAL", + "VEDA", + "VEIL", + "VEIN", + "VEND", + "VENT", + "VERB", + "VERY", + "VETO", + "VICE", + "VIEW", + "VINE", + "VISE", + "VOID", + "VOLT", + "VOTE", + "WACK", + "WADE", + "WAGE", + "WAIL", + "WAIT", + "WAKE", + "WALE", + "WALK", + "WALL", + "WALT", + "WAND", + "WANE", + "WANG", + "WANT", + "WARD", + "WARM", + "WARN", + "WART", + "WASH", + "WAST", + "WATS", + "WATT", + "WAVE", + "WAVY", + "WAYS", + "WEAK", + "WEAL", + "WEAN", + "WEAR", + "WEED", + "WEEK", + "WEIR", + "WELD", + "WELL", + "WELT", + "WENT", + "WERE", + "WERT", + "WEST", + "WHAM", + "WHAT", + "WHEE", + "WHEN", + "WHET", + "WHOA", + "WHOM", + "WICK", + "WIFE", + "WILD", + "WILL", + "WIND", + "WINE", + "WING", + "WINK", + "WINO", + "WIRE", + "WISE", + "WISH", + "WITH", + "WOLF", + "WONT", + "WOOD", + "WOOL", + "WORD", + "WORE", + "WORK", + "WORM", + "WORN", + "WOVE", + "WRIT", + "WYNN", + "YALE", + "YANG", + "YANK", + "YARD", + "YARN", + "YAWL", + "YAWN", + "YEAH", + "YEAR", + "YELL", + "YOGA", + "YOKE", +] + def _checksummed_long(key): thesum, k = 0, key @@ -299,6 +2093,7 @@ def _checksummed_long(key): k = k >> 2 return (key << 2) | (thesum % 4) + def _sixword_from_long(key): key = _checksummed_long(key) words = [] @@ -307,6 +2102,7 @@ def _sixword_from_long(key): key = key >> 11 return ' '.join(words) + def _long_from_raw(thehash): """Fold to a long, a digest supplied as a string.""" hashnum = 0 @@ -315,20 +2111,23 @@ def _long_from_raw(thehash): hashnum |= ord(bytes([h])) return hashnum + def _sixword_from_raw(key): return _sixword_from_long(_long_from_raw(key)) + def _fold_md4_or_md5(digest): if len(digest) < 16: raise ValueError('digest is too short') result = b'' for i in range(0, 8): - #print(len(digest[i])) + # print(len(digest[i])) one = ord(bytes([digest[i]])) - two = ord(bytes([digest[i+8]])) - result = result + bytes([one^two]) + two = ord(bytes([digest[i + 8]])) + result = result + bytes([one ^ two]) return result + def otp(password, seed, sequence): """ Calculates a one-time password hash using the given password, seed, and diff --git a/src/Exscript/util/daemonize.py b/src/Exscript/util/daemonize.py index 1d943fe9..ec48e21b 100644 --- a/src/Exscript/util/daemonize.py +++ b/src/Exscript/util/daemonize.py @@ -23,6 +23,7 @@ """ Daemonizing a process. """ + import sys import os diff --git a/src/Exscript/util/decorator.py b/src/Exscript/util/decorator.py index 80d812a9..fee1973f 100644 --- a/src/Exscript/util/decorator.py +++ b/src/Exscript/util/decorator.py @@ -23,6 +23,7 @@ """ Decorators for callbacks passed to Queue.run(). """ + from ..protocols.exception import LoginFailure from .impl import add_label, copy_labels, get_label # noqa: F401 @@ -41,9 +42,11 @@ def bind(function, *args, **kwargs): :rtype: function :return: The wrapped function. """ + def decorated(*inner_args, **inner_kwargs): kwargs.update(inner_kwargs) return function(*(inner_args + args), **kwargs) + copy_labels(function, decorated) return decorated @@ -80,12 +83,14 @@ def shell(job, host, conn): :rtype: object :return: The return value of the called function. """ + def decorated(job, host, conn, *args, **kwargs): os = conn.guess_os() func = map.get(os) if func is None: raise Exception('No handler for %s found.' % os) return func(job, host, conn, *args, **kwargs) + return decorated @@ -105,6 +110,7 @@ def _decorate(flush=True, attempts=1, only_authenticate=False): :rtype: function :return: The wrapped function. """ + def decorator(function): def decorated(job, host, conn, *args, **kwargs): failed = 0 @@ -121,8 +127,10 @@ def decorated(job, host, conn, *args, **kwargs): continue break return function(job, host, conn, *args, **kwargs) + copy_labels(function, decorated) return decorated + return decorator diff --git a/src/Exscript/util/event.py b/src/Exscript/util/event.py index e9d1a1a4..5060b496 100644 --- a/src/Exscript/util/event.py +++ b/src/Exscript/util/event.py @@ -23,13 +23,13 @@ """ A simple signal/event mechanism. """ + from threading import Lock from . import weakmethod class Event: - """ A simple signal/event mechanism, to be used like this:: diff --git a/src/Exscript/util/file.py b/src/Exscript/util/file.py index a8ffa07a..a1185e8a 100644 --- a/src/Exscript/util/file.py +++ b/src/Exscript/util/file.py @@ -23,6 +23,7 @@ """ Utilities for reading data from files. """ + import base64 import binascii import configparser @@ -76,11 +77,9 @@ def get_accounts_from_file(filename, encoding='utf-8', password_encoding='utf-8' return accounts -def get_hosts_from_file(filename, - default_protocol='ssh', - default_domain='', - remove_duplicates=False, - encoding='utf-8'): +def get_hosts_from_file( + filename, default_protocol='ssh', default_domain='', remove_duplicates=False, encoding='utf-8' +): """ Reads a list of hostnames from the file with the given name. @@ -117,10 +116,7 @@ def get_hosts_from_file(filename, return hosts -def get_hosts_from_csv(filename, - default_protocol='ssh', - default_domain='', - encoding='utf-8'): +def get_hosts_from_csv(filename, default_protocol='ssh', default_domain='', encoding='utf-8'): """ Reads a list of hostnames and variables from the tab-separated .csv file with the given name. The first line of the file must contain the column diff --git a/src/Exscript/util/impl.py b/src/Exscript/util/impl.py index b655f3bc..9db5e4b6 100644 --- a/src/Exscript/util/impl.py +++ b/src/Exscript/util/impl.py @@ -23,6 +23,7 @@ """ Development tools. """ + import sys import traceback import warnings @@ -120,9 +121,7 @@ def deprecation(msg): """ Prints a deprecation warning. """ - warnings.warn(msg, - category=DeprecationWarning, - stacklevel=3) + warnings.warn(msg, category=DeprecationWarning, stacklevel=3) def deprecated(func): @@ -130,9 +129,11 @@ def deprecated(func): A decorator for marking functions as deprecated. Results in a printed warning message when the function is used. """ + def decorated(*args, **kwargs): deprecation('Call to deprecated function %s.' % func.__name__) return func(*args, **kwargs) + decorated.__name__ = func.__name__ decorated.__doc__ = func.__doc__ decorated.__dict__.update(func.__dict__) @@ -143,15 +144,18 @@ def synchronized(func): """ Decorator for synchronizing method access. """ + @wraps(func) def wrapped(self, *args, **kwargs): try: rlock = self._sync_lock except AttributeError: from multiprocessing import RLock + rlock = self.__dict__.setdefault('_sync_lock', RLock()) with rlock: return func(self, *args, **kwargs) + return wrapped @@ -159,6 +163,7 @@ def debug(func): """ Decorator that prints a message whenever a function is entered or left. """ + @wraps(func) def wrapped(*args, **kwargs): arg = repr(args) + ' ' + repr(kwargs) @@ -172,11 +177,11 @@ def wrapped(*args, **kwargs): arg = repr(result) sys.stdout.write('Leaving ' + func.__name__ + '(): ' + arg + '\n') return result + return wrapped class Decorator: - def __init__(self, obj): self.__dict__['obj'] = obj @@ -193,7 +198,6 @@ def __getattr__(self, name): class _Context(Decorator): - def __enter__(self): return self @@ -202,7 +206,6 @@ def __exit__(self, thetype, value, traceback): class Context(_Context): - def __exit__(self, thetype, value, traceback): return self.release() diff --git a/src/Exscript/util/interact.py b/src/Exscript/util/interact.py index 00d20bb4..15eec777 100644 --- a/src/Exscript/util/interact.py +++ b/src/Exscript/util/interact.py @@ -23,6 +23,7 @@ """ Tools for interacting with the user on the command line. """ + import configparser import getpass import os @@ -33,7 +34,6 @@ class InputHistory: - """ When prompting a user for input it is often useful to record his input in a file, and use previous input as a default value. @@ -41,9 +41,7 @@ class InputHistory: allow for such functionality. """ - def __init__(self, - filename='~/.exscript_history', - section=os.path.basename(sys.argv[0])): + def __init__(self, filename='~/.exscript_history', section=os.path.basename(sys.argv[0])): """ Constructor. The filename argument allows for listing on or more config files, and is passed to Python's RawConfigParser; please @@ -114,13 +112,7 @@ def set(self, key, value): return value -def prompt(key, - message, - default=None, - doverh=True, - strip=True, - check=None, - history=None): +def prompt(key, message, default=None, doverh=True, strip=True, check=None, history=None): """ Prompt the user for input. This function is similar to Python's built in raw_input, with the following differences: @@ -216,9 +208,11 @@ def get_filename(key, message, default=None, history=None): :type history: :class:`InputHistory` or None :param history: The history used for recording default values, or None. """ + def _validate(string): if not os.path.isfile(string): return 'File not found. Please enter a filename.' + return prompt(key, message, default, True, _validate, history) diff --git a/src/Exscript/util/ip.py b/src/Exscript/util/ip.py index b5f10892..e66c6121 100644 --- a/src/Exscript/util/ip.py +++ b/src/Exscript/util/ip.py @@ -23,6 +23,7 @@ """ Wrapper around the ipv4 and ipv6 modules to handle both, ipv4 and ipv6. """ + from Exscript.util import ipv4 from Exscript.util import ipv6 diff --git a/src/Exscript/util/ipv4.py b/src/Exscript/util/ipv4.py index 0a1f8452..208a5772 100644 --- a/src/Exscript/util/ipv4.py +++ b/src/Exscript/util/ipv4.py @@ -23,6 +23,7 @@ """ IPv4 address calculation and conversion. """ + import math import re import socket @@ -249,7 +250,7 @@ def matches_prefix(ip, prefix): network, pfxlen = parse_prefix(prefix) network_int = ip2int(network) mask_int = pfxlen2mask_int(pfxlen) - return ip_int&mask_int == network_int&mask_int + return ip_int & mask_int == network_int & mask_int def is_private(ip): diff --git a/src/Exscript/util/log.py b/src/Exscript/util/log.py index fd7042e8..b2926a2e 100644 --- a/src/Exscript/util/log.py +++ b/src/Exscript/util/log.py @@ -23,6 +23,7 @@ """ Logging utilities. """ + from .. import FileLogger from .impl import add_label @@ -42,6 +43,7 @@ def log_to(logger): def decorator(function): func = add_label(function, 'log_to', logger_id=logger_id) return func + return decorator diff --git a/src/Exscript/util/mail.py b/src/Exscript/util/mail.py index 37d1ee1c..6760f745 100644 --- a/src/Exscript/util/mail.py +++ b/src/Exscript/util/mail.py @@ -23,6 +23,7 @@ """ Sending and formatting emails. """ + import mimetypes import os import re @@ -42,12 +43,11 @@ # # Helpers. (non-public) # -_varname_re = re.compile(r'[a-z][\w_]*', re.I) +_varname_re = re.compile(r'[a-z][\w_]*', re.I) _string_re = re.compile(r'(\\?){([\w_]+)}', re.I) class _TemplateParser: - """ This exists for backward compatibility; Python 2.3 does not come with a similar way for string substitution yet. @@ -84,8 +84,7 @@ def parse(self, template, **kwargs): def _render_template(string, **vars): - default = {'date': time.strftime('%Y-%m-%d'), - 'user': getuser()} + default = {'date': time.strftime('%Y-%m-%d'), 'user': getuser()} default.update(vars) parser = _TemplateParser() return parser.parse(string, **default) @@ -106,24 +105,18 @@ def _cleanup_mail_addresses(recipients): rcpt = re.split(r'\s*[,;\r\n]\s*', recipients.lower()) return [str(r) for r in sorted(set(rcpt)) if r.strip()] + # # Public. # class Mail: - """ Represents an email. """ - def __init__(self, - sender=None, - to='', - cc='', - bcc='', - subject='', - body=''): + def __init__(self, sender=None, to='', cc='', bcc='', subject='', body=''): """ Creates a new email with the given values. If the given sender is None, one will be automatically chosen diff --git a/src/Exscript/util/match.py b/src/Exscript/util/match.py index e5b93e25..99ce27bc 100644 --- a/src/Exscript/util/match.py +++ b/src/Exscript/util/match.py @@ -23,6 +23,7 @@ """ Shorthands for regular expression matching. """ + import re from ..protocols import Protocol diff --git a/src/Exscript/util/pidutil.py b/src/Exscript/util/pidutil.py index 2ebcaa5e..e2fb0a7d 100644 --- a/src/Exscript/util/pidutil.py +++ b/src/Exscript/util/pidutil.py @@ -23,6 +23,7 @@ """ Handling PID (process id) files. """ + import errno import fcntl import logging diff --git a/src/Exscript/util/report.py b/src/Exscript/util/report.py index 264a9232..cf762ad4 100644 --- a/src/Exscript/util/report.py +++ b/src/Exscript/util/report.py @@ -75,10 +75,7 @@ def summarize(logger): return '\n'.join(summary) -def format(logger, - show_successful=True, - show_errors=True, - show_traceback=True): +def format(logger, show_successful=True, show_errors=True, show_traceback=True): """ Prints a report of the actions that were logged by the given Logger. The report contains a list of successful actions, as well as the full diff --git a/src/Exscript/util/sigint.py b/src/Exscript/util/sigint.py index fcaf4201..f5261322 100644 --- a/src/Exscript/util/sigint.py +++ b/src/Exscript/util/sigint.py @@ -23,6 +23,7 @@ """ A class for catching SIGINT, such that CTRL+c works. """ + import os import signal import sys @@ -30,7 +31,6 @@ class SigIntWatcher: - """ This class solves two problems with multithreaded programs in Python: @@ -56,7 +56,7 @@ def __init__(self): self.child = os.fork() except AttributeError: # platforms that don't have os.fork self.child = 0 - except RuntimeError: # prevent "not holding the import lock" on some systems + except RuntimeError: # prevent "not holding the import lock" on some systems self.child = 0 if self.child: self._watch() diff --git a/src/Exscript/util/sigintcatcher.py b/src/Exscript/util/sigintcatcher.py index 853f6f40..3c680dd8 100644 --- a/src/Exscript/util/sigintcatcher.py +++ b/src/Exscript/util/sigintcatcher.py @@ -36,5 +36,7 @@ So in general it is recommended to use the :class:`sigint.SigIntWatcher()` class directly. """ + from Exscript.util.sigint import SigIntWatcher + _watcher = SigIntWatcher() diff --git a/src/Exscript/util/start.py b/src/Exscript/util/start.py index ef62336a..3078bee1 100644 --- a/src/Exscript/util/start.py +++ b/src/Exscript/util/start.py @@ -23,6 +23,7 @@ """ Quickstart methods for the Exscript queue. """ + from .. import Queue from .decorator import autoauthenticate, autologin from .interact import read_login diff --git a/src/Exscript/util/syslog.py b/src/Exscript/util/syslog.py index 0a177e4d..26947e13 100644 --- a/src/Exscript/util/syslog.py +++ b/src/Exscript/util/syslog.py @@ -23,18 +23,21 @@ """ Send messages to a syslog server. """ + import os import socket import sys import syslog as _syslog -def netlog(message, - source=None, - host='localhost', - port=514, - priority=_syslog.LOG_DEBUG, - facility=_syslog.LOG_USER): +def netlog( + message, + source=None, + host='localhost', + port=514, + priority=_syslog.LOG_DEBUG, + facility=_syslog.LOG_USER, +): """ Python's built in syslog module does not support networking, so this is the alternative. diff --git a/src/Exscript/util/template.py b/src/Exscript/util/template.py index 37c78c6e..3375baa3 100644 --- a/src/Exscript/util/template.py +++ b/src/Exscript/util/template.py @@ -23,6 +23,7 @@ """ Executing Exscript templates on a connection. """ + from Exscript import stdlib from Exscript.interpreter import Parser @@ -198,7 +199,7 @@ def paste(conn, string, no_escaping=False, **kwargs): return _run(conn, None, string, parser_args, **kwargs) -def paste_file(conn, filename, no_escaping=False, **kwargs): +def paste_file(conn, filename, no_escaping=False, **kwargs): """ Convenience wrapper around paste() that reads the template from a file instead. diff --git a/src/Exscript/util/tty.py b/src/Exscript/util/tty.py index c26eb174..f3e8d40a 100644 --- a/src/Exscript/util/tty.py +++ b/src/Exscript/util/tty.py @@ -23,6 +23,7 @@ """ TTY utilities. """ + import os import struct import sys @@ -115,8 +116,7 @@ def get_terminal_size(default_rows=25, default_cols=80): # Try `stty size` with open(os.devnull, 'w') as devnull: try: - process = Popen(['stty', 'size'], stderr=devnull, stdout=PIPE, - close_fds=True) + process = Popen(['stty', 'size'], stderr=devnull, stdout=PIPE, close_fds=True) except (OSError, ValueError): pass else: diff --git a/src/Exscript/util/url.py b/src/Exscript/util/url.py index 746a8068..2cf8038e 100644 --- a/src/Exscript/util/url.py +++ b/src/Exscript/util/url.py @@ -23,6 +23,7 @@ """ Working with URLs (as used in URL formatted hostnames). """ + import re from urllib.parse import quote, urlencode, urlparse, urlsplit @@ -36,6 +37,7 @@ def _make_hexmap(): hexmap['%02X' % i] = chr(i) return hexmap + _HEXTOCHR = _make_hexmap() _WELL_KNOWN_PORTS = { @@ -51,6 +53,7 @@ def _make_hexmap(): 'netconf': 830, } + def _unquote(string): """_unquote('abc%20def') -> 'abc def'.""" result = string.split('%') @@ -64,6 +67,7 @@ def _unquote(string): result[i] = chr(int(item[:2], 16)) + item[2:] return ''.join(result) + def _urlparse_qs(url): """ Parse a URL query string and return the components as a dictionary. @@ -97,8 +101,8 @@ def _urlparse_qs(url): return result -class Url: +class Url: """ Represents a URL. """ @@ -123,9 +127,7 @@ def __str__(self): url = '' if self.protocol is not None: url += self.protocol + '://' - if self.username is not None or \ - self.password1 is not None or \ - self.password2 is not None: + if self.username is not None or self.password1 is not None or self.password2 is not None: if self.username is not None: url += quote(self.username, '') if self.password1 is not None or self.password2 is not None: @@ -200,7 +202,7 @@ def from_string(url, default_protocol='ssh'): # fails on our protocol names otherwise. prefix = result.protocol + '://' if url.startswith(prefix): - url = url[len(prefix):] + url = url[len(prefix) :] url = 'http://' + url # Parse the remaining url. diff --git a/src/Exscript/util/weakmethod.py b/src/Exscript/util/weakmethod.py index d5bbc665..bb2e8fba 100644 --- a/src/Exscript/util/weakmethod.py +++ b/src/Exscript/util/weakmethod.py @@ -23,23 +23,24 @@ """ Weak references to bound and unbound methods. """ + import weakref class DeadMethodCalled(Exception): - """ Raised by :class:`WeakMethod` if it is called when the referenced object is already dead. """ + pass class WeakMethod: - """ Do not create this class directly; use :class:`ref()` instead. """ + __slots__ = 'name', 'callback' def __init__(self, name, callback): diff --git a/src/Exscript/util/xml.py b/src/Exscript/util/xml.py index 3930c712..deabfd59 100644 --- a/src/Exscript/util/xml.py +++ b/src/Exscript/util/xml.py @@ -33,6 +33,7 @@ class XMLValidationError(Exception): Provides the XML string that failed in its `xmlstring` attribute. """ + def __init__(self, *args: object, xmlstring=None) -> None: super().__init__(*args) self.xmlstring = xmlstring diff --git a/src/Exscript/version.py b/src/Exscript/version.py index f96ad9e0..8cd59634 100644 --- a/src/Exscript/version.py +++ b/src/Exscript/version.py @@ -1,6 +1,7 @@ """ The version of the installed Exscript distribution. """ + from importlib.metadata import PackageNotFoundError, version try: diff --git a/src/Exscript/workqueue/__init__.py b/src/Exscript/workqueue/__init__.py index 711dd7a3..0d73b47c 100644 --- a/src/Exscript/workqueue/__init__.py +++ b/src/Exscript/workqueue/__init__.py @@ -26,5 +26,8 @@ from .task import Task from .workqueue import WorkQueue -__all__ = [name for name, obj in list(locals().items()) - if not (name.startswith('_') or inspect.ismodule(obj))] +__all__ = [ + name + for name, obj in list(locals().items()) + if not (name.startswith('_') or inspect.ismodule(obj)) +] diff --git a/src/Exscript/workqueue/job.py b/src/Exscript/workqueue/job.py index 143b53c9..a307d0f9 100644 --- a/src/Exscript/workqueue/job.py +++ b/src/Exscript/workqueue/job.py @@ -31,7 +31,6 @@ class _ChildWatcher(threading.Thread): - def __init__(self, child, callback): threading.Thread.__init__(self) self.child = child @@ -60,7 +59,6 @@ def run(self): def _make_process_class(base, clsname): class process_cls(base): - def __init__(self, id, function, name, data): base.__init__(self, name=name) self.id = id @@ -85,22 +83,17 @@ def run(self): def start(self, pipe): self.pipe = pipe base.start(self) + process_cls.__name__ = clsname return process_cls + Thread = _make_process_class(threading.Thread, 'Thread') Process = _make_process_class(multiprocessing.Process, 'Process') class Job: - __slots__ = ('id', - 'func', - 'name', - 'times', - 'failures', - 'data', - 'child', - 'watcher') + __slots__ = ('id', 'func', 'name', 'times', 'failures', 'data', 'child', 'watcher') def __init__(self, function, name, times, data): self.id = None diff --git a/src/Exscript/workqueue/mainloop.py b/src/Exscript/workqueue/mainloop.py index cb51ba38..8d132cff 100644 --- a/src/Exscript/workqueue/mainloop.py +++ b/src/Exscript/workqueue/mainloop.py @@ -35,7 +35,6 @@ class MainLoop(threading.Thread): - def __init__(self, collection, job_cls): threading.Thread.__init__(self) self._logger = _logger @@ -67,6 +66,7 @@ def conditional_append(queue): job = Job(function, name, times, data) job.id = queue.append(job, name) return job.id + return self.collection.with_lock(conditional_append) def priority_enqueue(self, function, name, force_start, times, data): @@ -74,12 +74,7 @@ def priority_enqueue(self, function, name, force_start, times, data): job.id = self.collection.appendleft(job, name, force=force_start) return job.id - def priority_enqueue_or_raise(self, - function, - name, - force_start, - times, - data): + def priority_enqueue_or_raise(self, function, name, force_start, times, data): def conditional_append(queue): job = queue.get_from_name(name) if job is None: @@ -88,6 +83,7 @@ def conditional_append(queue): return job.id queue.prioritize(job, force=force_start) return None + return self.collection.with_lock(conditional_append) def wait_for(self, job_id): diff --git a/src/Exscript/workqueue/pipeline.py b/src/Exscript/workqueue/pipeline.py index 49bc8d0a..6b5f6915 100644 --- a/src/Exscript/workqueue/pipeline.py +++ b/src/Exscript/workqueue/pipeline.py @@ -26,7 +26,6 @@ class Pipeline: - """ A collection that is similar to Python's Queue object, except it also tracks items that are currently sleeping or in progress. @@ -284,9 +283,7 @@ def __next__(self): continue # Wait until enough slots are available. - if len(self.working) - \ - len(self.sleeping) - \ - len(self.force) >= self.max_working: + if len(self.working) - len(self.sleeping) - len(self.force) >= self.max_working: self.condition.wait() continue diff --git a/src/Exscript/workqueue/task.py b/src/Exscript/workqueue/task.py index 350fc203..48ec9915 100644 --- a/src/Exscript/workqueue/task.py +++ b/src/Exscript/workqueue/task.py @@ -23,11 +23,11 @@ """ Represents a batch of enqueued actions. """ + from Exscript.util.event import Event class Task: - """ Represents a batch of running actions. """ diff --git a/src/Exscript/workqueue/workqueue.py b/src/Exscript/workqueue/workqueue.py index 35a500a1..9d3b3fce 100644 --- a/src/Exscript/workqueue/workqueue.py +++ b/src/Exscript/workqueue/workqueue.py @@ -32,16 +32,12 @@ class WorkQueue: - """ This class implements the asynchronous workqueue and is the main API for using the workqueue module. """ - def __init__(self, - collection=None, - max_threads=1, - mode='threading'): + def __init__(self, collection=None, max_threads=1, mode='threading'): """ Constructor. @@ -52,7 +48,9 @@ def __init__(self, if mode == 'threading': self.job_cls = Thread elif mode == 'multiprocessing': - self._logger.warning("Mode 'multiprocessing' is not recommended, see: https://github.com/knipknap/exscript/issues/230") + self._logger.warning( + "Mode 'multiprocessing' is not recommended, see: https://github.com/knipknap/exscript/issues/230" + ) self.job_cls = Process else: raise TypeError('invalid "mode" argument: ' + repr(mode)) @@ -165,12 +163,7 @@ def enqueue_or_ignore(self, function, name=None, times=1, data=None): self._check_if_ready() return self.main_loop.enqueue_or_ignore(function, name, times, data) - def priority_enqueue(self, - function, - name=None, - force_start=False, - times=1, - data=None): + def priority_enqueue(self, function, name=None, force_start=False, times=1, data=None): """ Like :class:`enqueue()`, but adds the given function at the top of the queue. @@ -191,18 +184,9 @@ def priority_enqueue(self, :return: The id of the new job. """ self._check_if_ready() - return self.main_loop.priority_enqueue(function, - name, - force_start, - times, - data) - - def priority_enqueue_or_raise(self, - function, - name=None, - force_start=False, - times=1, - data=None): + return self.main_loop.priority_enqueue(function, name, force_start, times, data) + + def priority_enqueue_or_raise(self, function, name=None, force_start=False, times=1, data=None): """ Like priority_enqueue(), but if a function with the same name is already in the queue, the existing function is moved to the top of @@ -221,11 +205,7 @@ def priority_enqueue_or_raise(self, :return: The id of the new job. """ self._check_if_ready() - return self.main_loop.priority_enqueue_or_raise(function, - name, - force_start, - times, - data) + return self.main_loop.priority_enqueue_or_raise(function, name, force_start, times, data) def unpause(self): """ diff --git a/tests/Exscript/AccountManagerTest.py b/tests/Exscript/AccountManagerTest.py index 71c9be06..28f113d6 100644 --- a/tests/Exscript/AccountManagerTest.py +++ b/tests/Exscript/AccountManagerTest.py @@ -124,5 +124,7 @@ def testReleaseAccounts(self): def suite(): return unittest.TestLoader().loadTestsFromTestCase(AccountManagerTest) + + if __name__ == '__main__': unittest.TextTestRunner(verbosity=2).run(suite()) diff --git a/tests/Exscript/AccountPoolTest.py b/tests/Exscript/AccountPoolTest.py index beee1b5b..1f39fb1c 100644 --- a/tests/Exscript/AccountPoolTest.py +++ b/tests/Exscript/AccountPoolTest.py @@ -52,8 +52,7 @@ def testGetAccountFromHash(self): def testGetAccountFromName(self): self.testAddAccount() - self.assertEqual(self.account2, - self.accm.get_account_from_name(self.user2)) + self.assertEqual(self.account2, self.accm.get_account_from_name(self.user2)) def testNAccounts(self): self.testAddAccount() @@ -115,5 +114,7 @@ def testReleaseAccounts(self): def suite(): return unittest.TestLoader().loadTestsFromTestCase(AccountPoolTest) + + if __name__ == '__main__': unittest.TextTestRunner(verbosity=2).run(suite()) diff --git a/tests/Exscript/AccountTest.py b/tests/Exscript/AccountTest.py index 132430e2..d6bd1b49 100644 --- a/tests/Exscript/AccountTest.py +++ b/tests/Exscript/AccountTest.py @@ -11,21 +11,16 @@ def setUp(self): self.password1 = 'test1' self.password2 = 'test2' self.key = PrivateKey() - self.account = Account(self.user, - self.password1, - self.password2, - self.key) + self.account = Account(self.user, self.password1, self.password2, self.key) def testConstructor(self): key = PrivateKey() account = Account(self.user, self.password1, key=key) self.assertEqual(account.get_key(), key) - self.assertEqual(account.get_password(), - account.get_authorization_password()) + self.assertEqual(account.get_password(), account.get_authorization_password()) account = Account(self.user, self.password1, self.password2) - self.assertNotEqual(account.get_password(), - account.get_authorization_password()) + self.assertNotEqual(account.get_password(), account.get_authorization_password()) def testContext(self): with self.account as account: @@ -72,14 +67,12 @@ def testGetPassword(self): self.assertEqual(self.password1, self.account.get_password()) def testSetAuthorizationPassword(self): - self.assertEqual(self.password2, - self.account.get_authorization_password()) + self.assertEqual(self.password2, self.account.get_authorization_password()) self.account.set_authorization_password('foo') self.assertEqual('foo', self.account.get_authorization_password()) def testGetAuthorizationPassword(self): - self.assertEqual(self.password2, - self.account.get_authorization_password()) + self.assertEqual(self.password2, self.account.get_authorization_password()) def testGetKey(self): self.assertEqual(self.key, self.account.get_key()) @@ -87,5 +80,7 @@ def testGetKey(self): def suite(): return unittest.TestLoader().loadTestsFromTestCase(AccountTest) + + if __name__ == '__main__': unittest.TextTestRunner(verbosity=2).run(suite()) diff --git a/tests/Exscript/FileLoggerTest.py b/tests/Exscript/FileLoggerTest.py index ec91da55..af2f9b9a 100644 --- a/tests/Exscript/FileLoggerTest.py +++ b/tests/Exscript/FileLoggerTest.py @@ -33,8 +33,7 @@ def testConstructor(self): def testAddLog(self): log = LoggerTest.testAddLog(self) - self.assertTrue( - os.path.isfile(self.logfile), 'No such file: ' + self.logfile) + self.assertTrue(os.path.isfile(self.logfile), 'No such file: ' + self.logfile) self.assertFalse(os.path.exists(self.errfile)) return log @@ -89,5 +88,7 @@ def testLogSucceeded2(self): def suite(): return unittest.TestLoader().loadTestsFromTestCase(FileLoggerTest) + + if __name__ == '__main__': unittest.TextTestRunner(verbosity=2).run(suite()) diff --git a/tests/Exscript/HostTest.py b/tests/Exscript/HostTest.py index 4c5a4578..8e11bed5 100644 --- a/tests/Exscript/HostTest.py +++ b/tests/Exscript/HostTest.py @@ -23,44 +23,41 @@ def testConstructor(self): for url, result in urls: host = Host(url) uri = Url.from_string(url) - self.assertEqual(host.get_name(), uri.hostname) + self.assertEqual(host.get_name(), uri.hostname) self.assertEqual(host.get_address(), uri.hostname) - self.assertEqual(host.get_uri().split('&').sort(), - str(uri).split('&').sort()) + self.assertEqual(host.get_uri().split('&').sort(), str(uri).split('&').sort()) def testSetUri(self): for url, result in urls: self.host.set_uri(url) uri = Url.from_string(url) - self.assertEqual(self.host.get_name(), uri.hostname) + self.assertEqual(self.host.get_name(), uri.hostname) self.assertEqual(self.host.get_address(), uri.hostname) def testGetUri(self): for url, result in urls: host = Host(url) uri = Url.from_string(url) - self.assertEqual(host.get_uri().split('&').sort(), - str(uri).split('&').sort()) + self.assertEqual(host.get_uri().split('&').sort(), str(uri).split('&').sort()) def testGetDict(self): host = Host('foo') host.set_address('foo2') - self.assertEqual(host.get_dict(), {'hostname': 'foo', - 'address': 'foo2', - 'protocol': 'ssh', - 'port': 22}) + self.assertEqual( + host.get_dict(), {'hostname': 'foo', 'address': 'foo2', 'protocol': 'ssh', 'port': 22} + ) def testSetAddress(self): self.host.set_protocol('dummy') self.host.set_address('test.org') self.assertEqual(self.host.get_protocol(), 'dummy') - self.assertEqual(self.host.get_name(), 'localhost') - self.assertEqual(self.host.get_address(), 'test.org') + self.assertEqual(self.host.get_name(), 'localhost') + self.assertEqual(self.host.get_address(), 'test.org') self.host.set_address('001.002.003.004') self.assertEqual(self.host.get_protocol(), 'dummy') - self.assertEqual(self.host.get_name(), 'localhost') - self.assertEqual(self.host.get_address(), '1.2.3.4') + self.assertEqual(self.host.get_name(), 'localhost') + self.assertEqual(self.host.get_address(), '1.2.3.4') def testGetAddress(self): self.assertEqual(self.host.get_address(), 'localhost') @@ -71,8 +68,8 @@ def testSetName(self): self.host.set_protocol('dummy') self.host.set_name('test.org') self.assertEqual(self.host.get_protocol(), 'dummy') - self.assertEqual(self.host.get_name(), 'test.org') - self.assertEqual(self.host.get_address(), 'localhost') + self.assertEqual(self.host.get_name(), 'test.org') + self.assertEqual(self.host.get_address(), 'localhost') self.host.set_name('testhost') self.assertEqual(self.host.get_name(), 'testhost') @@ -91,8 +88,7 @@ def testSetOption(self): self.assertRaises(TypeError, self.host.set_option, 'test', True) self.assertEqual(self.host.get_options(), {}) self.assertEqual(self.host.get_option('verify_fingerprint'), None) - self.assertEqual( - self.host.get_option('verify_fingerprint', False), False) + self.assertEqual(self.host.get_option('verify_fingerprint', False), False) self.host.set_option('verify_fingerprint', True) self.assertEqual(self.host.get_option('verify_fingerprint'), True) self.assertEqual(self.host.get_options(), {'verify_fingerprint': True}) @@ -130,7 +126,7 @@ def testSet(self): def testSetAll(self): self.testSet() self.host.set_all({'test1': 1, 'test2': 2}) - self.assertEqual(self.host.get('test'), None) + self.assertEqual(self.host.get('test'), None) self.assertEqual(self.host.get('test1'), 1) self.assertEqual(self.host.get('test2'), 2) @@ -151,11 +147,11 @@ def testAppend(self): def testSetDefault(self): self.testSet() - self.assertEqual(self.host.get('test'), 3) + self.assertEqual(self.host.get('test'), 3) self.assertEqual(self.host.get('test2'), None) - self.host.set_default('test', 5) + self.host.set_default('test', 5) self.host.set_default('test2', 1) - self.assertEqual(self.host.get('test'), 3) + self.assertEqual(self.host.get('test'), 3) self.assertEqual(self.host.get('test2'), 1) def testHasKey(self): @@ -165,13 +161,15 @@ def testHasKey(self): def testGet(self): self.testSet() - self.assertEqual(self.host.get('test'), 3) - self.assertEqual(self.host.get('test2'), None) - self.assertEqual(self.host.get('test', 1), 3) + self.assertEqual(self.host.get('test'), 3) + self.assertEqual(self.host.get('test2'), None) + self.assertEqual(self.host.get('test', 1), 3) self.assertEqual(self.host.get('test2', 1), 1) def suite(): return unittest.TestLoader().loadTestsFromTestCase(HostTest) + + if __name__ == '__main__': unittest.TextTestRunner(verbosity=2).run(suite()) diff --git a/tests/Exscript/KnownIssuesTest.py b/tests/Exscript/KnownIssuesTest.py index 37cc2a9e..ce3a0692 100644 --- a/tests/Exscript/KnownIssuesTest.py +++ b/tests/Exscript/KnownIssuesTest.py @@ -6,6 +6,7 @@ an "unexpected success" and the suite fails, so the fix must remove the marker and move the test next to the tests of the fixed module. """ + import os import socket import sys @@ -53,14 +54,13 @@ def start_daemon(self, daemon_cls, device, **kwargs): port = free_port() daemon = daemon_cls(HOST, port, device, **kwargs) daemon.start() - time.sleep(.2) + time.sleep(0.2) self.addCleanup(daemon.join) # cleanups run LIFO: exit() first self.addCleanup(daemon.exit) return port class HostKeyTest(ServerMixin, unittest.TestCase): - @unittest.expectedFailure # S1: known_hosts is only loaded after the check; no [host]:port lookup def testKnownHostsMismatchIsRejected(self): port = self.start_daemon(SSHd, VirtualDevice(HOST), key=PrivateKey.from_file(KEYFILE)) @@ -101,13 +101,16 @@ def testTelnet(self): @unittest.expectedFailure # SSH2 only searches the last 150 characters after each read def testSSH2(self): port = self.start_daemon(SSHd, self.device, key=PrivateKey.from_file(KEYFILE)) - conn = SSH2(timeout=1, connect_rate_limit=0, verify_fingerprint=False, - preferred_authentications=['password']) + conn = SSH2( + timeout=1, + connect_rate_limit=0, + verify_fingerprint=False, + preferred_authentications=['password'], + ) self.expectMarker(conn, port) class SecretHygieneTest(unittest.TestCase): - @unittest.expectedFailure # S3: Protocol.send() logs passwords at debug level 4 def testDebugLogRedactsPassword(self): conn = Dummy(device=VirtualDevice('r1', echo=False, strict=False), debug=4) @@ -120,7 +123,6 @@ def testDebugLogRedactsPassword(self): @unittest.skipIf(ExscriptInstrumentor is None, 'requires the "otel" extra') class OtelTest(unittest.TestCase): - def setUp(self): self.exporter = InMemorySpanExporter() provider = TracerProvider() @@ -145,22 +147,27 @@ def testLoginWithPositionalAccount(self): @unittest.expectedFailure # S2: commands (incl. passwords) and responses are exported def testNoSecretsOrDeviceOutputExportedByDefault(self): self.runSession(lambda conn, account: conn.login(account=account)) - exported = [str(value) - for span in self.exporter.get_finished_spans() - for value in (span.attributes or {}).values()] + exported = [ + str(value) + for span in self.exporter.get_finished_spans() + for value in (span.attributes or {}).values() + ] self.assertEqual([v for v in exported if PASSWORD in v or 'PRIVATE-COMMUNITY' in v], []) class DriverTest(unittest.TestCase): - @unittest.expectedFailure # assigned to self._error_re instead of self.error_re def testIsamUsesItsErrorPatterns(self): self.assertIs(driver_map['isam'].error_re, isam._error_re) @unittest.expectedFailure # any "Last login" line selects ericsson_ban, which never matches "$" def testLastLoginLineDoesNotBreakLogin(self): - device = VirtualDevice('host', login_type=VirtualDevice.LOGIN_TYPE_NONE, strict=False, - banner='Welcome\nLast login: Mon Oct 26 10:00:00 2026 from 192.0.2.1\n') + device = VirtualDevice( + 'host', + login_type=VirtualDevice.LOGIN_TYPE_NONE, + strict=False, + banner='Welcome\nLast login: Mon Oct 26 10:00:00 2026 from 192.0.2.1\n', + ) device.set_prompt('user@host:~$ ') device.add_command('uname', 'Linux') conn = Dummy(device=device) @@ -171,7 +178,6 @@ def testLastLoginLineDoesNotBreakLogin(self): class ConnectRateLimitTest(unittest.TestCase): - def enforce(self, host, starts=None): conn = SSH2(connect_rate_limit=60) # 1.1 s spacing per host conn.host = host @@ -185,12 +191,12 @@ def testOtherHostsAreNotDelayed(self): self.enforce(host + '-a') waiting = threading.Thread(target=self.enforce, args=(host + '-a',)) waiting.start() - time.sleep(.05) + time.sleep(0.05) start = time.monotonic() self.enforce(host + '-b') elapsed = time.monotonic() - start waiting.join() - self.assertLess(elapsed, .5) + self.assertLess(elapsed, 0.5) @unittest.expectedFailure # concurrent connects to one host may start together def testConcurrentConnectsToOneHostAreSpaced(self): @@ -208,5 +214,7 @@ def testConcurrentConnectsToOneHostAreSpaced(self): def suite(): return unittest.TestLoader().loadTestsFromModule(sys.modules[__name__]) + + if __name__ == '__main__': unittest.TextTestRunner(verbosity=2).run(suite()) diff --git a/tests/Exscript/LexerTest.py b/tests/Exscript/LexerTest.py index 1784a1ba..978b9935 100644 --- a/tests/Exscript/LexerTest.py +++ b/tests/Exscript/LexerTest.py @@ -5,7 +5,6 @@ class LexerTest(unittest.TestCase): - def testForward(self): # Public API for parsers built on parselib. lexer = Lexer(object) @@ -16,5 +15,7 @@ def testForward(self): def suite(): return unittest.TestLoader().loadTestsFromTestCase(LexerTest) + + if __name__ == '__main__': unittest.TextTestRunner(verbosity=2).run(suite()) diff --git a/tests/Exscript/LogTest.py b/tests/Exscript/LogTest.py index 0c0bf565..3050ad7c 100644 --- a/tests/Exscript/LogTest.py +++ b/tests/Exscript/LogTest.py @@ -73,5 +73,7 @@ def testHasEnded(self): def suite(): return unittest.TestLoader().loadTestsFromTestCase(LogTest) + + if __name__ == '__main__': unittest.TextTestRunner(verbosity=2).run(suite()) diff --git a/tests/Exscript/LogfileTest.py b/tests/Exscript/LogfileTest.py index 10e38389..7854ecad 100644 --- a/tests/Exscript/LogfileTest.py +++ b/tests/Exscript/LogfileTest.py @@ -46,5 +46,7 @@ def testSucceeded(self): def suite(): return unittest.TestLoader().loadTestsFromTestCase(LogfileTest) + + if __name__ == '__main__': unittest.TextTestRunner(verbosity=2).run(suite()) diff --git a/tests/Exscript/LoggerTest.py b/tests/Exscript/LoggerTest.py index 332f13a2..466f5427 100644 --- a/tests/Exscript/LoggerTest.py +++ b/tests/Exscript/LoggerTest.py @@ -161,5 +161,7 @@ def testLogSucceeded(self): def suite(): return unittest.TestLoader().loadTestsFromTestCase(LoggerTest) + + if __name__ == '__main__': unittest.TextTestRunner(verbosity=2).run(suite()) diff --git a/tests/Exscript/QueueTest.py b/tests/Exscript/QueueTest.py index e5a318f2..d80086ee 100644 --- a/tests/Exscript/QueueTest.py +++ b/tests/Exscript/QueueTest.py @@ -23,6 +23,7 @@ logger = logging.getLogger('TestSuite.QueueTest') logger.addHandler(logging.NullHandler()) + def count_calls(job, data, **kwargs): assert hasattr(job, 'start') assert 'testarg' in kwargs @@ -88,6 +89,8 @@ def read(self): class LogManager(BaseManager): pass + + LogManager.register('Log', Log) @@ -100,10 +103,7 @@ def createQueue(self, logdir=None, **kwargs): self.queue.destroy() self.out = self.manager.Log() self.err = self.manager.Log() - self.queue = Queue(mode=self.mode, - stdout=self.out, - stderr=self.err, - **kwargs) + self.queue = Queue(mode=self.mode, stdout=self.out, stderr=self.err, **kwargs) self.accm = self.queue.account_manager if logdir is not None: self.logger = FileLogger(logdir) @@ -147,20 +147,20 @@ def testConstructor(self): # Test all verbosity levels. levels = ( - (-1, 1, ('', ''), ('', ''), ('', 'tb')), - (-1, 2, ('', ''), ('', ''), ('', 'tb')), - (0, 1, ('', ''), ('', 'no_tb'), ('', 'tb')), - (0, 2, ('', ''), ('', 'no_tb'), ('', 'tb')), - (1, 1, ('hello', ''), ('hello', 'no_tb'), ('hello', 'tb')), - (1, 2, ('[', ''), ('[', 'no_tb'), ('[', 'tb')), - (2, 1, ('hello', ''), ('hello', 'tb'), ('hello', 'tb')), - (2, 2, ('[', ''), ('[', 'tb'), ('[', 'tb')), - (3, 1, ('hello', ''), ('hello', 'tb'), ('hello', 'tb')), - (3, 2, ('[', ''), ('[', 'tb'), ('[', 'tb')), - (4, 1, ('hello', ''), ('hello', 'tb'), ('hello', 'tb')), - (4, 2, ('[', ''), ('[', 'tb'), ('[', 'tb')), - (5, 1, ('hello', ''), ('hello', 'tb'), ('hello', 'tb')), - (5, 2, ('[', ''), ('[', 'tb'), ('[', 'tb')), + (-1, 1, ('', ''), ('', ''), ('', 'tb')), + (-1, 2, ('', ''), ('', ''), ('', 'tb')), + (0, 1, ('', ''), ('', 'no_tb'), ('', 'tb')), + (0, 2, ('', ''), ('', 'no_tb'), ('', 'tb')), + (1, 1, ('hello', ''), ('hello', 'no_tb'), ('hello', 'tb')), + (1, 2, ('[', ''), ('[', 'no_tb'), ('[', 'tb')), + (2, 1, ('hello', ''), ('hello', 'tb'), ('hello', 'tb')), + (2, 2, ('[', ''), ('[', 'tb'), ('[', 'tb')), + (3, 1, ('hello', ''), ('hello', 'tb'), ('hello', 'tb')), + (3, 2, ('[', ''), ('[', 'tb'), ('[', 'tb')), + (4, 1, ('hello', ''), ('hello', 'tb'), ('hello', 'tb')), + (4, 2, ('[', ''), ('[', 'tb'), ('[', 'tb')), + (5, 1, ('hello', ''), ('hello', 'tb'), ('hello', 'tb')), + (5, 2, ('[', ''), ('[', 'tb'), ('[', 'tb')), ) for level, max_threads, with_simple, with_error, with_fatal in levels: msg = 'Testing verbose=%s with max_threads=%s (%s)' @@ -198,11 +198,13 @@ def testCreatePipe(self): pipe = self.queue._create_pipe() pipe.send(('acquire-account', None)) response = pipe.recv() - expected = (account.__hash__(), - account.get_name(), - account.get_password(), - account.get_authorization_password(), - account.get_key()) + expected = ( + account.__hash__(), + account.get_name(), + account.get_password(), + account.get_authorization_password(), + account.get_key(), + ) self.assertEqual(response, expected) pipe.send(('release-account', account.__hash__())) response = pipe.recv() @@ -258,9 +260,11 @@ def start_cb(data, job, host, conn): match_called = Value(ctypes.c_bool, False) start_called = Value(ctypes.c_bool, False) account_hash = Value(ctypes.c_long, 0) - data = {'match-called': match_called, - 'start-called': start_called, - 'account-hash': account_hash} + data = { + 'match-called': match_called, + 'start-called': start_called, + 'account-hash': account_hash, + } self.queue.add_account_pool(pool2, partial(match_cb, data)) self.assertEqual(self.accm.default_pool, pool1) @@ -269,9 +273,9 @@ def start_cb(data, job, host, conn): self.queue.run('dummy://dummy', partial(start_cb, data)) self.queue.shutdown() data = dict((k, v.value) for (k, v) in list(data.items())) - self.assertEqual(data, {'match-called': True, - 'start-called': True, - 'account-hash': account2.__hash__()}) + self.assertEqual( + data, {'match-called': True, 'start-called': True, 'account-hash': account2.__hash__()} + ) def startTask(self): self.testAddAccount() @@ -295,14 +299,14 @@ def testJoin(self): self.assertTrue(self.queue.is_completed()) def testShutdown(self): - task = self.startTask() # this also adds an account + task = self.startTask() # this also adds an account self.queue.shutdown() self.assertTrue(task.is_completed()) self.assertTrue(self.queue.is_completed()) self.assertEqual(self.accm.default_pool.n_accounts(), 1) def testDestroy(self): - task = self.startTask() # this also adds an account + task = self.startTask() # this also adds an account self.queue.destroy() self.assertTrue(self.queue.is_completed()) self.assertEqual(self.accm.default_pool.n_accounts(), 0) @@ -328,7 +332,7 @@ def testRun(self): data = Value('i', 0) hosts = ['dummy://dummy1', 'dummy://dummy2'] func = bind(count_calls2, data, testarg=1) - self.queue.run(hosts, func) + self.queue.run(hosts, func) self.queue.run('dummy://dummy3', func) self.queue.shutdown() self.assertEqual(data.value, 3) @@ -342,7 +346,7 @@ def testRunOrIgnore(self): hosts = ['dummy://dummy1', 'dummy://dummy2', 'dummy://dummy1'] func = bind(count_calls2, data, testarg=1) self.queue.workqueue.pause() - self.queue.run_or_ignore(hosts, func) + self.queue.run_or_ignore(hosts, func) self.queue.run_or_ignore('dummy://dummy2', func) self.queue.workqueue.unpause() self.queue.shutdown() @@ -372,7 +376,7 @@ def testPriorityRunOrRaise(self): hosts = ['dummy://dummy1', 'dummy://dummy2', 'dummy://dummy1'] func = bind(count_calls2, data, testarg=1) self.queue.workqueue.pause() - self.queue.priority_run_or_raise(hosts, func) + self.queue.priority_run_or_raise(hosts, func) self.queue.priority_run_or_raise('dummy://dummy2', func) self.queue.workqueue.unpause() self.queue.shutdown() @@ -414,11 +418,11 @@ def testEnqueue(self): def testQueueLogging(self): task = self.startTask() while not task.is_completed(): - time.sleep(.2) + time.sleep(0.2) # The following function is not synchronous with the above, so add # a timeout to avoid races. - time.sleep(.2) + time.sleep(0.2) self.assertTrue(self.queue.is_completed()) logfiles = os.listdir(self.tempdir) @@ -442,5 +446,7 @@ def suite(): suite1 = loader.loadTestsFromTestCase(QueueTest) # suite2 = loader.loadTestsFromTestCase(QueueTestMultiProcessing) return unittest.TestSuite((suite1,)) + + if __name__ == '__main__': unittest.TextTestRunner(verbosity=2).run(suite()) diff --git a/tests/Exscript/TemplateTest.py b/tests/Exscript/TemplateTest.py index c9fd9584..2526a9e2 100644 --- a/tests/Exscript/TemplateTest.py +++ b/tests/Exscript/TemplateTest.py @@ -47,23 +47,22 @@ def dummy_cb(job, host, conn, template_test): log.data = log.data.replace('\r\n', '\n') with open(expected, newline='') as fp: expected_data = fp.read() - #open(expected, 'w').write(log.data) - #open('output', 'w').write(log.data) - #open('exp', 'w').write(expected_data) + # open(expected, 'w').write(log.data) + # open('output', 'w').write(log.data) + # open('exp', 'w').write(expected_data) template_test.assertEqual(log.data, expected_data) class IOSDummy(Dummy): - def __init__(self, *args, **kwargs): device = IOSEmulator('dummy', strict=False) Dummy.__init__(self, device=device, **kwargs) + protocols.protocol_map['ios'] = IOSDummy class TemplateTest(unittest.TestCase): - def setUp(self): account = Account('sab', '') self.queue = Queue(verbose=0, max_threads=1) @@ -95,5 +94,7 @@ def testTemplates(self): def suite(): return unittest.TestLoader().loadTestsFromTestCase(TemplateTest) + + if __name__ == '__main__': unittest.TextTestRunner(verbosity=2).run(suite()) diff --git a/tests/Exscript/emulators/CommandSetTest.py b/tests/Exscript/emulators/CommandSetTest.py index 735c9fa3..d55595dd 100644 --- a/tests/Exscript/emulators/CommandSetTest.py +++ b/tests/Exscript/emulators/CommandSetTest.py @@ -25,6 +25,7 @@ def testAdd(self): def sayhello(cmd): return 'hello' + cs.add('hi', sayhello) self.assertEqual(cs.eval('hi'), 'hello') @@ -37,5 +38,7 @@ def testEval(self): def suite(): return unittest.TestLoader().loadTestsFromTestCase(CommandSetTest) + + if __name__ == '__main__': unittest.TextTestRunner(verbosity=2).run(suite()) diff --git a/tests/Exscript/emulators/IOSEmulatorTest.py b/tests/Exscript/emulators/IOSEmulatorTest.py index b43cc05c..ed3267c8 100644 --- a/tests/Exscript/emulators/IOSEmulatorTest.py +++ b/tests/Exscript/emulators/IOSEmulatorTest.py @@ -18,17 +18,15 @@ class IOSEmulatorTest(VirtualDeviceTest): def testAddCommand(self): VirtualDeviceTest.testAddCommand(self) - cs = self.cls('myhost', - strict=True, - echo=False, - login_type=self.cls.LOGIN_TYPE_NONE) + cs = self.cls('myhost', strict=True, echo=False, login_type=self.cls.LOGIN_TYPE_NONE) response = cs.do('show version') - self.assertTrue(response.startswith( - '\nCisco Internetwork Operating'), response) + self.assertTrue(response.startswith('\nCisco Internetwork Operating'), response) def suite(): return unittest.TestLoader().loadTestsFromTestCase(IOSEmulatorTest) + + if __name__ == '__main__': unittest.TextTestRunner(verbosity=2).run(suite()) diff --git a/tests/Exscript/emulators/VirtualDeviceTest.py b/tests/Exscript/emulators/VirtualDeviceTest.py index 0c3ae139..c3fa2a94 100644 --- a/tests/Exscript/emulators/VirtualDeviceTest.py +++ b/tests/Exscript/emulators/VirtualDeviceTest.py @@ -24,16 +24,14 @@ def testSetPrompt(self): self.testGetPrompt() def testAddCommand(self): - cs = self.cls('myhost', - strict=True, - echo=False, - login_type=self.cls.LOGIN_TYPE_NONE) + cs = self.cls('myhost', strict=True, echo=False, login_type=self.cls.LOGIN_TYPE_NONE) self.assertRaises(Exception, cs.do, 'foo') cs.add_command('foo', 'bar') self.assertEqual(cs.do('foo'), 'bar\n' + self.prompt) def sayhello(cmd): return 'hello' + cs.add_command('hi$', sayhello) self.assertEqual(cs.do('hi'), 'hello\n' + self.prompt) cs.add_command('hi2$', sayhello, prompt=False) @@ -43,20 +41,16 @@ def testAddCommandsFromFile(self): pass # TODO: Implement missing test. def testInit(self): - cs = self.cls('myhost', - login_type=self.cls.LOGIN_TYPE_PASSWORDONLY) + cs = self.cls('myhost', login_type=self.cls.LOGIN_TYPE_PASSWORDONLY) self.assertEqual(cs.init(), self.banner + self.passwdprompt) - cs = self.cls('myhost', - login_type=self.cls.LOGIN_TYPE_USERONLY) + cs = self.cls('myhost', login_type=self.cls.LOGIN_TYPE_USERONLY) self.assertEqual(cs.init(), self.banner + self.userprompt) - cs = self.cls('myhost', - login_type=self.cls.LOGIN_TYPE_BOTH) + cs = self.cls('myhost', login_type=self.cls.LOGIN_TYPE_BOTH) self.assertEqual(cs.init(), self.banner + self.userprompt) - cs = self.cls('myhost', - login_type=self.cls.LOGIN_TYPE_NONE) + cs = self.cls('myhost', login_type=self.cls.LOGIN_TYPE_NONE) self.assertEqual(cs.init(), self.banner + self.prompt) def testDo(self): @@ -65,5 +59,7 @@ def testDo(self): def suite(): return unittest.TestLoader().loadTestsFromTestCase(VirtualDeviceTest) + + if __name__ == '__main__': unittest.TextTestRunner(verbosity=2).run(suite()) diff --git a/tests/Exscript/emulators/run_suite.py b/tests/Exscript/emulators/run_suite.py index 0520fafd..49b76509 100755 --- a/tests/Exscript/emulators/run_suite.py +++ b/tests/Exscript/emulators/run_suite.py @@ -13,9 +13,11 @@ logger = logging.getLogger('TestSuite') logger.addHandler(logging.NullHandler()) + def uppercase(match): return match.group(1).upper() + correlated = dict() @@ -84,7 +86,9 @@ def decorator(func): def wrapper(*args, **kwargs): logger.debug(msg) return func(*args, **kwargs) + return wrapper + return decorator @@ -105,7 +109,7 @@ def correlate_class(theclass): continue # Format the function names. - testname = re.sub(r'_(\w)', uppercase, name) + testname = re.sub(r'_(\w)', uppercase, name) testname = re.sub(r'(\d\w)', uppercase, testname) testname = 'test' + re.sub(r'^(\w)', uppercase, testname) testmethod = theclass.__name__ + '.' + testname @@ -115,8 +119,7 @@ def correlate_class(theclass): # Throw an error if the function does not have a test. if testname in dir(theclass): continue - if ismodule(theclass.CORRELATE) and \ - value.__module__ != theclass.CORRELATE.__name__: + if ismodule(theclass.CORRELATE) and value.__module__ != theclass.CORRELATE.__name__: continue # function was imported. if both in correlated: continue @@ -170,6 +173,7 @@ def suite(): def recursive_suite(): return load_suite(find('.', '*Test.py')) + if __name__ == '__main__': # Parse CLI options. if len(sys.argv) == 1: diff --git a/tests/Exscript/protocols/DummyTest.py b/tests/Exscript/protocols/DummyTest.py index cdf547ee..43e64818 100644 --- a/tests/Exscript/protocols/DummyTest.py +++ b/tests/Exscript/protocols/DummyTest.py @@ -47,9 +47,9 @@ def testDummy(self): protocol.close() # Test login with user prompt. - device = VirtualDevice(self.hostname, - echo=True, - login_type=VirtualDevice.LOGIN_TYPE_USERONLY) + device = VirtualDevice( + self.hostname, echo=True, login_type=VirtualDevice.LOGIN_TYPE_USERONLY + ) protocol = self._create_dummy_and_eat_banner(device) self.assertEqual(str(protocol._buffer), 'User: ') protocol.login(self.account, flush=False) @@ -57,9 +57,9 @@ def testDummy(self): protocol.close() # Test login with password prompt. - device = VirtualDevice(self.hostname, - echo=True, - login_type=VirtualDevice.LOGIN_TYPE_PASSWORDONLY) + device = VirtualDevice( + self.hostname, echo=True, login_type=VirtualDevice.LOGIN_TYPE_PASSWORDONLY + ) protocol = self._create_dummy_and_eat_banner(device) self.assertEqual(str(protocol._buffer), 'Password: ') protocol.login(self.account, flush=False) @@ -67,17 +67,15 @@ def testDummy(self): protocol.close() # Test login without user/password prompt. - device = VirtualDevice(self.hostname, - echo=True, - login_type=VirtualDevice.LOGIN_TYPE_NONE) + device = VirtualDevice(self.hostname, echo=True, login_type=VirtualDevice.LOGIN_TYPE_NONE) protocol = self._create_dummy_and_eat_banner(device) self.assertEqual(str(protocol._buffer), self.prompt) protocol.close() # Test login with user prompt and wait parameter. - device = VirtualDevice(self.hostname, - echo=True, - login_type=VirtualDevice.LOGIN_TYPE_USERONLY) + device = VirtualDevice( + self.hostname, echo=True, login_type=VirtualDevice.LOGIN_TYPE_USERONLY + ) protocol = self._create_dummy_and_eat_banner(device) self.assertEqual(str(protocol._buffer), 'User: ') protocol.login(self.account) @@ -86,13 +84,13 @@ def testDummy(self): protocol.close() # Test login with password prompt and wait parameter. - device = VirtualDevice(self.hostname, - echo=True, - login_type=VirtualDevice.LOGIN_TYPE_PASSWORDONLY) + device = VirtualDevice( + self.hostname, echo=True, login_type=VirtualDevice.LOGIN_TYPE_PASSWORDONLY + ) protocol = self._create_dummy_and_eat_banner(device) self.assertEqual(str(protocol._buffer), 'Password: ') protocol.login(self.account) - self.assertEqual(str(protocol._buffer), '') + self.assertEqual(str(protocol._buffer), '') self.assertEqual(protocol.response, self.password) protocol.close() @@ -105,9 +103,7 @@ def testDummy(self): protocol.close() # Test a custom response. - device = VirtualDevice(self.hostname, - echo=True, - login_type=VirtualDevice.LOGIN_TYPE_NONE) + device = VirtualDevice(self.hostname, echo=True, login_type=VirtualDevice.LOGIN_TYPE_NONE) protocol = Dummy(device=device) command = re.compile(r'testcommand') response = 'hello world\r\n%s> ' % self.hostname @@ -129,5 +125,7 @@ def testDummy(self): def suite(): return unittest.TestLoader().loadTestsFromTestCase(DummyTest) + + if __name__ == '__main__': unittest.TextTestRunner(verbosity=2).run(suite()) diff --git a/tests/Exscript/protocols/OsGuesserCorpusTest.py b/tests/Exscript/protocols/OsGuesserCorpusTest.py index 911b059b..30ae530f 100644 --- a/tests/Exscript/protocols/OsGuesserCorpusTest.py +++ b/tests/Exscript/protocols/OsGuesserCorpusTest.py @@ -11,6 +11,7 @@ After an intended change, regenerate the file with ``EXSCRIPT_UPDATE_GOLDEN=1`` and review the diff. """ + import json import os import unittest @@ -24,29 +25,48 @@ PROTOCOL_INFOS = ['', 'SSH-2.0-Cisco-1.25', 'SSH-2.0-OpenSSH_9.6'] READ_SIZES = [64, 65536] GENERIC_OUTPUT = '\r\n'.join( - ['interface GigabitEthernet0/%d\r\n description uplink #%d to core\r\n' - ' ip address 10.0.%d.1 255.255.255.0\r\n!' % (i, i, i) for i in range(40)] + [ + 'interface GigabitEthernet0/%d\r\n description uplink #%d to core\r\n' + ' ip address 10.0.%d.1 255.255.255.0\r\n!' % (i, i, i) + for i in range(40) + ] + ['# end of config', 'Building configuration...', 'Current configuration : 8231 bytes'] ) -PROMPT_ONLY = ['router#', 'switch>', 'RP/0/RSP0/CPU0:xr-pe1#', 'user@host:~$', 'root@host:~#', - '', 'A:sros-pe1#', '[admin@MikroTik] >', 'FGT60E #', 'core-sw01(config)#', - 'admin@junos-mx> '] +PROMPT_ONLY = [ + 'router#', + 'switch>', + 'RP/0/RSP0/CPU0:xr-pe1#', + 'user@host:~$', + 'root@host:~#', + '', + 'A:sros-pe1#', + '[admin@MikroTik] >', + 'FGT60E #', + 'core-sw01(config)#', + 'admin@junos-mx> ', +] def feed(guesser, data, authenticated, size): for i in range(0, len(data), size): - guesser.check_received_data(data[i:i + size], authenticated) + guesser.check_received_data(data[i : i + size], authenticated) def snapshot(guesser): - return (dict(guesser._info), guesser._auth_buffer, - guesser._protocol_buffer, guesser._response_buffer) + return ( + dict(guesser._info), + guesser._auth_buffer, + guesser._protocol_buffer, + guesser._response_buffer, + ) def restore(state): guesser = OsGuesser() - (guesser._info, guesser._auth_buffer, - guesser._protocol_buffer, guesser._response_buffer) = dict(state[0]), *state[1:] + (guesser._info, guesser._auth_buffer, guesser._protocol_buffer, guesser._response_buffer) = ( + dict(state[0]), + *state[1:], + ) return guesser @@ -76,13 +96,17 @@ def run_corpus(): for info in PROTOCOL_INFOS: replay(name, banner, info, '\r\n' + GENERIC_OUTPUT + '\r\n' + last_line, results) for prompt in PROMPT_ONLY: - replay('prompt-only:' + prompt, 'Last login: Mon Oct 26 10:00:00 2026\r\n', - 'SSH-2.0-OpenSSH_9.6', '\r\n' + GENERIC_OUTPUT + '\r\n' + prompt, results) + replay( + 'prompt-only:' + prompt, + 'Last login: Mon Oct 26 10:00:00 2026\r\n', + 'SSH-2.0-OpenSSH_9.6', + '\r\n' + GENERIC_OUTPUT + '\r\n' + prompt, + results, + ) return results class OsGuesserCorpusTest(unittest.TestCase): - def testCorpusMatchesBaseline(self): results = run_corpus() if os.environ.get('EXSCRIPT_UPDATE_GOLDEN'): @@ -92,13 +116,18 @@ def testCorpusMatchesBaseline(self): self.skipTest('golden file regenerated: ' + GOLDEN) with open(GOLDEN, encoding='utf-8') as fp: expected = json.load(fp) - changed = sorted(key for key in expected.keys() | results.keys() - if expected.get(key) != results.get(key)) - details = ['%s: %s -> %s' % (key, expected.get(key), results.get(key)) for key in changed[:20]] + changed = sorted( + key for key in expected.keys() | results.keys() if expected.get(key) != results.get(key) + ) + details = [ + '%s: %s -> %s' % (key, expected.get(key), results.get(key)) for key in changed[:20] + ] self.assertEqual(changed, [], 'OS guessing changed:\n' + '\n'.join(details)) def suite(): return unittest.TestLoader().loadTestsFromTestCase(OsGuesserCorpusTest) + + if __name__ == '__main__': unittest.TextTestRunner(verbosity=2).run(suite()) diff --git a/tests/Exscript/protocols/OsGuesserTest.py b/tests/Exscript/protocols/OsGuesserTest.py index dcec0381..ab158637 100644 --- a/tests/Exscript/protocols/OsGuesserTest.py +++ b/tests/Exscript/protocols/OsGuesserTest.py @@ -23,32 +23,34 @@ def testReset(self): self.testSet() def testSet(self): - self.assertEqual(self.sa.get('test'), None) - self.assertEqual(self.sa.get('test', 0), None) - self.assertEqual(self.sa.get('test', 50), None) + self.assertEqual(self.sa.get('test'), None) + self.assertEqual(self.sa.get('test', 0), None) + self.assertEqual(self.sa.get('test', 50), None) self.assertEqual(self.sa.get('test', 100), None) self.sa.set('test', 'foo', 0) - self.assertEqual(self.sa.get('test'), 'foo') - self.assertEqual(self.sa.get('test', 0), 'foo') + self.assertEqual(self.sa.get('test'), 'foo') + self.assertEqual(self.sa.get('test', 0), 'foo') self.assertEqual(self.sa.get('test', 10), None) self.sa.set('test', 'foo', 10) - self.assertEqual(self.sa.get('test'), 'foo') - self.assertEqual(self.sa.get('test', 0), 'foo') + self.assertEqual(self.sa.get('test'), 'foo') + self.assertEqual(self.sa.get('test', 0), 'foo') self.assertEqual(self.sa.get('test', 10), 'foo') self.assertEqual(self.sa.get('test', 11), None) self.sa.set('test', 'foo', 5) - self.assertEqual(self.sa.get('test'), 'foo') - self.assertEqual(self.sa.get('test', 0), 'foo') + self.assertEqual(self.sa.get('test'), 'foo') + self.assertEqual(self.sa.get('test', 0), 'foo') self.assertEqual(self.sa.get('test', 10), 'foo') self.assertEqual(self.sa.get('test', 11), None) def testSetFromMatch(self): - match_list = ((re.compile('on'), 'uno', 50), - (re.compile('two'), 'doe', 0), - (re.compile('one'), 'eins', 90)) + match_list = ( + (re.compile('on'), 'uno', 50), + (re.compile('two'), 'doe', 0), + (re.compile('one'), 'eins', 90), + ) self.assertEqual(self.sa.get('test'), None) self.sa.set_from_match('test', match_list, '2two2') @@ -88,16 +90,20 @@ def testCheckProtocolInfo(self): def testCheckNetconfCapabilities(self): osg = OsGuesser() - osg.check_netconf_capabilities([ - 'urn:ietf:params:netconf:base:1.0', - 'urn:ietf:params:netconf:capability:candidate:1.0', - 'urn:ietf:params:netconf:capability:confirmed-commit:1.0', - 'http://xml.juniper.net/netconf/junos/1.0' - ]) + osg.check_netconf_capabilities( + [ + 'urn:ietf:params:netconf:base:1.0', + 'urn:ietf:params:netconf:capability:candidate:1.0', + 'urn:ietf:params:netconf:capability:confirmed-commit:1.0', + 'http://xml.juniper.net/netconf/junos/1.0', + ] + ) self.assertEqual(osg.get('os'), "junos") def suite(): return unittest.TestLoader().loadTestsFromTestCase(OsGuesserTest) + + if __name__ == '__main__': unittest.TextTestRunner(verbosity=2).run(suite()) diff --git a/tests/Exscript/protocols/ProtocolTest.py b/tests/Exscript/protocols/ProtocolTest.py index 9e47ab19..33baa8ea 100644 --- a/tests/Exscript/protocols/ProtocolTest.py +++ b/tests/Exscript/protocols/ProtocolTest.py @@ -21,11 +21,11 @@ class ProtocolTest(unittest.TestCase): - """ Since protocols.Protocol is abstract, this test is only a base class for other protocols. It does not do anything fancy on its own. """ + CORRELATE = Protocol def setUp(self): @@ -40,7 +40,7 @@ def setUp(self): self.createDaemon() if self.daemon is not None: self.daemon.start() - time.sleep(.2) + time.sleep(0.2) # LIFO cleanup order! self.addCleanup(self.daemon.join) self.addCleanup(self.daemon.exit) @@ -57,11 +57,10 @@ def createVirtualDevice(self): self.prompt = self.hostname + '> ' self.device = VirtualDevice(self.hostname, echo=True) ls_response = '-rw-r--r-- 1 sab nmc 1628 Aug 18 10:02 file' - self.device.add_command('ls', ls_response) - self.device.add_command('df', 'foobar') + self.device.add_command('ls', ls_response) + self.device.add_command('df', 'foobar') self.device.add_command('exit', '') - self.device.add_command('this-command-causes-an-error', - '\ncommand not found') + self.device.add_command('this-command-causes-an-error', '\ncommand not found') def createDaemon(self): pass @@ -99,35 +98,39 @@ def _trymatch(self, prompts, string): return None def testPrompts(self): - prompts = ('[sam123@home ~]$', - '[MyHost-A1]', - '', - 'sam@knip:~/Code/exscript$', - 'sam@MyHost-X123>', - 'sam@MyHost-X123#', - 'MyHost-ABC-CDE123>', - 'MyHost-A1#', - 'S-ABC#', - '0123456-1-1-abc#', - '0123456-1-1-a>', - 'MyHost-A1(config)#', - 'MyHost-A1(config)>', - 'RP/0/RP0/CPU0:A-BC2#', - 'FA/0/1/2/3>', - 'FA/0/1/2/3(config)>', - 'FA/0/1/2/3(config)#', - 'ec-c3-c27s99(su)->', - 'foobar:0>', - 'admin@s-x-a6.a.bc.de.fg:/# ', - 'admin@s-x-a6.a.bc.de.fg:/% ') - notprompts = ('one two', - ' [MyHost-A1]', - '[edit]\r', - '[edit]\n', - '[edit foo]\r', - '[edit foo]\n', - '[edit foo]\r\n', - '[edit one two]') + prompts = ( + '[sam123@home ~]$', + '[MyHost-A1]', + '', + 'sam@knip:~/Code/exscript$', + 'sam@MyHost-X123>', + 'sam@MyHost-X123#', + 'MyHost-ABC-CDE123>', + 'MyHost-A1#', + 'S-ABC#', + '0123456-1-1-abc#', + '0123456-1-1-a>', + 'MyHost-A1(config)#', + 'MyHost-A1(config)>', + 'RP/0/RP0/CPU0:A-BC2#', + 'FA/0/1/2/3>', + 'FA/0/1/2/3(config)>', + 'FA/0/1/2/3(config)#', + 'ec-c3-c27s99(su)->', + 'foobar:0>', + 'admin@s-x-a6.a.bc.de.fg:/# ', + 'admin@s-x-a6.a.bc.de.fg:/% ', + ) + notprompts = ( + 'one two', + ' [MyHost-A1]', + '[edit]\r', + '[edit]\n', + '[edit foo]\r', + '[edit foo]\n', + '[edit foo]\r\n', + '[edit one two]', + ) prompt_re = self.protocol.get_prompt() for prompt in prompts: if not self._trymatch(prompt_re, '\n' + prompt): @@ -240,36 +243,37 @@ def _test_prompt_setter(self, getter, setter): self.assertEqual(regex, initial_regex) def testSetUsernamePrompt(self): - self._test_prompt_setter(self.protocol.get_username_prompt, - self.protocol.set_username_prompt) + self._test_prompt_setter( + self.protocol.get_username_prompt, self.protocol.set_username_prompt + ) def testGetUsernamePrompt(self): pass # Already tested in testSetUsernamePrompt() def testSetPasswordPrompt(self): - self._test_prompt_setter(self.protocol.get_password_prompt, - self.protocol.set_password_prompt) + self._test_prompt_setter( + self.protocol.get_password_prompt, self.protocol.set_password_prompt + ) def testGetPasswordPrompt(self): pass # Already tested in testSetPasswordPrompt() def testSetPrompt(self): - self._test_prompt_setter(self.protocol.get_prompt, - self.protocol.set_prompt) + self._test_prompt_setter(self.protocol.get_prompt, self.protocol.set_prompt) def testGetPrompt(self): pass # Already tested in testSetPrompt() def testSetErrorPrompt(self): - self._test_prompt_setter(self.protocol.get_error_prompt, - self.protocol.set_error_prompt) + self._test_prompt_setter(self.protocol.get_error_prompt, self.protocol.set_error_prompt) def testGetErrorPrompt(self): pass # Already tested in testSetErrorPrompt() def testSetLoginErrorPrompt(self): - self._test_prompt_setter(self.protocol.get_login_error_prompt, - self.protocol.set_login_error_prompt) + self._test_prompt_setter( + self.protocol.get_login_error_prompt, self.protocol.set_login_error_prompt + ) def testGetLoginErrorPrompt(self): pass # Already tested in testSetLoginErrorPrompt() @@ -303,9 +307,7 @@ def testConnect(self): def testLogin(self): # Test can not work on the abstract base. if self.protocol.__class__ == Protocol: - self.assertRaises(Exception, - self.protocol.login, - self.account) + self.assertRaises(Exception, self.protocol.login, self.account) return # Password login. self.doLogin(flush=False) @@ -332,9 +334,7 @@ def testLogin(self): def testAuthenticate(self): # Test can not work on the abstract base. if self.protocol.__class__ == Protocol: - self.assertRaises(Exception, - self.protocol.authenticate, - self.account) + self.assertRaises(Exception, self.protocol.authenticate, self.account) return self.doConnect() @@ -381,9 +381,7 @@ def testIsProtocolAuthenticated(self): def testAppAuthenticate(self): # Test can not work on the abstract base. if self.protocol.__class__ == Protocol: - self.assertRaises(Exception, - self.protocol.app_authenticate, - self.account) + self.assertRaises(Exception, self.protocol.app_authenticate, self.account) return self.testProtocolAuthenticate() self.doAppAuthenticate(flush=False) @@ -574,9 +572,9 @@ def testExecute(self): # Make sure that we raise an error if the device responds # with something that matches any of the error prompts. self.protocol.set_error_prompt('.') - self.assertRaises(InvalidCommandException, - self.protocol.execute, - 'this-command-causes-an-error') + self.assertRaises( + InvalidCommandException, self.protocol.execute, 'this-command-causes-an-error' + ) def testWaitfor(self): # Test can not work on the abstract base. @@ -619,15 +617,16 @@ def testAddMonitor(self): def monitor_cb(thedata, *args, **kwargs): thedata['args'] = args thedata['kwargs'] = kwargs + data = {} self.protocol.add_monitor('abc', partial(monitor_cb, data)) # Simulate some non-matching data. - self.protocol._buffer.append(u'aaa') + self.protocol._buffer.append('aaa') self.assertEqual(data, {}) # Simulate some matching data. - self.protocol._buffer.append(u'abc') + self.protocol._buffer.append('abc') self.assertEqual(len(data.get('args')), 3) self.assertEqual(data.get('args')[0], self.protocol) self.assertEqual(data.get('args')[1], 0) @@ -658,9 +657,7 @@ def testCancelExpect(self): self.doLogin() self.protocol.data_received_event.connect(self._cancel_cb) self.protocol.send('ls\r') - self.assertRaises(ExpectCancelledException, - self.protocol.expect, - 'notgoingtohappen') + self.assertRaises(ExpectCancelledException, self.protocol.expect, 'notgoingtohappen') def testInteract(self): # Test can not work on the abstract base. @@ -675,18 +672,18 @@ def testSwitchDeviceContext(self): # Set up initial device state including OS guesser to verify capture/restore. # Simulate guesser state that would have been built up during a session. - self.protocol._os_guesser._info = {'os': ('ios', 80)} - self.protocol._os_guesser._auth_buffer = 'banner data' + self.protocol._os_guesser._info = {'os': ('ios', 80)} + self.protocol._os_guesser._auth_buffer = 'banner data' self.protocol._os_guesser._protocol_buffer = 'SSH-2.0-OpenSSH' self.protocol._os_guesser._response_buffer = 'router#' self.protocol._auto_driver = drivers.driver_map[self.protocol.guess_os()] - self.protocol._app_authenticated = True - self.protocol._app_authorized = True - self.protocol._last_account = self.account - self.protocol._manual_prompt_re = [re.compile(r'router#')] - self.protocol._manual_error_re = [re.compile(r'% Error')] - self.protocol._manual_user_re = [re.compile(r'Username:')] - self.protocol._manual_password_re = [re.compile(r'Password:')] + self.protocol._app_authenticated = True + self.protocol._app_authorized = True + self.protocol._last_account = self.account + self.protocol._manual_prompt_re = [re.compile(r'router#')] + self.protocol._manual_error_re = [re.compile(r'% Error')] + self.protocol._manual_user_re = [re.compile(r'Username:')] + self.protocol._manual_password_re = [re.compile(r'Password:')] self.protocol._manual_login_error_re = [re.compile(r'Login failed')] self.protocol.set_driver('junos') @@ -737,9 +734,9 @@ def testSwitchDeviceContext(self): # -- Nested switch: local shell (reset_auth=False, no driver param) -- hop_account = Account('hop_user', password='hop_pass') - self.protocol._app_authenticated = True # simulate auth on the hop - self.protocol._app_authorized = True - self.protocol._last_account = hop_account + self.protocol._app_authenticated = True # simulate auth on the hop + self.protocol._app_authorized = True + self.protocol._last_account = hop_account self.protocol.switch_device_context(reset_auth=False) self.assertEqual(len(self.protocol._device_context_stack), 2) @@ -875,8 +872,8 @@ def testReconnect(self): # close() resets session state; response and logfile path are preserved. self.assertFalse(self.protocol._connected) - self.assertIsNone(self.protocol._log) # FD closed by _reset() - self.assertEqual(logpath, self.protocol.logfile) # path survives close + self.assertIsNone(self.protocol._log) # FD closed by _reset() + self.assertEqual(logpath, self.protocol.logfile) # path survives close self.assertEqual(0, self.protocol._buffer.size()) self.assertEqual('unknown', self.protocol.guess_os()) self.assertFalse(self.protocol.is_app_authenticated()) @@ -912,5 +909,7 @@ def testReconnect(self): def suite(): return unittest.TestLoader().loadTestsFromTestCase(ProtocolTest) + + if __name__ == '__main__': unittest.TextTestRunner(verbosity=2).run(suite()) diff --git a/tests/Exscript/protocols/SSH2Test.py b/tests/Exscript/protocols/SSH2Test.py index 472b14e1..5890d2de 100644 --- a/tests/Exscript/protocols/SSH2Test.py +++ b/tests/Exscript/protocols/SSH2Test.py @@ -39,5 +39,7 @@ def testAuthenticate(self): def suite(): return unittest.TestLoader().loadTestsFromTestCase(SSH2Test) + + if __name__ == '__main__': unittest.TextTestRunner(verbosity=2).run(suite()) diff --git a/tests/Exscript/protocols/TelnetTest.py b/tests/Exscript/protocols/TelnetTest.py index 781257d8..80bd7f4f 100644 --- a/tests/Exscript/protocols/TelnetTest.py +++ b/tests/Exscript/protocols/TelnetTest.py @@ -23,5 +23,7 @@ def testConstructor(self): def suite(): return unittest.TestLoader().loadTestsFromTestCase(TelnetTest) + + if __name__ == '__main__': unittest.TextTestRunner(verbosity=2).run(suite()) diff --git a/tests/Exscript/protocols/pseudodev.py b/tests/Exscript/protocols/pseudodev.py index cfdda69e..3ef65d36 100644 --- a/tests/Exscript/protocols/pseudodev.py +++ b/tests/Exscript/protocols/pseudodev.py @@ -1,5 +1,7 @@ commands = ( - ('show version', """ + ( + 'show version', + """ Cisco Internetwork Operating System Software IOS (tm) GS Software (C12KPRP-P-M), Version 12.0(32)SY6c, RELEASE SOFTWARE (fc3) Technical Support: http://www.cisco.com/techsupport @@ -43,9 +45,11 @@ 250880K bytes of ATA PCMCIA card at slot 0 (Sector size 512 bytes). 65536K bytes of Flash internal SIMM (Sector size 256K). Configuration register is 0x2102 -"""), - - (r'show diag \d+', """ +""", + ), + ( + r'show diag \d+', + """ SLOT 0 (RP/LC 0 ): 16 Port ISE Packet Over SONET OC-3c/STM-1 Single Mode/IR LC connector MAIN: type 79, 800-19733-08 rev A0 Deviation: 0 @@ -70,5 +74,6 @@ TX Packet Memory size: 268435456 bytes, Packet Memory pagesize: 16384 bytes RX Packet Memory size: 268435456 bytes, Packet Memory pagesize: 16384 bytes 0 crashes since restart -"""), +""", + ), ) diff --git a/tests/Exscript/protocols/run_suite.py b/tests/Exscript/protocols/run_suite.py index 0520fafd..49b76509 100755 --- a/tests/Exscript/protocols/run_suite.py +++ b/tests/Exscript/protocols/run_suite.py @@ -13,9 +13,11 @@ logger = logging.getLogger('TestSuite') logger.addHandler(logging.NullHandler()) + def uppercase(match): return match.group(1).upper() + correlated = dict() @@ -84,7 +86,9 @@ def decorator(func): def wrapper(*args, **kwargs): logger.debug(msg) return func(*args, **kwargs) + return wrapper + return decorator @@ -105,7 +109,7 @@ def correlate_class(theclass): continue # Format the function names. - testname = re.sub(r'_(\w)', uppercase, name) + testname = re.sub(r'_(\w)', uppercase, name) testname = re.sub(r'(\d\w)', uppercase, testname) testname = 'test' + re.sub(r'^(\w)', uppercase, testname) testmethod = theclass.__name__ + '.' + testname @@ -115,8 +119,7 @@ def correlate_class(theclass): # Throw an error if the function does not have a test. if testname in dir(theclass): continue - if ismodule(theclass.CORRELATE) and \ - value.__module__ != theclass.CORRELATE.__name__: + if ismodule(theclass.CORRELATE) and value.__module__ != theclass.CORRELATE.__name__: continue # function was imported. if both in correlated: continue @@ -170,6 +173,7 @@ def suite(): def recursive_suite(): return load_suite(find('.', '*Test.py')) + if __name__ == '__main__': # Parse CLI options. if len(sys.argv) == 1: diff --git a/tests/Exscript/run_suite.py b/tests/Exscript/run_suite.py index 0520fafd..49b76509 100755 --- a/tests/Exscript/run_suite.py +++ b/tests/Exscript/run_suite.py @@ -13,9 +13,11 @@ logger = logging.getLogger('TestSuite') logger.addHandler(logging.NullHandler()) + def uppercase(match): return match.group(1).upper() + correlated = dict() @@ -84,7 +86,9 @@ def decorator(func): def wrapper(*args, **kwargs): logger.debug(msg) return func(*args, **kwargs) + return wrapper + return decorator @@ -105,7 +109,7 @@ def correlate_class(theclass): continue # Format the function names. - testname = re.sub(r'_(\w)', uppercase, name) + testname = re.sub(r'_(\w)', uppercase, name) testname = re.sub(r'(\d\w)', uppercase, testname) testname = 'test' + re.sub(r'^(\w)', uppercase, testname) testmethod = theclass.__name__ + '.' + testname @@ -115,8 +119,7 @@ def correlate_class(theclass): # Throw an error if the function does not have a test. if testname in dir(theclass): continue - if ismodule(theclass.CORRELATE) and \ - value.__module__ != theclass.CORRELATE.__name__: + if ismodule(theclass.CORRELATE) and value.__module__ != theclass.CORRELATE.__name__: continue # function was imported. if both in correlated: continue @@ -170,6 +173,7 @@ def suite(): def recursive_suite(): return load_suite(find('.', '*Test.py')) + if __name__ == '__main__': # Parse CLI options. if len(sys.argv) == 1: diff --git a/tests/Exscript/servers/SSHdTest.py b/tests/Exscript/servers/SSHdTest.py index 73658ed4..551074be 100644 --- a/tests/Exscript/servers/SSHdTest.py +++ b/tests/Exscript/servers/SSHdTest.py @@ -19,5 +19,7 @@ def _create_client(self): def suite(): return unittest.TestLoader().loadTestsFromTestCase(SSHdTest) + + if __name__ == '__main__': unittest.TextTestRunner(verbosity=2).run(suite()) diff --git a/tests/Exscript/servers/ServerTest.py b/tests/Exscript/servers/ServerTest.py index b24eb152..638b55f7 100644 --- a/tests/Exscript/servers/ServerTest.py +++ b/tests/Exscript/servers/ServerTest.py @@ -33,9 +33,9 @@ def _create_client(self): def _add_commands(self): self.device.add_command('exit', self.daemon.exit_command) - self.device.add_command('ls', 'ok1') - self.device.add_command('ll', 'ok2\nfoobar:1>', prompt=False) - self.device.add_command('.+', 'Unknown command.') + self.device.add_command('ls', 'ok1') + self.device.add_command('ll', 'ok2\nfoobar:1>', prompt=False) + self.device.add_command('.+', 'Unknown command.') def testConstructor(self): # Test can not work on the abstract base. @@ -81,5 +81,7 @@ def testExit(self): def suite(): return unittest.TestLoader().loadTestsFromTestCase(ServerTest) + + if __name__ == '__main__': unittest.TextTestRunner(verbosity=2).run(suite()) diff --git a/tests/Exscript/servers/TelnetdTest.py b/tests/Exscript/servers/TelnetdTest.py index 27e2d80e..35868d55 100644 --- a/tests/Exscript/servers/TelnetdTest.py +++ b/tests/Exscript/servers/TelnetdTest.py @@ -19,5 +19,7 @@ def _create_client(self): def suite(): return unittest.TestLoader().loadTestsFromTestCase(TelnetdTest) + + if __name__ == '__main__': unittest.TextTestRunner(verbosity=2).run(suite()) diff --git a/tests/Exscript/servers/run_suite.py b/tests/Exscript/servers/run_suite.py index 0520fafd..49b76509 100755 --- a/tests/Exscript/servers/run_suite.py +++ b/tests/Exscript/servers/run_suite.py @@ -13,9 +13,11 @@ logger = logging.getLogger('TestSuite') logger.addHandler(logging.NullHandler()) + def uppercase(match): return match.group(1).upper() + correlated = dict() @@ -84,7 +86,9 @@ def decorator(func): def wrapper(*args, **kwargs): logger.debug(msg) return func(*args, **kwargs) + return wrapper + return decorator @@ -105,7 +109,7 @@ def correlate_class(theclass): continue # Format the function names. - testname = re.sub(r'_(\w)', uppercase, name) + testname = re.sub(r'_(\w)', uppercase, name) testname = re.sub(r'(\d\w)', uppercase, testname) testname = 'test' + re.sub(r'^(\w)', uppercase, testname) testmethod = theclass.__name__ + '.' + testname @@ -115,8 +119,7 @@ def correlate_class(theclass): # Throw an error if the function does not have a test. if testname in dir(theclass): continue - if ismodule(theclass.CORRELATE) and \ - value.__module__ != theclass.CORRELATE.__name__: + if ismodule(theclass.CORRELATE) and value.__module__ != theclass.CORRELATE.__name__: continue # function was imported. if both in correlated: continue @@ -170,6 +173,7 @@ def suite(): def recursive_suite(): return load_suite(find('.', '*Test.py')) + if __name__ == '__main__': # Parse CLI options. if len(sys.argv) == 1: diff --git a/tests/Exscript/util/bufferTest.py b/tests/Exscript/util/bufferTest.py index 2e3aa96d..2b912e49 100644 --- a/tests/Exscript/util/bufferTest.py +++ b/tests/Exscript/util/bufferTest.py @@ -95,6 +95,7 @@ def testAddMonitor(self): def monitor_cb(thedata, *args, **kwargs): thedata['args'] = args thedata['kwargs'] = kwargs + data = {} b.add_monitor('abc', partial(monitor_cb, data)) @@ -127,5 +128,7 @@ def monitor_cb(thedata, *args, **kwargs): def suite(): return unittest.TestLoader().loadTestsFromTestCase(bufferTest) + + if __name__ == '__main__': unittest.TextTestRunner(verbosity=2).run(suite()) diff --git a/tests/Exscript/util/castTest.py b/tests/Exscript/util/castTest.py index ca2a82eb..f0ff261a 100644 --- a/tests/Exscript/util/castTest.py +++ b/tests/Exscript/util/castTest.py @@ -11,19 +11,22 @@ class castTest(unittest.TestCase): def testToList(self): from Exscript.util.cast import to_list - self.assertEqual(to_list(None), [None]) - self.assertEqual(to_list([]), []) - self.assertEqual(to_list('test'), ['test']) + + self.assertEqual(to_list(None), [None]) + self.assertEqual(to_list([]), []) + self.assertEqual(to_list('test'), ['test']) self.assertEqual(to_list(['test']), ['test']) def testToHost(self): from Exscript.util.cast import to_host - self.assertIsInstance(to_host('localhost'), Host) + + self.assertIsInstance(to_host('localhost'), Host) self.assertIsInstance(to_host(Host('localhost')), Host) self.assertRaises(TypeError, to_host, None) def testToHosts(self): from Exscript.util.cast import to_hosts + self.assertRaises(TypeError, to_hosts, None) result = to_hosts([]) @@ -49,12 +52,14 @@ def testToHosts(self): def testToRegex(self): from Exscript.util.cast import to_regex + self.assertTrue(hasattr(to_regex('regex'), 'match')) self.assertTrue(hasattr(to_regex(re.compile('regex')), 'match')) self.assertRaises(TypeError, to_regex, None) def testToRegexs(self): from Exscript.util.cast import to_regexs + self.assertRaises(TypeError, to_regexs, None) result = to_regexs([]) @@ -83,5 +88,7 @@ def testToRegexs(self): def suite(): return unittest.TestLoader().loadTestsFromTestCase(castTest) + + if __name__ == '__main__': unittest.TextTestRunner(verbosity=2).run(suite()) diff --git a/tests/Exscript/util/cryptTest.py b/tests/Exscript/util/cryptTest.py index d0dfeddd..e1823064 100644 --- a/tests/Exscript/util/cryptTest.py +++ b/tests/Exscript/util/cryptTest.py @@ -8,6 +8,7 @@ class cryptTest(unittest.TestCase): def testOtp(self): from Exscript.util.crypt import otp + hash = otp('password', 'abc123', 9999) self.assertEqual('ACTA AMMO CAR WEB BIN YAP', hash) hash = otp('password', 'abc123', 9998) @@ -20,5 +21,7 @@ def testOtp(self): def suite(): return unittest.TestLoader().loadTestsFromTestCase(cryptTest) + + if __name__ == '__main__': unittest.TextTestRunner(verbosity=2).run(suite()) diff --git a/tests/Exscript/util/decoratorTest.py b/tests/Exscript/util/decoratorTest.py index e80c0439..887008a3 100644 --- a/tests/Exscript/util/decoratorTest.py +++ b/tests/Exscript/util/decoratorTest.py @@ -10,7 +10,6 @@ class FakeConnection: - def __init__(self, os=None): self.os = os self.data = {} @@ -52,6 +51,7 @@ def bind_cb(self, job, bound_arg1, bound_arg2, **kwargs): def testBind(self): from Exscript.util.decorator import bind + bound = bind(self.bind_cb, 'one', 'two', three=3) result = bound(FakeJob()) self.assertEqual(result, 123) @@ -64,6 +64,7 @@ def junos_cb(self, job, *args): def testOsFunctionMapper(self): from Exscript.util.decorator import os_function_mapper + cb_map = {'ios': self.ios_cb, 'junos': self.junos_cb} mapper = os_function_mapper(cb_map) job = FakeJob() @@ -90,6 +91,7 @@ def autologin_cb(self, job, host, conn, *args, **kwargs): def testAutologin(self): from Exscript.util.decorator import autologin + job = FakeJob() host = job.data['host'] conn = job.data['conn'] = FakeConnection() @@ -107,6 +109,7 @@ def testAutologin(self): def fail(data, *args, **kwargs): data.value += 1 raise LoginFailure('intended login failure') + conn.login = partial(fail, data) # Test retry functionality. @@ -114,14 +117,7 @@ def fail(data, *args, **kwargs): bound = decor(self.autologin_cb) job = FakeJob() job.data['conn'] = conn - self.assertRaises(LoginFailure, - bound, - job, - host, - conn, - 'one', - 'two', - three=3) + self.assertRaises(LoginFailure, bound, job, host, conn, 'one', 'two', three=3) self.assertEqual(data.value, 5) def autoauthenticate_cb(self, job, host, conn, *args, **kwargs): @@ -131,6 +127,7 @@ def autoauthenticate_cb(self, job, host, conn, *args, **kwargs): def testAutoauthenticate(self): from Exscript.util.decorator import autoauthenticate + job = FakeJob() host = job.data['host'] conn = job.data['conn'] = FakeConnection() @@ -148,6 +145,7 @@ def testAutoauthenticate(self): def fail(data, *args, **kwargs): data.value += 1 raise LoginFailure('intended login failure') + conn.authenticate = partial(fail, data) # Test retry functionality. @@ -155,14 +153,7 @@ def fail(data, *args, **kwargs): bound = decor(self.autoauthenticate_cb) job = FakeJob() job.data['conn'] = conn - self.assertRaises(LoginFailure, - bound, - job, - host, - conn, - 'one', - 'two', - three=3) + self.assertRaises(LoginFailure, bound, job, host, conn, 'one', 'two', three=3) self.assertEqual(data.value, 5) def testDeprecated(self): @@ -171,5 +162,7 @@ def testDeprecated(self): def suite(): return unittest.TestLoader().loadTestsFromTestCase(decoratorTest) + + if __name__ == '__main__': unittest.TextTestRunner(verbosity=2).run(suite()) diff --git a/tests/Exscript/util/eventTest.py b/tests/Exscript/util/eventTest.py index caef5291..7742abdd 100644 --- a/tests/Exscript/util/eventTest.py +++ b/tests/Exscript/util/eventTest.py @@ -34,6 +34,7 @@ def testListen(self): def thefunction(): pass + ref = self.event.listen(thefunction) self.assertIsInstance(ref, WeakMethod) self.assertEqual(self.event.n_subscribers(), 1) @@ -60,19 +61,19 @@ def testIsConnected(self): def testEmit(self): self.event.connect(self.callback) - self.assertEqual(self.args, None) + self.assertEqual(self.args, None) self.assertEqual(self.kwargs, None) self.event.emit() - self.assertEqual(self.args, ()) + self.assertEqual(self.args, ()) self.assertEqual(self.kwargs, {}) self.event.emit('test') - self.assertEqual(self.args, ('test',)) + self.assertEqual(self.args, ('test',)) self.assertEqual(self.kwargs, {}) self.event.emit('test', foo='bar') - self.assertEqual(self.args, ('test',)) + self.assertEqual(self.args, ('test',)) self.assertEqual(self.kwargs, {'foo': 'bar'}) self.event.disconnect(self.callback) @@ -81,15 +82,15 @@ def testEmit(self): self.kwargs = None self.event.emit() - self.assertEqual(self.args, ()) + self.assertEqual(self.args, ()) self.assertEqual(self.kwargs, {}) self.event.emit('test') - self.assertEqual(self.args, ('test',)) + self.assertEqual(self.args, ('test',)) self.assertEqual(self.kwargs, {}) self.event.emit('test', foo='bar') - self.assertEqual(self.args, ('test',)) + self.assertEqual(self.args, ('test',)) self.assertEqual(self.kwargs, {'foo': 'bar'}) self.event.disconnect(self.callback) @@ -114,5 +115,7 @@ def testDisconnectAll(self): def suite(): return unittest.TestLoader().loadTestsFromTestCase(eventTest) + + if __name__ == '__main__': unittest.TextTestRunner(verbosity=2).run(suite()) diff --git a/tests/Exscript/util/fileTest.py b/tests/Exscript/util/fileTest.py index 5a5828da..ee00c25b 100644 --- a/tests/Exscript/util/fileTest.py +++ b/tests/Exscript/util/fileTest.py @@ -5,13 +5,15 @@ import Exscript.util.file from tempfile import NamedTemporaryFile -account_pool = [('user1', 'password1'), - ('user2', 'password2'), - ('user3', 'password3'), - ('user4', 'password4')] +account_pool = [ + ('user1', 'password1'), + ('user2', 'password2'), + ('user3', 'password3'), + ('user4', 'password4'), +] hosts = ['localhost', '1.2.3.4', 'ssh://test', 'telnet://another:23'] -expected_hosts = ['localhost', '1.2.3.4', 'test', 'another'] +expected_hosts = ['localhost', '1.2.3.4', 'test', 'another'] class fileTest(unittest.TestCase): @@ -48,6 +50,7 @@ def tearDown(self): def testGetAccountsFromFile(self): from Exscript.util.file import get_accounts_from_file + accounts = get_accounts_from_file(self.account_file.name) result = [(a.get_name(), a.get_password()) for a in accounts] result.sort() @@ -55,11 +58,13 @@ def testGetAccountsFromFile(self): def testGetHostsFromFile(self): from Exscript.util.file import get_hosts_from_file + result = get_hosts_from_file(self.host_file.name) self.assertEqual([h.get_name() for h in result], expected_hosts) def testGetHostsFromCsv(self): from Exscript.util.file import get_hosts_from_csv + result = get_hosts_from_csv(self.csv_host_file.name) hostnames = [h.get_name() for h in result] testvars = [h.get('test')[0] for h in result] @@ -68,6 +73,7 @@ def testGetHostsFromCsv(self): def testLoadLib(self): from Exscript.util.file import load_lib + functions = load_lib(self.lib_file.name) name = os.path.splitext(os.path.basename(self.lib_file.name))[0] self.assertEqual({name + '.test': object}, functions) @@ -75,5 +81,7 @@ def testLoadLib(self): def suite(): return unittest.TestLoader().loadTestsFromTestCase(fileTest) + + if __name__ == '__main__': unittest.TextTestRunner(verbosity=2).run(suite()) diff --git a/tests/Exscript/util/interactTest.py b/tests/Exscript/util/interactTest.py index bf583ac2..6cadb279 100644 --- a/tests/Exscript/util/interactTest.py +++ b/tests/Exscript/util/interactTest.py @@ -72,5 +72,7 @@ def suite(): thesuite.addTest(loader.loadTestsFromTestCase(InputHistoryTest)) thesuite.addTest(loader.loadTestsFromTestCase(interactTest)) return thesuite + + if __name__ == '__main__': unittest.TextTestRunner(verbosity=2).run(suite()) diff --git a/tests/Exscript/util/ipTest.py b/tests/Exscript/util/ipTest.py index 5e9c7985..9110ded7 100644 --- a/tests/Exscript/util/ipTest.py +++ b/tests/Exscript/util/ipTest.py @@ -8,23 +8,27 @@ class ipTest(unittest.TestCase): def testIsIp(self): from Exscript.util.ip import is_ip + self.assertTrue(is_ip('0.0.0.0')) self.assertTrue(is_ip('::')) self.assertTrue(not is_ip('1')) def testNormalizeIp(self): from Exscript.util.ip import normalize_ip + self.assertEqual(normalize_ip('0.128.255.0'), '000.128.255.000') - self.assertEqual(normalize_ip('1234:0:01:02::'), - '1234:0000:0001:0002:0000:0000:0000:0000') + self.assertEqual(normalize_ip('1234:0:01:02::'), '1234:0000:0001:0002:0000:0000:0000:0000') def testCleanIp(self): from Exscript.util.ip import clean_ip + self.assertEqual(clean_ip('192.168.010.001'), '192.168.10.1') self.assertEqual(clean_ip('1234:0:0:0:0:0:0:000A'), '1234::a') def suite(): return unittest.TestLoader().loadTestsFromTestCase(ipTest) + + if __name__ == '__main__': unittest.TextTestRunner(verbosity=2).run(suite()) diff --git a/tests/Exscript/util/ipv4Test.py b/tests/Exscript/util/ipv4Test.py index 8c92aa9e..0c5d4d7e 100644 --- a/tests/Exscript/util/ipv4Test.py +++ b/tests/Exscript/util/ipv4Test.py @@ -9,6 +9,7 @@ class ipv4Test(unittest.TestCase): def testIsIp(self): from Exscript.util.ipv4 import is_ip + self.assertTrue(is_ip('0.0.0.0')) self.assertTrue(is_ip('255.255.255.255')) self.assertTrue(is_ip('1.2.3.4')) @@ -21,73 +22,78 @@ def testIsIp(self): def testNormalizeIp(self): from Exscript.util.ipv4 import normalize_ip - self.assertEqual(normalize_ip('0.0.0.0'), '000.000.000.000') + + self.assertEqual(normalize_ip('0.0.0.0'), '000.000.000.000') self.assertEqual(normalize_ip('255.255.255.255'), '255.255.255.255') self.assertEqual(normalize_ip('001.002.003.004'), '001.002.003.004') self.assertEqual(normalize_ip('192.168.010.001'), '192.168.010.001') - self.assertEqual(normalize_ip('0.128.255.0'), '000.128.255.000') + self.assertEqual(normalize_ip('0.128.255.0'), '000.128.255.000') def testCleanIp(self): from Exscript.util.ipv4 import clean_ip - self.assertEqual(clean_ip('0.0.0.0'), '0.0.0.0') + + self.assertEqual(clean_ip('0.0.0.0'), '0.0.0.0') self.assertEqual(clean_ip('255.255.255.255'), '255.255.255.255') self.assertEqual(clean_ip('001.002.003.004'), '1.2.3.4') self.assertEqual(clean_ip('192.168.010.001'), '192.168.10.1') - self.assertEqual(clean_ip('0.128.255.0'), '0.128.255.0') + self.assertEqual(clean_ip('0.128.255.0'), '0.128.255.0') def testIp2Int(self): from Exscript.util.ipv4 import ip2int - self.assertEqual(ip2int('0.0.0.0'), 0x00000000) + + self.assertEqual(ip2int('0.0.0.0'), 0x00000000) self.assertEqual(ip2int('255.255.255.255'), 0xFFFFFFFF) - self.assertEqual(ip2int('255.255.255.0'), 0xFFFFFF00) - self.assertEqual(ip2int('0.255.255.0'), 0x00FFFF00) - self.assertEqual(ip2int('0.128.255.0'), 0x0080FF00) + self.assertEqual(ip2int('255.255.255.0'), 0xFFFFFF00) + self.assertEqual(ip2int('0.255.255.0'), 0x00FFFF00) + self.assertEqual(ip2int('0.128.255.0'), 0x0080FF00) def testInt2Ip(self): from Exscript.util.ipv4 import int2ip, ip2int - for ip in ('0.0.0.0', - '255.255.255.255', - '255.255.255.0', - '0.255.255.0', - '0.128.255.0'): + + for ip in ('0.0.0.0', '255.255.255.255', '255.255.255.0', '0.255.255.0', '0.128.255.0'): self.assertEqual(int2ip(ip2int(ip)), ip) def testPfxlen2MaskInt(self): from Exscript.util.ipv4 import int2ip, pfxlen2mask_int + self.assertEqual(int2ip(pfxlen2mask_int(32)), '255.255.255.255') self.assertEqual(int2ip(pfxlen2mask_int(31)), '255.255.255.254') self.assertEqual(int2ip(pfxlen2mask_int(30)), '255.255.255.252') - self.assertEqual(int2ip(pfxlen2mask_int(2)), '192.0.0.0') - self.assertEqual(int2ip(pfxlen2mask_int(1)), '128.0.0.0') - self.assertEqual(int2ip(pfxlen2mask_int(0)), '0.0.0.0') + self.assertEqual(int2ip(pfxlen2mask_int(2)), '192.0.0.0') + self.assertEqual(int2ip(pfxlen2mask_int(1)), '128.0.0.0') + self.assertEqual(int2ip(pfxlen2mask_int(0)), '0.0.0.0') def testPfxlen2Mask(self): from Exscript.util.ipv4 import pfxlen2mask + self.assertEqual(pfxlen2mask(32), '255.255.255.255') self.assertEqual(pfxlen2mask(31), '255.255.255.254') self.assertEqual(pfxlen2mask(30), '255.255.255.252') - self.assertEqual(pfxlen2mask(2), '192.0.0.0') - self.assertEqual(pfxlen2mask(1), '128.0.0.0') - self.assertEqual(pfxlen2mask(0), '0.0.0.0') + self.assertEqual(pfxlen2mask(2), '192.0.0.0') + self.assertEqual(pfxlen2mask(1), '128.0.0.0') + self.assertEqual(pfxlen2mask(0), '0.0.0.0') def testMask2Pfxlen(self): from Exscript.util.ipv4 import mask2pfxlen + self.assertEqual(32, mask2pfxlen('255.255.255.255')) self.assertEqual(31, mask2pfxlen('255.255.255.254')) self.assertEqual(30, mask2pfxlen('255.255.255.252')) - self.assertEqual(2, mask2pfxlen('192.0.0.0')) - self.assertEqual(1, mask2pfxlen('128.0.0.0')) - self.assertEqual(0, mask2pfxlen('0.0.0.0')) + self.assertEqual(2, mask2pfxlen('192.0.0.0')) + self.assertEqual(1, mask2pfxlen('128.0.0.0')) + self.assertEqual(0, mask2pfxlen('0.0.0.0')) def testParsePrefix(self): from Exscript.util.ipv4 import parse_prefix + self.assertEqual(('1.2.3.4', 24), parse_prefix('1.2.3.4')) - self.assertEqual(('1.2.3.4', 32), parse_prefix('1.2.3.4', 32)) + self.assertEqual(('1.2.3.4', 32), parse_prefix('1.2.3.4', 32)) self.assertEqual(('1.2.3.4', 15), parse_prefix('1.2.3.4/15')) self.assertEqual(('1.2.3.4', 15), parse_prefix('1.2.3.4/15', 32)) def testNetwork(self): from Exscript.util.ipv4 import network + self.assertEqual(network('10.0.0.0/30'), '10.0.0.0') self.assertEqual(network('10.0.0.1/30'), '10.0.0.0') self.assertEqual(network('10.0.0.2/30'), '10.0.0.0') @@ -97,6 +103,7 @@ def testNetwork(self): def testBroadcast(self): from Exscript.util.ipv4 import broadcast + self.assertEqual(broadcast('10.0.0.0/30'), '10.0.0.3') self.assertEqual(broadcast('10.0.0.1/30'), '10.0.0.3') self.assertEqual(broadcast('10.0.0.2/30'), '10.0.0.3') @@ -106,6 +113,7 @@ def testBroadcast(self): def testRemoteIp(self): from Exscript.util.ipv4 import remote_ip + self.assertEqual(remote_ip('10.0.0.0'), '10.0.0.3') self.assertEqual(remote_ip('10.0.0.1'), '10.0.0.2') self.assertEqual(remote_ip('10.0.0.2'), '10.0.0.1') @@ -113,6 +121,7 @@ def testRemoteIp(self): def testMatchesPrefix(self): from Exscript.util.ipv4 import matches_prefix + self.assertTrue(matches_prefix('10.0.0.0', '10.0.0.0/8')) self.assertTrue(matches_prefix('10.255.255.255', '10.0.0.0/8')) self.assertFalse(matches_prefix('11.0.0.0', '10.0.0.0/8')) @@ -120,6 +129,7 @@ def testMatchesPrefix(self): def testIsPrivate(self): from Exscript.util.ipv4 import is_private + self.assertFalse(is_private('9.255.255.255')) self.assertTrue(is_private('10.0.0.0')) self.assertTrue(is_private('10.255.255.255')) @@ -139,12 +149,15 @@ def testSort(self): import random from Exscript.util.ipv4 import sort - ip_list = ['0.0.0.0', - '0.0.0.255', - '1.2.3.4', - '255.255.0.255', - '255.255.255.255', - '255.255.255.255'] + + ip_list = [ + '0.0.0.0', + '0.0.0.255', + '1.2.3.4', + '255.255.0.255', + '255.255.255.255', + '255.255.255.255', + ] ip_list_copy = ip_list[:] for i in range(50): random.shuffle(ip_list_copy) @@ -153,5 +166,7 @@ def testSort(self): def suite(): return unittest.TestLoader().loadTestsFromTestCase(ipv4Test) + + if __name__ == '__main__': unittest.TextTestRunner(verbosity=2).run(suite()) diff --git a/tests/Exscript/util/ipv6Test.py b/tests/Exscript/util/ipv6Test.py index 2c68f253..617edb68 100644 --- a/tests/Exscript/util/ipv6Test.py +++ b/tests/Exscript/util/ipv6Test.py @@ -8,6 +8,7 @@ class ipv6Test(unittest.TestCase): def testIsIp(self): from Exscript.util.ipv6 import is_ip + self.assertTrue(is_ip('::')) self.assertTrue(is_ip('1::')) self.assertTrue(is_ip('::A')) @@ -25,16 +26,12 @@ def testIsIp(self): def testNormalizeIp(self): from Exscript.util.ipv6 import normalize_ip - self.assertEqual(normalize_ip('::'), - '0000:0000:0000:0000:0000:0000:0000:0000') - self.assertEqual(normalize_ip('1::'), - '0001:0000:0000:0000:0000:0000:0000:0000') - self.assertEqual(normalize_ip('::A'), - '0000:0000:0000:0000:0000:0000:0000:000a') - self.assertEqual(normalize_ip('1234::2222'), - '1234:0000:0000:0000:0000:0000:0000:2222') - self.assertEqual(normalize_ip('1234:0:01:02::'), - '1234:0000:0001:0002:0000:0000:0000:0000') + + self.assertEqual(normalize_ip('::'), '0000:0000:0000:0000:0000:0000:0000:0000') + self.assertEqual(normalize_ip('1::'), '0001:0000:0000:0000:0000:0000:0000:0000') + self.assertEqual(normalize_ip('::A'), '0000:0000:0000:0000:0000:0000:0000:000a') + self.assertEqual(normalize_ip('1234::2222'), '1234:0000:0000:0000:0000:0000:0000:2222') + self.assertEqual(normalize_ip('1234:0:01:02::'), '1234:0000:0001:0002:0000:0000:0000:0000') def testCleanIp(self): from Exscript.util.ipv6 import clean_ip @@ -50,6 +47,7 @@ def testCleanIp(self): def testParsePrefix(self): from Exscript.util.ipv6 import parse_prefix + self.assertEqual(('A::A', 24), parse_prefix('A::A/24', 22)) self.assertEqual(('1:0:1:2::', 128), parse_prefix('1:0:1:2::')) self.assertEqual(('1:0:1:2::', 64), parse_prefix('1:0:1:2::', 64)) @@ -57,5 +55,7 @@ def testParsePrefix(self): def suite(): return unittest.TestLoader().loadTestsFromTestCase(ipv6Test) + + if __name__ == '__main__': unittest.TextTestRunner(verbosity=2).run(suite()) diff --git a/tests/Exscript/util/mailTest.py b/tests/Exscript/util/mailTest.py index 890483f5..e75af12f 100644 --- a/tests/Exscript/util/mailTest.py +++ b/tests/Exscript/util/mailTest.py @@ -105,8 +105,7 @@ def testGetRecipients(self): self.mail.set_to('test1') self.mail.set_cc('test2') self.mail.set_bcc('test3') - self.assertEqual(self.mail.get_recipients(), - ['test1', 'test2', 'test3']) + self.assertEqual(self.mail.get_recipients(), ['test1', 'test2', 'test3']) def testSetSubject(self): self.assertEqual(self.mail.get_subject(), '') @@ -158,11 +157,13 @@ def checkResult(self, mail): def testFromTemplateString(self): from Exscript.util.mail import from_template_string + mail = from_template_string(template, **vars) self.checkResult(mail) def testFromTemplate(self): from Exscript.util.mail import from_template + tmpfile = tempfile.NamedTemporaryFile() tmpfile.write(template.encode('utf8')) tmpfile.flush() @@ -178,5 +179,7 @@ def suite(): mail_cls = unittest.TestLoader().loadTestsFromTestCase(MailTest) mail_module = unittest.TestLoader().loadTestsFromTestCase(mailTest) return unittest.TestSuite([mail_cls, mail_module]) + + if __name__ == '__main__': unittest.TextTestRunner(verbosity=2).run(suite()) diff --git a/tests/Exscript/util/matchTest.py b/tests/Exscript/util/matchTest.py index a9a17fee..231c2982 100644 --- a/tests/Exscript/util/matchTest.py +++ b/tests/Exscript/util/matchTest.py @@ -16,8 +16,7 @@ def testFirstMatch(self): self.assertEqual(first_match(string, r'(\S+)'), 'my') self.assertEqual(first_match(string, r'(aaa) (\S+)'), (None, None)) self.assertEqual(first_match(string, r'(\S+) (\S+)'), ('my', 'test')) - self.assertEqual( - first_match("24.1632", r'(\d+)\.(\d+)'), ('24', '1632')) + self.assertEqual(first_match("24.1632", r'(\d+)\.(\d+)'), ('24', '1632')) self.assertEqual(first_match("24", r'(\d+)\.?(\d+)?'), ('24', None)) multi_line = 'hello\nworld\nhello world' @@ -38,5 +37,7 @@ def testAnyMatch(self): def suite(): return unittest.TestLoader().loadTestsFromTestCase(matchTest) + + if __name__ == '__main__': unittest.TextTestRunner(verbosity=2).run(suite()) diff --git a/tests/Exscript/util/reportTest.py b/tests/Exscript/util/reportTest.py index 53df752c..770c939a 100644 --- a/tests/Exscript/util/reportTest.py +++ b/tests/Exscript/util/reportTest.py @@ -15,14 +15,11 @@ class FakeQueue: class FakeJob: - def __init__(self, name='fake'): self.function = lambda x: None self.name = name self.failures = 0 - self.data = {'pipe': 0, - 'stdout': sys.stdout, - 'host': Host('foo')} + self.data = {'pipe': 0, 'stdout': sys.stdout, 'host': Host('foo')} class FakeError(Exception): @@ -62,6 +59,7 @@ def createSucceededLog(self): def testStatus(self): from Exscript.util.report import status + self.createSucceededLog() expect = 'One action done (succeeded)' self.assertEqual(status(self.logger), expect) @@ -76,6 +74,7 @@ def testStatus(self): def testSummarize(self): from Exscript.util.report import summarize + self.createSucceededLog() self.createAbortedLog() expected = 'fake1: ok\nfake2: FakeError' @@ -83,11 +82,13 @@ def testSummarize(self): def testFormat(self): from Exscript.util.report import format + self.createSucceededLog() self.createAbortedLog() self.createSucceededLog() file = os.path.splitext(__file__)[0] - expected = ''' + expected = ( + ''' Failed actions: --------------- fake2: @@ -100,13 +101,16 @@ def testFormat(self): Successful actions: ------------------- fake1 -fake3'''.strip() % (file, self.createAbortedLog.__code__.co_firstlineno + 3) - expected = expected.replace('FakeError\n', - 'util.reportTest.FakeError\n') +fake3'''.strip() + % (file, self.createAbortedLog.__code__.co_firstlineno + 3) + ) + expected = expected.replace('FakeError\n', 'util.reportTest.FakeError\n') self.assertEqual(format(self.logger), expected) def suite(): return unittest.TestLoader().loadTestsFromTestCase(reportTest) + + if __name__ == '__main__': unittest.TextTestRunner(verbosity=2).run(suite()) diff --git a/tests/Exscript/util/run_suite.py b/tests/Exscript/util/run_suite.py index 0520fafd..49b76509 100755 --- a/tests/Exscript/util/run_suite.py +++ b/tests/Exscript/util/run_suite.py @@ -13,9 +13,11 @@ logger = logging.getLogger('TestSuite') logger.addHandler(logging.NullHandler()) + def uppercase(match): return match.group(1).upper() + correlated = dict() @@ -84,7 +86,9 @@ def decorator(func): def wrapper(*args, **kwargs): logger.debug(msg) return func(*args, **kwargs) + return wrapper + return decorator @@ -105,7 +109,7 @@ def correlate_class(theclass): continue # Format the function names. - testname = re.sub(r'_(\w)', uppercase, name) + testname = re.sub(r'_(\w)', uppercase, name) testname = re.sub(r'(\d\w)', uppercase, testname) testname = 'test' + re.sub(r'^(\w)', uppercase, testname) testmethod = theclass.__name__ + '.' + testname @@ -115,8 +119,7 @@ def correlate_class(theclass): # Throw an error if the function does not have a test. if testname in dir(theclass): continue - if ismodule(theclass.CORRELATE) and \ - value.__module__ != theclass.CORRELATE.__name__: + if ismodule(theclass.CORRELATE) and value.__module__ != theclass.CORRELATE.__name__: continue # function was imported. if both in correlated: continue @@ -170,6 +173,7 @@ def suite(): def recursive_suite(): return load_suite(find('.', '*Test.py')) + if __name__ == '__main__': # Parse CLI options. if len(sys.argv) == 1: diff --git a/tests/Exscript/util/startTest.py b/tests/Exscript/util/startTest.py index 327bed04..38069ddd 100644 --- a/tests/Exscript/util/startTest.py +++ b/tests/Exscript/util/startTest.py @@ -38,15 +38,12 @@ def doTest(self, function): self.assertEqual(self.data.value, 3) # Run on multiple hosts with multiple threads. - function(self.account, - hosts, - self.callback, - max_threads=2, - verbose=0) + function(self.account, hosts, self.callback, max_threads=2, verbose=0) self.assertEqual(self.data.value, 5) def testRun(self): from Exscript.util.start import run + self.doTest(run) def testQuickrun(self): @@ -54,6 +51,7 @@ def testQuickrun(self): def testStart(self): from Exscript.util.start import start + self.doTest(start) def testQuickstart(self): @@ -62,5 +60,7 @@ def testQuickstart(self): def suite(): return unittest.TestLoader().loadTestsFromTestCase(startTest) + + if __name__ == '__main__': unittest.TextTestRunner(verbosity=2).run(suite()) diff --git a/tests/Exscript/util/syslogTest.py b/tests/Exscript/util/syslogTest.py index 77e5e2b9..00e8c25a 100644 --- a/tests/Exscript/util/syslogTest.py +++ b/tests/Exscript/util/syslogTest.py @@ -14,5 +14,7 @@ def testNetlog(self): def suite(): return unittest.TestLoader().loadTestsFromTestCase(castTest) + + if __name__ == '__main__': unittest.TextTestRunner(verbosity=2).run(suite()) diff --git a/tests/Exscript/util/ttyTest.py b/tests/Exscript/util/ttyTest.py index a0e0fe10..fc5a0fed 100644 --- a/tests/Exscript/util/ttyTest.py +++ b/tests/Exscript/util/ttyTest.py @@ -60,7 +60,7 @@ def testGetTerminalSize(self): # (1111, 1111) if the LINES and COLUMNS env variables are set, # so there is no way to tests whether stty was used or the env # vars. - #self.assertNotEqual(get_terminal_size(), (1111, 1111)) + # self.assertNotEqual(get_terminal_size(), (1111, 1111)) self.assertNotEqual(get_terminal_size(10, 10), (25, 80)) except OSError: pass # "stty" not found. @@ -69,13 +69,15 @@ def testGetTerminalSize(self): os.environ['PATH'] = '' self._unredirect() # Same travis problem as above... - #self.assertNotEqual(get_terminal_size(), (1111, 1111)) - #self.assertNotEqual(get_terminal_size(10, 10), (1111, 1111)) + # self.assertNotEqual(get_terminal_size(), (1111, 1111)) + # self.assertNotEqual(get_terminal_size(10, 10), (1111, 1111)) self.assertNotEqual(get_terminal_size(10, 10), (25, 80)) os.environ['PATH'] = oldpath def suite(): return unittest.TestLoader().loadTestsFromTestCase(ttyTest) + + if __name__ == '__main__': unittest.TextTestRunner(verbosity=2).run(suite()) diff --git a/tests/Exscript/util/urlTest.py b/tests/Exscript/util/urlTest.py index aaeb85ae..bee4c3c8 100644 --- a/tests/Exscript/util/urlTest.py +++ b/tests/Exscript/util/urlTest.py @@ -5,92 +5,61 @@ urls = [ # No protocol. - ('testhost', - 'ssh://testhost:22'), - ('testhost?myvar=testvalue', - 'ssh://testhost:22?myvar=testvalue'), - + ('testhost', 'ssh://testhost:22'), + ('testhost?myvar=testvalue', 'ssh://testhost:22?myvar=testvalue'), # No protocol + empty user. - ('@testhost', - 'ssh://@testhost:22'), - ('@testhost?myvar=testvalue', - 'ssh://@testhost:22?myvar=testvalue'), - + ('@testhost', 'ssh://@testhost:22'), + ('@testhost?myvar=testvalue', 'ssh://@testhost:22?myvar=testvalue'), # No protocol + user. - ('user@testhost', - 'ssh://user@testhost:22'), - ('user:password@testhost', - 'ssh://user:password@testhost:22'), - ('user:password:password2@testhost', - 'ssh://user:password:password2@testhost:22'), - + ('user@testhost', 'ssh://user@testhost:22'), + ('user:password@testhost', 'ssh://user:password@testhost:22'), + ('user:password:password2@testhost', 'ssh://user:password:password2@testhost:22'), # No protocol + empty password 1. - ('user:@testhost', - 'ssh://user:@testhost:22'), - ('user::password2@testhost', - 'ssh://user::password2@testhost:22'), - (':@testhost', - 'ssh://:@testhost:22'), - + ('user:@testhost', 'ssh://user:@testhost:22'), + ('user::password2@testhost', 'ssh://user::password2@testhost:22'), + (':@testhost', 'ssh://:@testhost:22'), # No protocol + empty password 2. - ('user:password:@testhost', - 'ssh://user:password:@testhost:22'), - ('user::@testhost', - 'ssh://user::@testhost:22'), - ('::@testhost', - 'ssh://::@testhost:22'), - (':password:@testhost', - 'ssh://:password:@testhost:22'), - + ('user:password:@testhost', 'ssh://user:password:@testhost:22'), + ('user::@testhost', 'ssh://user::@testhost:22'), + ('::@testhost', 'ssh://::@testhost:22'), + (':password:@testhost', 'ssh://:password:@testhost:22'), # Protocol. - ('telnet://testhost', - 'telnet://testhost:23'), - ('telnet://testhost?myvar=testvalue', - 'telnet://testhost:23?myvar=testvalue'), - + ('telnet://testhost', 'telnet://testhost:23'), + ('telnet://testhost?myvar=testvalue', 'telnet://testhost:23?myvar=testvalue'), # Protocol + empty user. - ('ssh://@testhost', - 'ssh://@testhost:22'), - ('ssh://:password@testhost', - 'ssh://:password@testhost:22'), - ('ssh://:password:password2@testhost', - 'ssh://:password:password2@testhost:22'), - + ('ssh://@testhost', 'ssh://@testhost:22'), + ('ssh://:password@testhost', 'ssh://:password@testhost:22'), + ('ssh://:password:password2@testhost', 'ssh://:password:password2@testhost:22'), # Protocol + user. - ('ssh://user@testhost', - 'ssh://user@testhost:22'), - ('ssh://user@testhost?myvar=testvalue', - 'ssh://user@testhost:22?myvar=testvalue'), - ('ssh://user:password@testhost', - 'ssh://user:password@testhost:22'), - ('ssh://user:password@testhost?myvar=testvalue', - 'ssh://user:password@testhost:22?myvar=testvalue'), - ('ssh://user:password@testhost', - 'ssh://user:password@testhost:22'), - ('ssh://user:password:password2@testhost', - 'ssh://user:password:password2@testhost:22'), - + ('ssh://user@testhost', 'ssh://user@testhost:22'), + ('ssh://user@testhost?myvar=testvalue', 'ssh://user@testhost:22?myvar=testvalue'), + ('ssh://user:password@testhost', 'ssh://user:password@testhost:22'), + ( + 'ssh://user:password@testhost?myvar=testvalue', + 'ssh://user:password@testhost:22?myvar=testvalue', + ), + ('ssh://user:password@testhost', 'ssh://user:password@testhost:22'), + ('ssh://user:password:password2@testhost', 'ssh://user:password:password2@testhost:22'), # Multiple arguments. - ('ssh://user:password@testhost?myvar=testvalue&myvar2=test%202', - 'ssh://user:password@testhost:22?myvar=testvalue&myvar2=test+2'), - ('ssh://user:password@testhost?myvar=testvalue&myvar2=test%202', - 'ssh://user:password@testhost:22?myvar=testvalue&myvar2=test+2'), - + ( + 'ssh://user:password@testhost?myvar=testvalue&myvar2=test%202', + 'ssh://user:password@testhost:22?myvar=testvalue&myvar2=test+2', + ), + ( + 'ssh://user:password@testhost?myvar=testvalue&myvar2=test%202', + 'ssh://user:password@testhost:22?myvar=testvalue&myvar2=test+2', + ), # Encoding. - ('foo://%27M%7B7Zk:%27%2FM%7B7Zyk:C7%26Rt%3Ea@ULM-SZRC1:22', - 'foo://%27M%7B7Zk:%27%2FM%7B7Zyk:C7%26Rt%3Ea@ULM-SZRC1:22'), - + ( + 'foo://%27M%7B7Zk:%27%2FM%7B7Zyk:C7%26Rt%3Ea@ULM-SZRC1:22', + 'foo://%27M%7B7Zk:%27%2FM%7B7Zyk:C7%26Rt%3Ea@ULM-SZRC1:22', + ), # Pseudo protocol. - ('pseudo://../my/path', - 'pseudo://../my/path'), - ('pseudo://../path', - 'pseudo://../path'), - ('pseudo://filename', - 'pseudo://filename'), - ('pseudo:///abspath', - 'pseudo:///abspath'), - ('pseudo:///abs/path', - 'pseudo:///abs/path'), + ('pseudo://../my/path', 'pseudo://../my/path'), + ('pseudo://../path', 'pseudo://../path'), + ('pseudo://filename', 'pseudo://filename'), + ('pseudo:///abspath', 'pseudo:///abspath'), + ('pseudo:///abs/path', 'pseudo:///abs/path'), ] @@ -121,5 +90,7 @@ def testFromString(self): def suite(): return unittest.TestLoader().loadTestsFromTestCase(urlTest) + + if __name__ == '__main__': unittest.TextTestRunner(verbosity=2).run(suite()) diff --git a/tests/Exscript/util/weakmethodTest.py b/tests/Exscript/util/weakmethodTest.py index 9b454116..0a802612 100644 --- a/tests/Exscript/util/weakmethodTest.py +++ b/tests/Exscript/util/weakmethodTest.py @@ -5,7 +5,6 @@ class TestClass: - def callback(self, *args, **kwargs): self.args = args self.kwargs = kwargs @@ -52,6 +51,7 @@ def testCall(self): def function(data, *args, **kwargs): data['args'] = args data['kwargs'] = kwargs + d = {} f = ref(function) f(d, 'one', two=True) @@ -72,5 +72,7 @@ def function(data, *args, **kwargs): def suite(): return unittest.TestLoader().loadTestsFromTestCase(weakmethodTest) + + if __name__ == '__main__': unittest.TextTestRunner(verbosity=2).run(suite()) diff --git a/tests/Exscript/workqueue/JobTest.py b/tests/Exscript/workqueue/JobTest.py index beae57d4..4ee41c39 100644 --- a/tests/Exscript/workqueue/JobTest.py +++ b/tests/Exscript/workqueue/JobTest.py @@ -32,14 +32,15 @@ def testStart(self): pass # See testRun() -@unittest.skipIf(get_all_start_methods()[0] != 'fork', - 'Process jobs need the "fork" start method (the default only on Linux < 3.14)') +@unittest.skipIf( + get_all_start_methods()[0] != 'fork', + 'Process jobs need the "fork" start method (the default only on Linux < 3.14)', +) class ProcessTest(ThreadTest): CORRELATE = Process class JobTest(unittest.TestCase): - def testConstructor(self): job = Job(do_nothing, 'myaction', 1, 'foo') self.assertEqual(job.name, 'myaction') @@ -61,5 +62,7 @@ def suite(): suite2 = loader.loadTestsFromTestCase(ProcessTest) suite3 = loader.loadTestsFromTestCase(JobTest) return unittest.TestSuite((suite1, suite2, suite3)) + + if __name__ == '__main__': unittest.TextTestRunner(verbosity=2).run(suite()) diff --git a/tests/Exscript/workqueue/MainLoopTest.py b/tests/Exscript/workqueue/MainLoopTest.py index 35077462..bc2ab0fa 100644 --- a/tests/Exscript/workqueue/MainLoopTest.py +++ b/tests/Exscript/workqueue/MainLoopTest.py @@ -27,5 +27,7 @@ def testMainLoop(self): def suite(): return unittest.TestLoader().loadTestsFromTestCase(MainLoopTest) + + if __name__ == '__main__': unittest.TextTestRunner(verbosity=2).run(suite()) diff --git a/tests/Exscript/workqueue/PipelineTest.py b/tests/Exscript/workqueue/PipelineTest.py index 8149c319..6036aa25 100644 --- a/tests/Exscript/workqueue/PipelineTest.py +++ b/tests/Exscript/workqueue/PipelineTest.py @@ -123,7 +123,6 @@ def testStop(self): thread_completed = Value('i', 0) class deadlock_until_stop(Thread): - def run(inner_self): self.assertEqual(next(self.pipeline), item1) self.assertEqual(next(self.pipeline), None) # *** @@ -156,7 +155,6 @@ def testPause(self): self.pipeline.pause() class complete_all(Thread): - def run(inner_self): while True: task = next(self.pipeline) @@ -168,7 +166,7 @@ def run(inner_self): thread.daemon = True thread.start() - time.sleep(.2) # hack: wait long enough for the task to complete. + time.sleep(0.2) # hack: wait long enough for the task to complete. self.assertEqual(len(self.pipeline), 1) # should not be completed. self.pipeline.unpause() self.pipeline.wait_all() # now it should not deadlock. @@ -210,10 +208,10 @@ def testWaitForId(self): item = next(self.pipeline) class complete_item(Thread): - def run(inner_self): - time.sleep(.1) + time.sleep(0.1) self.pipeline.task_done(item) + thread = complete_item() thread.daemon = True thread.start() @@ -235,10 +233,10 @@ def testWait(self): item = next(self.pipeline) class complete_item(Thread): - def run(inner_self): - time.sleep(.1) + time.sleep(0.1) self.pipeline.task_done(item) + thread = complete_item() thread.daemon = True thread.start() @@ -253,13 +251,13 @@ def testWaitAll(self): self.pipeline.append(item2) class complete_all(Thread): - def run(inner_self): while True: task = next(self.pipeline) if task is None: break self.pipeline.task_done(task) + thread = complete_all() thread.daemon = True thread.start() @@ -373,5 +371,7 @@ def testNext(self): def suite(): return unittest.TestLoader().loadTestsFromTestCase(PipelineTest) + + if __name__ == '__main__': unittest.TextTestRunner(verbosity=2).run(suite()) diff --git a/tests/Exscript/workqueue/TaskTest.py b/tests/Exscript/workqueue/TaskTest.py index 0f13f288..5e2bc984 100644 --- a/tests/Exscript/workqueue/TaskTest.py +++ b/tests/Exscript/workqueue/TaskTest.py @@ -45,5 +45,7 @@ def testAddJobId(self): def suite(): return unittest.TestLoader().loadTestsFromTestCase(TaskTest) + + if __name__ == '__main__': unittest.TextTestRunner(verbosity=2).run(suite()) diff --git a/tests/Exscript/workqueue/WorkQueueTest.py b/tests/Exscript/workqueue/WorkQueueTest.py index 5c1b3630..f16e5b14 100644 --- a/tests/Exscript/workqueue/WorkQueueTest.py +++ b/tests/Exscript/workqueue/WorkQueueTest.py @@ -20,6 +20,7 @@ def burn_time(job): job.data.value += 1 time.sleep(random.random()) + nop = lambda x: None @@ -71,7 +72,7 @@ def testEnqueue(self): self.wq.wait_until_done() # Check whether each has run successfully. - self.assertEqual(0, self.wq.get_length()) + self.assertEqual(0, self.wq.get_length()) self.assertEqual(222, data.value) self.wq.shutdown(True) self.assertEqual(0, self.wq.get_length()) @@ -164,6 +165,7 @@ def testIsPaused(self): def testGetRunningJobs(self): def function(job): self.assertEqual(self.wq.get_running_jobs(), [job]) + self.assertEqual(self.wq.get_running_jobs(), []) self.wq.enqueue(function) self.wq.shutdown(True) @@ -175,5 +177,7 @@ def testGetLength(self): def suite(): return unittest.TestLoader().loadTestsFromTestCase(WorkQueueTest) + + if __name__ == '__main__': unittest.TextTestRunner(verbosity=2).run(suite()) diff --git a/tests/Exscript/workqueue/run_suite.py b/tests/Exscript/workqueue/run_suite.py index 0520fafd..49b76509 100755 --- a/tests/Exscript/workqueue/run_suite.py +++ b/tests/Exscript/workqueue/run_suite.py @@ -13,9 +13,11 @@ logger = logging.getLogger('TestSuite') logger.addHandler(logging.NullHandler()) + def uppercase(match): return match.group(1).upper() + correlated = dict() @@ -84,7 +86,9 @@ def decorator(func): def wrapper(*args, **kwargs): logger.debug(msg) return func(*args, **kwargs) + return wrapper + return decorator @@ -105,7 +109,7 @@ def correlate_class(theclass): continue # Format the function names. - testname = re.sub(r'_(\w)', uppercase, name) + testname = re.sub(r'_(\w)', uppercase, name) testname = re.sub(r'(\d\w)', uppercase, testname) testname = 'test' + re.sub(r'^(\w)', uppercase, testname) testmethod = theclass.__name__ + '.' + testname @@ -115,8 +119,7 @@ def correlate_class(theclass): # Throw an error if the function does not have a test. if testname in dir(theclass): continue - if ismodule(theclass.CORRELATE) and \ - value.__module__ != theclass.CORRELATE.__name__: + if ismodule(theclass.CORRELATE) and value.__module__ != theclass.CORRELATE.__name__: continue # function was imported. if both in correlated: continue @@ -170,6 +173,7 @@ def suite(): def recursive_suite(): return load_suite(find('.', '*Test.py')) + if __name__ == '__main__': # Parse CLI options. if len(sys.argv) == 1: diff --git a/tests/conftest.py b/tests/conftest.py index 87140bf5..7db03b56 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,4 +1,5 @@ """pytest configuration for the unittest-based suite.""" + import inspect import unittest @@ -10,7 +11,10 @@ def pytest_pycollect_makeitem(collector, name, obj): from it; without this hook pytest would run the base tests once more for every importing module. """ - if (inspect.isclass(obj) and issubclass(obj, unittest.TestCase) - and obj.__module__ != collector.module.__name__): + if ( + inspect.isclass(obj) + and issubclass(obj, unittest.TestCase) + and obj.__module__ != collector.module.__name__ + ): return [] return None diff --git a/tests/templates/builtinvars/pseudodev.py b/tests/templates/builtinvars/pseudodev.py index a3e1ec2c..1360a416 100644 --- a/tests/templates/builtinvars/pseudodev.py +++ b/tests/templates/builtinvars/pseudodev.py @@ -1,2 +1 @@ -commands = ( -) +commands = () diff --git a/tests/templates/escaping/pseudodev.py b/tests/templates/escaping/pseudodev.py index de850778..5d0a0a75 100644 --- a/tests/templates/escaping/pseudodev.py +++ b/tests/templates/escaping/pseudodev.py @@ -1,3 +1 @@ -commands = ( - (r'.*', ''), -) +commands = ((r'.*', ''),) diff --git a/tests/templates/netconf/pseudodev.py b/tests/templates/netconf/pseudodev.py index cf771d73..cd795526 100644 --- a/tests/templates/netconf/pseudodev.py +++ b/tests/templates/netconf/pseudodev.py @@ -1,15 +1,15 @@ # This test lacks real Netconf emulation but it's sufficient to test RPC parsing. commands = ( -( -""" + ( + """ abc """.strip(), -""" + """ abc """, -), + ), ) diff --git a/tests/templates/stdlib.connection/pseudodev.py b/tests/templates/stdlib.connection/pseudodev.py index a3e1ec2c..1360a416 100644 --- a/tests/templates/stdlib.connection/pseudodev.py +++ b/tests/templates/stdlib.connection/pseudodev.py @@ -1,2 +1 @@ -commands = ( -) +commands = () diff --git a/tests/templates/stdlib.crypt/pseudodev.py b/tests/templates/stdlib.crypt/pseudodev.py index a3e1ec2c..1360a416 100644 --- a/tests/templates/stdlib.crypt/pseudodev.py +++ b/tests/templates/stdlib.crypt/pseudodev.py @@ -1,2 +1 @@ -commands = ( -) +commands = () diff --git a/tests/templates/stdlib.list/pseudodev.py b/tests/templates/stdlib.list/pseudodev.py index a3e1ec2c..1360a416 100644 --- a/tests/templates/stdlib.list/pseudodev.py +++ b/tests/templates/stdlib.list/pseudodev.py @@ -1,2 +1 @@ -commands = ( -) +commands = () diff --git a/tests/templates/stdlib.string/pseudodev.py b/tests/templates/stdlib.string/pseudodev.py index a3e1ec2c..1360a416 100644 --- a/tests/templates/stdlib.string/pseudodev.py +++ b/tests/templates/stdlib.string/pseudodev.py @@ -1,2 +1 @@ -commands = ( -) +commands = () diff --git a/tests/templates/syntax/pseudodev.py b/tests/templates/syntax/pseudodev.py index ab309b3b..0df9b6bc 100644 --- a/tests/templates/syntax/pseudodev.py +++ b/tests/templates/syntax/pseudodev.py @@ -1,8 +1,10 @@ commands = ( - ('ls -1.*', """ + ( + 'ls -1.*', + """ hello testme -"""), - - ('ls -l .+', lambda x: '\n' + x) +""", + ), + ('ls -l .+', lambda x: '\n' + x), ) diff --git a/tests/templates/whitespace/pseudodev.py b/tests/templates/whitespace/pseudodev.py index a4e4ba46..38fa51ca 100644 --- a/tests/templates/whitespace/pseudodev.py +++ b/tests/templates/whitespace/pseudodev.py @@ -1,8 +1,10 @@ commands = ( - ('ls -1.*', """ + ( + 'ls -1.*', + """ hello testme -"""), - - ('ls -l .+', lambda x: x) +""", + ), + ('ls -l .+', lambda x: x), ) diff --git a/tools/bench_bulk.py b/tools/bench_bulk.py index 2f0fd0df..1d24f120 100755 --- a/tools/bench_bulk.py +++ b/tools/bench_bulk.py @@ -8,6 +8,7 @@ python tools/bench_bulk.py [--mib 4] [--runs 3] [--json results.json] """ + import argparse import json import socket @@ -28,7 +29,6 @@ class _SSHServerInterface(paramiko.ServerInterface): - def __init__(self, allowed): self.allowed = allowed @@ -126,8 +126,11 @@ def bench_reassembly(size): def bench_ios_login(host_key): - server = Server(host_key=host_key, version='SSH-2.0-Cisco-1.25', - allowed='publickey,keyboard-interactive,password') + server = Server( + host_key=host_key, + version='SSH-2.0-Cisco-1.25', + allowed='publickey,keyboard-interactive,password', + ) conn = SSH2(verify_fingerprint=False) # default connect rate limit, like real usage start = time.perf_counter() conn.connect('127.0.0.1', server.port) @@ -142,7 +145,9 @@ def bench_ios_login(host_key): def main(): parser = argparse.ArgumentParser(description=__doc__.split('\n\n')[0]) parser.add_argument('--mib', type=int, default=4, help='response size in MiB (default: 4)') - parser.add_argument('--runs', type=int, default=3, help='runs per measurement, median is reported') + parser.add_argument( + '--runs', type=int, default=3, help='runs per measurement, median is reported' + ) parser.add_argument('--json', help='also write the results to this file') args = parser.parse_args() size = args.mib * 1024 * 1024 @@ -152,7 +157,11 @@ def main(): for name, make_conn, server_kwargs in ( ('telnet_execute', lambda: Telnet(connect_rate_limit=0), {}), - ('ssh_execute', lambda: SSH2(connect_rate_limit=0, verify_fingerprint=False), {'host_key': host_key}), + ( + 'ssh_execute', + lambda: SSH2(connect_rate_limit=0, verify_fingerprint=False), + {'host_key': host_key}, + ), ): server = Server(payload, **server_kwargs) runs = [bench_execute(make_conn(), server, size) for _ in range(args.runs)] @@ -161,7 +170,8 @@ def main(): results[name + '_cpu_s'] = round(statistics.median(r[1] for r in runs), 3) results['netconf_reassembly_s'] = round( - statistics.median(bench_reassembly(size) for _ in range(args.runs)), 3) + statistics.median(bench_reassembly(size) for _ in range(args.runs)), 3 + ) elapsed, handshakes, driver = bench_ios_login(host_key) results['ios_login_s'] = round(elapsed, 2) results['ios_login_handshakes'] = handshakes From b16e2b6c3a9335da54ae64aaee61e828e6398f66 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Fri, 30 Oct 2026 18:25:17 +0100 Subject: [PATCH 008/250] Apply safe ruff fixes and add a lint baseline The rules with remaining findings are ignored: style rules everywhere, security rules only in the files that have findings, so new files are checked. .git-blame-ignore-revs lists the formatting commit. --- .git-blame-ignore-revs | 3 +++ pyproject.toml | 24 +++++++++++++++++++ scripts/exscript | 16 ++++++------- src/Exscript/account.py | 2 +- src/Exscript/emulators/command.py | 2 +- src/Exscript/logger.py | 4 ++-- src/Exscript/protocols/netconf.py | 4 ++-- src/Exscript/protocols/protocol.py | 4 ++-- src/Exscript/protocols/ssh2.py | 4 ++-- src/Exscript/protocols/telnet.py | 2 +- src/Exscript/protocols/telnetlib.py | 6 ++--- src/Exscript/queue.py | 2 +- src/Exscript/servers/httpd.py | 6 ++--- src/Exscript/servers/server.py | 2 +- src/Exscript/servers/sshd.py | 4 ++-- src/Exscript/servers/telnetd.py | 2 +- src/Exscript/stdlib/file.py | 3 ++- src/Exscript/stdlib/ipv4.py | 2 +- src/Exscript/stdlib/util.py | 2 +- src/Exscript/util/cast.py | 2 +- src/Exscript/util/crypt.py | 2 +- src/Exscript/util/daemonize.py | 4 ++-- src/Exscript/util/file.py | 10 ++++---- src/Exscript/util/ip.py | 3 +-- src/Exscript/util/pidutil.py | 4 ++-- src/Exscript/util/template.py | 4 ++-- src/Exscript/util/tty.py | 2 +- tests/Exscript/AccountManagerTest.py | 2 +- tests/Exscript/AccountPoolTest.py | 1 - tests/Exscript/FileLoggerTest.py | 9 +++---- tests/Exscript/HostTest.py | 1 - tests/Exscript/KnownIssuesTest.py | 1 - tests/Exscript/LexerTest.py | 1 - tests/Exscript/LogTest.py | 1 - tests/Exscript/LogfileTest.py | 2 -- tests/Exscript/LoggerTest.py | 2 -- tests/Exscript/PrivateKeyTest.py | 1 - tests/Exscript/QueueTest.py | 1 - tests/Exscript/emulators/IOSEmulatorTest.py | 1 - tests/Exscript/protocols/DummyTest.py | 1 - .../Exscript/protocols/OsGuesserCorpusTest.py | 1 - tests/Exscript/protocols/OsGuesserTest.py | 1 - tests/Exscript/protocols/ProtocolTest.py | 4 +--- tests/Exscript/protocols/SSH2Test.py | 1 - tests/Exscript/protocols/TelnetTest.py | 1 - tests/Exscript/servers/SSHdTest.py | 1 - tests/Exscript/servers/ServerTest.py | 4 +--- tests/Exscript/servers/TelnetdTest.py | 1 - tests/Exscript/util/bufferTest.py | 2 -- tests/Exscript/util/castTest.py | 3 +-- tests/Exscript/util/decoratorTest.py | 2 -- tests/Exscript/util/eventTest.py | 1 + tests/Exscript/util/fileTest.py | 6 ++--- tests/Exscript/util/interactTest.py | 2 +- tests/Exscript/util/ipv4Test.py | 1 - tests/Exscript/util/mailTest.py | 4 ++-- tests/Exscript/util/reportTest.py | 4 +--- tests/Exscript/util/startTest.py | 2 +- tests/Exscript/util/ttyTest.py | 4 +--- tests/Exscript/util/urlTest.py | 1 - tests/Exscript/util/weakmethodTest.py | 1 - tests/Exscript/workqueue/JobTest.py | 2 -- tests/Exscript/workqueue/MainLoopTest.py | 1 - tests/Exscript/workqueue/PipelineTest.py | 3 --- tests/Exscript/workqueue/TaskTest.py | 2 +- tests/Exscript/workqueue/WorkQueueTest.py | 5 +--- tools/bench_bulk.py | 1 - 67 files changed, 97 insertions(+), 113 deletions(-) create mode 100644 .git-blame-ignore-revs diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs new file mode 100644 index 00000000..c698b2ca --- /dev/null +++ b/.git-blame-ignore-revs @@ -0,0 +1,3 @@ +# Mechanical reformatting commits; use: git config blame.ignoreRevsFile .git-blame-ignore-revs +# Format the code base with ruff +466247c9eb884d2da1b9900bff82a6fa464c76a1 diff --git a/pyproject.toml b/pyproject.toml index 66db3475..053e346c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -88,10 +88,34 @@ quote-style = "preserve" select = ["E", "W", "F", "I", "UP", "B", "S"] ignore = [ "UP031", # printf-style formatting is used throughout + # Baseline of existing findings: fix a rule everywhere, then remove it here. + "B004", "B006", "B007", "B008", "B011", "B017", "B018", "B904", "B905", + "E402", "E501", "E721", "E722", "E731", "E741", "F401", "F841", "UP003", ] [tool.ruff.lint.per-file-ignores] "tests/**" = ["S101", "S104", "S105", "S106", "S107", "S108", "S311"] +# Security baseline: existing findings per file, to be reviewed (S1-S6); new code is checked. +"demos/template/template.py" = ["S101"] +"scripts/exscript" = ["S307", "S605"] +"src/Exscript/emulators/iosemu.py" = ["S105"] +"src/Exscript/emulators/vdevice.py" = ["S105"] +"src/Exscript/interpreter/string.py" = ["S105"] +"src/Exscript/interpreter/template.py" = ["S105"] +"src/Exscript/protocols/netconf.py" = ["S110"] +"src/Exscript/protocols/protocol.py" = ["S101", "S110"] +"src/Exscript/protocols/ssh2.py" = ["S110"] +"src/Exscript/protocols/telnet.py" = ["S101", "S110"] +"src/Exscript/servers/httpd.py" = ["S105", "S324"] +"src/Exscript/stdlib/mysys.py" = ["S602"] +"src/Exscript/util/crypt.py" = ["S303"] +"src/Exscript/util/file.py" = ["S107"] +"src/Exscript/util/pidutil.py" = ["S110"] +"src/Exscript/util/tty.py" = ["S607"] +"src/Exscript/workqueue/pipeline.py" = ["S101"] +"tests/Exscript/QueueTest.py" = ["S110"] +"tests/Exscript/workqueue/JobTest.py" = ["S301"] +"tools/bench_bulk.py" = ["S101"] [tool.pytest.ini_options] testpaths = ["tests"] diff --git a/scripts/exscript b/scripts/exscript index 2e9d6916..b389bd70 100755 --- a/scripts/exscript +++ b/scripts/exscript @@ -121,9 +121,9 @@ def run(): if not args.filename: parser.error("Missing template filename") try: - with open(args.filename, "r") as fp: + with open(args.filename) as fp: script_content = fp.read() - except IOError as e: + except OSError as e: parser.error(str(e)) # Extract the hostnames out of the command line arguments. @@ -136,7 +136,7 @@ def run(): try: path = Path(os.path.expandvars(args.csv_hosts)).expanduser() csv_hosts = mkhosts_from_csv(path) - except IOError as e: + except OSError as e: parser.error(str(e)) if not csv_hosts: print(f"Warning: {args.csv_hosts} is empty.") @@ -147,7 +147,7 @@ def run(): try: path = Path(os.path.expandvars(args.hosts)).expanduser() txt_hosts = mkhosts_from_file(path) - except IOError as e: + except OSError as e: parser.error(str(e)) except ValueError as e: parser.error(e) @@ -163,7 +163,7 @@ def run(): spec = importlib.util.spec_from_file_location(path.stem, path) driver_module = importlib.util.module_from_spec(spec) spec.loader.exec_module(driver_module) - except IOError as e: + except OSError as e: parser.error(str(e)) except Exception: sys.stderr.write(f"Error in driver {args.load_driver}\n") @@ -201,7 +201,7 @@ def run(): if args.lib: try: functions = load_lib(args.lib) - except IOError as e: + except OSError as e: parser.error(str(e)) # Test whether the template compiles. @@ -240,7 +240,7 @@ def run(): path = Path(os.path.expandvars(args.account_pool)).expanduser() accounts = get_accounts_from_file(path) default_pool.add_account(accounts) - except IOError as e: + except OSError as e: parser.error(str(e)) if args.account_pool and default_pool.n_accounts() == 0: print(f"WARNING: No accounts found in account pool file ({args.account_pool})!") @@ -321,7 +321,7 @@ def run(): print(f"Creating log directory ({args.logdir})...") try: os.makedirs(args.logdir) - except IOError as e: + except OSError as e: parser.error(str(e)) # Enable logging. diff --git a/src/Exscript/account.py b/src/Exscript/account.py index 3e773913..99a9df8e 100644 --- a/src/Exscript/account.py +++ b/src/Exscript/account.py @@ -292,7 +292,7 @@ def acquire(self): elif self.account_hash: self.parent.send(('acquire-account-from-hash', self.account_hash)) else: - self.parent.send(('acquire-account')) + self.parent.send('acquire-account') response = self.parent.recv() if isinstance(response, Exception): diff --git a/src/Exscript/emulators/command.py b/src/Exscript/emulators/command.py index cd2f4347..99262b26 100644 --- a/src/Exscript/emulators/command.py +++ b/src/Exscript/emulators/command.py @@ -89,7 +89,7 @@ def add_from_file(self, filename, handler_decorator=None): named "commands" or if "commands" is not iterable. """ args = {} - with open(filename, "r", encoding="utf-8") as fin: + with open(filename, encoding="utf-8") as fin: source = fin.read() code = compile(source, filename, mode="exec") exec(code, args) # noqa: S102 diff --git a/src/Exscript/logger.py b/src/Exscript/logger.py index 0eaab1a9..c9ae3cf1 100644 --- a/src/Exscript/logger.py +++ b/src/Exscript/logger.py @@ -200,10 +200,10 @@ def __init__(self, name, filename, mode='a', delete=False): def __str__(self): data = '' if os.path.isfile(self.filename): - with open(self.filename, 'r') as thefile: + with open(self.filename) as thefile: data += thefile.read() if os.path.isfile(self.errorname): - with open(self.errorname, 'r') as thefile: + with open(self.errorname) as thefile: data += thefile.read() return data diff --git a/src/Exscript/protocols/netconf.py b/src/Exscript/protocols/netconf.py index eb84a679..086138bb 100644 --- a/src/Exscript/protocols/netconf.py +++ b/src/Exscript/protocols/netconf.py @@ -299,7 +299,7 @@ def login(self, account=None, app_account=None, flush=True): self.app_authorize() # just to set the flag def authenticate(self, account=None, app_account=None, flush=True): - super(Netconf, self).authenticate(account, app_account, flush=flush) + super().authenticate(account, app_account, flush=flush) # Authentication is complete and the subsystem channel is open. The # server waits unread in the channel buffer. Hand the channel to # ncclient now — it will consume the and start its IO thread. @@ -460,7 +460,7 @@ def _close(self, force=False): # SSH2 closes the paramiko channel, transport, and socket. # Closing the channel causes channel.recv() to return b'', which — # combined with _closing being set — causes the IO thread to exit. - super(Netconf, self)._close(force=force) + super()._close(force=force) # Wait for ncclient's IO thread to fully terminate. # Must be after super() so the channel is already closed when we join. diff --git a/src/Exscript/protocols/protocol.py b/src/Exscript/protocols/protocol.py index 39b8384e..5f48a851 100644 --- a/src/Exscript/protocols/protocol.py +++ b/src/Exscript/protocols/protocol.py @@ -1555,7 +1555,7 @@ def handle_sigwinch(signum, frame): while True: try: r, w, e = select.select([channel, stdin], [], []) - except select.error as e: + except OSError as e: code, message = e if code == errno.EINTR: # This may happen when SIGWINCH is called @@ -1566,7 +1566,7 @@ def handle_sigwinch(signum, frame): if channel in r: try: data = channel.recv(1024).decode(self.encoding) - except socket.timeout: + except TimeoutError: pass if not data: self._dbg(1, 'EOF from remote host') diff --git a/src/Exscript/protocols/ssh2.py b/src/Exscript/protocols/ssh2.py index 72a84b7f..7e8b1227 100644 --- a/src/Exscript/protocols/ssh2.py +++ b/src/Exscript/protocols/ssh2.py @@ -160,7 +160,7 @@ def _load_system_host_keys(self, filename=None): filename = os.path.expanduser('~/.ssh/known_hosts') try: self._system_host_keys.load(filename) - except IOError: + except OSError: pass return self._system_host_keys.load(filename) @@ -281,7 +281,7 @@ def _paramiko_auth_key(self, username, keys, password): return except SSHException as e: saved_exception = e - except IOError as e: + except OSError as e: saved_exception = e raise saved_exception diff --git a/src/Exscript/protocols/telnet.py b/src/Exscript/protocols/telnet.py index af1ae2fb..530d7de3 100644 --- a/src/Exscript/protocols/telnet.py +++ b/src/Exscript/protocols/telnet.py @@ -109,7 +109,7 @@ def _domatch(self, prompt, flush): return result, match def cancel_expect(self): - super(Telnet, self).cancel_expect() + super().cancel_expect() if self._tn: self._tn.cancel_expect = True diff --git a/src/Exscript/protocols/telnetlib.py b/src/Exscript/protocols/telnetlib.py index dc55e4e6..2b851bea 100644 --- a/src/Exscript/protocols/telnetlib.py +++ b/src/Exscript/protocols/telnetlib.py @@ -253,15 +253,15 @@ def open(self, host, port=0): self.sock = socket.socket(af, socktype, proto) self.sock.settimeout(self.connect_timeout) self.sock.connect(sa) - except socket.error as msg_: - msg = '{} => telnet://{}:{}'.format(msg_, host, port) + except OSError as msg_: + msg = f'{msg_} => telnet://{host}:{port}' if self.sock: self.sock.close() self.sock = None continue break if not self.sock: - raise socket.error(msg) + raise OSError(msg) def msg(self, msg, *args): """Print a debug message, when the debug level is > 0. diff --git a/src/Exscript/queue.py b/src/Exscript/queue.py index d9d3db8a..c132cb8e 100644 --- a/src/Exscript/queue.py +++ b/src/Exscript/queue.py @@ -190,7 +190,7 @@ def run(self): while True: try: request = self.to_child.recv() - except (EOFError, IOError): + except (OSError, EOFError): self.accm.release_accounts(self) break self._handle_request(request) diff --git a/src/Exscript/servers/httpd.py b/src/Exscript/servers/httpd.py index cab08225..96d8d152 100644 --- a/src/Exscript/servers/httpd.py +++ b/src/Exscript/servers/httpd.py @@ -220,7 +220,7 @@ def _do_POSTGET(self, handler): # Extract POST data, if any. Clumsy syntax due to Python 2 and # 2to3's lack of a byte literal. - self.data = "".encode() + self.data = b"" length = self.headers.get('Content-Length') if length and length.isdigit(): self.data = self.rfile.read(int(length)) @@ -262,7 +262,7 @@ def handle_POST(self): """ self.send_response(404) self.end_headers() - self.wfile.write('not found'.encode('utf8')) + self.wfile.write(b'not found') def handle_GET(self): """ @@ -271,7 +271,7 @@ def handle_GET(self): """ self.send_response(404) self.end_headers() - self.wfile.write('not found'.encode('utf8')) + self.wfile.write(b'not found') def send_response(self, code): """ diff --git a/src/Exscript/servers/server.py b/src/Exscript/servers/server.py index 0e4571b9..5bc3f886 100644 --- a/src/Exscript/servers/server.py +++ b/src/Exscript/servers/server.py @@ -91,7 +91,7 @@ def _poll_child_process(self): try: msg = self.to_parent.recv() self._dbg(1, 'Received command via pipe from parent: %s' % repr(msg)) - except socket.error: + except OSError: self.running = False return False if msg == 'shutdown': diff --git a/src/Exscript/servers/sshd.py b/src/Exscript/servers/sshd.py index cd105a62..d2c199da 100644 --- a/src/Exscript/servers/sshd.py +++ b/src/Exscript/servers/sshd.py @@ -129,7 +129,7 @@ def _recvline(self, channel, buf): return None, buf try: data = channel.recv(1024) - except socket.timeout: + except TimeoutError: continue if not data: return None, buf # client disconnected — keep server running @@ -195,7 +195,7 @@ def _handle_connection(self, conn): response = self.device.do(line) if response: channel.send(response.encode('utf8')) - except (socket.error, EOFError) as err: + except (OSError, EOFError) as err: msg = 'Client disappeared: ' + str(err) self._dbg(1, msg) except Exception as err: diff --git a/src/Exscript/servers/telnetd.py b/src/Exscript/servers/telnetd.py index 8dbaa2dd..cb908b6d 100644 --- a/src/Exscript/servers/telnetd.py +++ b/src/Exscript/servers/telnetd.py @@ -79,7 +79,7 @@ def _handle_connection(self, conn): response = self.device.do(line) if response: conn.send(response.encode('utf8')) - except (socket.error, EOFError) as err: + except (OSError, EOFError) as err: msg = 'Client disappeared: ' + str(err) self._dbg(1, msg) except Exception as err: diff --git a/src/Exscript/stdlib/file.py b/src/Exscript/stdlib/file.py index 924f56ee..796fc0ae 100644 --- a/src/Exscript/stdlib/file.py +++ b/src/Exscript/stdlib/file.py @@ -21,6 +21,7 @@ # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. import os + from Exscript.stdlib.util import secure_function @@ -97,7 +98,7 @@ def read(scope, filename): :rtype: string :return: The content of the file. """ - with open(filename[0], 'r') as fp: + with open(filename[0]) as fp: lines = fp.readlines() scope.define(__response__=lines) return lines diff --git a/src/Exscript/stdlib/ipv4.py b/src/Exscript/stdlib/ipv4.py index 9590a691..37bb2bd4 100644 --- a/src/Exscript/stdlib/ipv4.py +++ b/src/Exscript/stdlib/ipv4.py @@ -20,8 +20,8 @@ # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -from Exscript.util import ipv4 from Exscript.stdlib.util import secure_function +from Exscript.util import ipv4 @secure_function diff --git a/src/Exscript/stdlib/util.py b/src/Exscript/stdlib/util.py index 3b51a9cc..b73c8796 100644 --- a/src/Exscript/stdlib/util.py +++ b/src/Exscript/stdlib/util.py @@ -34,5 +34,5 @@ def secure_function(function): def wrapper(*args, **kwargs): return function(*args, **kwargs) - setattr(wrapper, '_is_secure', True) + wrapper._is_secure = True return wrapper diff --git a/src/Exscript/util/cast.py b/src/Exscript/util/cast.py index a6c4c4c5..be796315 100644 --- a/src/Exscript/util/cast.py +++ b/src/Exscript/util/cast.py @@ -62,7 +62,7 @@ def to_host(host, default_protocol='ssh', default_domain=''): raise TypeError('None can not be cast to Host') if hasattr(host, 'get_address'): return host - if default_domain and not '.' in host: + if default_domain and '.' not in host: host += '.' + default_domain return Exscript.Host(host, default_protocol=default_protocol) diff --git a/src/Exscript/util/crypt.py b/src/Exscript/util/crypt.py index 639c8b84..2f9fe22e 100644 --- a/src/Exscript/util/crypt.py +++ b/src/Exscript/util/crypt.py @@ -2148,7 +2148,7 @@ def otp(password, seed, sequence): if len(seed) not in list(range(1, 17)): raise ValueError('seed length') for x in seed: - if not x in _VALIDSEEDCHARACTERS: + if x not in _VALIDSEEDCHARACTERS: raise ValueError('seed composition') if sequence < 0: raise ValueError('sequence') diff --git a/src/Exscript/util/daemonize.py b/src/Exscript/util/daemonize.py index ec48e21b..abc7966d 100644 --- a/src/Exscript/util/daemonize.py +++ b/src/Exscript/util/daemonize.py @@ -24,14 +24,14 @@ Daemonizing a process. """ -import sys import os +import sys def _redirect_output(filename): out_log = open(filename, 'a+', 0) err_log = open(filename, 'a+', 0) - dev_null = open(os.devnull, 'r') + dev_null = open(os.devnull) os.close(sys.stdin.fileno()) os.close(sys.stdout.fileno()) os.close(sys.stderr.fileno()) diff --git a/src/Exscript/util/file.py b/src/Exscript/util/file.py index a1185e8a..2f5c64ac 100644 --- a/src/Exscript/util/file.py +++ b/src/Exscript/util/file.py @@ -98,12 +98,12 @@ def get_hosts_from_file( """ # Open the file. if not os.path.exists(filename): - raise IOError('No such file: %s' % filename) + raise OSError('No such file: %s' % filename) # Read the hostnames. have = set() hosts = [] - with open(filename, 'r', encoding=encoding) as file_handle: + with open(filename, encoding=encoding) as file_handle: for line in file_handle: hostname = line.split('#')[0].strip() if hostname == '': @@ -153,9 +153,9 @@ def get_hosts_from_csv(filename, default_protocol='ssh', default_domain='', enco """ # Open the file. if not os.path.exists(filename): - raise IOError('No such file: %s' % filename) + raise OSError('No such file: %s' % filename) - with open(filename, 'r', encoding=encoding) as file_handle: + with open(filename, encoding=encoding) as file_handle: # Read and check the header. header = file_handle.readline().rstrip() if re.search(r'^(?:hostname|address)\b', header) is None: @@ -220,7 +220,7 @@ def load_lib(filename): """ # Open the file. if not os.path.exists(filename): - raise IOError('No such file: %s' % filename) + raise OSError('No such file: %s' % filename) name = os.path.splitext(os.path.basename(filename))[0] # load_module() was removed in Python 3.15; an explicit loader also accepts other suffixes. diff --git a/src/Exscript/util/ip.py b/src/Exscript/util/ip.py index e66c6121..bea08d64 100644 --- a/src/Exscript/util/ip.py +++ b/src/Exscript/util/ip.py @@ -24,8 +24,7 @@ Wrapper around the ipv4 and ipv6 modules to handle both, ipv4 and ipv6. """ -from Exscript.util import ipv4 -from Exscript.util import ipv6 +from Exscript.util import ipv4, ipv6 def is_ip(string): diff --git a/src/Exscript/util/pidutil.py b/src/Exscript/util/pidutil.py index e2fb0a7d..112a61ea 100644 --- a/src/Exscript/util/pidutil.py +++ b/src/Exscript/util/pidutil.py @@ -48,7 +48,7 @@ def read(path): _logger.info("Checking pidfile '%s'", path) try: return int(open(path).read()) - except IOError as xxx_todo_changeme: + except OSError as xxx_todo_changeme: (code, text) = xxx_todo_changeme.args if code == errno.ENOENT: # no such file or directory return None @@ -140,5 +140,5 @@ def remove(path): _logger.info("Removing pidfile '%s'", path) try: os.unlink(path) - except IOError: + except OSError: pass diff --git a/src/Exscript/util/template.py b/src/Exscript/util/template.py index 3375baa3..8e60378a 100644 --- a/src/Exscript/util/template.py +++ b/src/Exscript/util/template.py @@ -166,7 +166,7 @@ def eval_file(conn, filename, strip_command=True, no_escaping=False, **kwargs): :param kwargs: Variables to define in the template. """ parser_args = {'strip_command': strip_command, 'no_escaping': no_escaping} - with open(filename, 'r') as fp: + with open(filename) as fp: return _run(conn, filename, fp.read(), parser_args, **kwargs) @@ -216,5 +216,5 @@ def paste_file(conn, filename, no_escaping=False, **kwargs): :param kwargs: Variables to define in the template. """ parser_args = {'no_prompt': True, 'no_escaping': no_escaping} - with open(filename, 'r') as fp: + with open(filename) as fp: return _run(conn, filename, fp.read(), parser_args, **kwargs) diff --git a/src/Exscript/util/tty.py b/src/Exscript/util/tty.py index f3e8d40a..dc8c2c70 100644 --- a/src/Exscript/util/tty.py +++ b/src/Exscript/util/tty.py @@ -39,7 +39,7 @@ def _get_terminal_size(fd): s = struct.pack('HHHH', 0, 0, 0, 0) try: x = fcntl.ioctl(fd, termios.TIOCGWINSZ, s) - except IOError: # Window size ioctl not supported. + except OSError: # Window size ioctl not supported. return None try: rows, cols, x_pixels, y_pixels = struct.unpack('HHHH', x) diff --git a/tests/Exscript/AccountManagerTest.py b/tests/Exscript/AccountManagerTest.py index 28f113d6..f48a37e4 100644 --- a/tests/Exscript/AccountManagerTest.py +++ b/tests/Exscript/AccountManagerTest.py @@ -1,6 +1,6 @@ import unittest -from Exscript.account import Account, AccountPool, AccountManager +from Exscript.account import Account, AccountManager, AccountPool class AccountManagerTest(unittest.TestCase): diff --git a/tests/Exscript/AccountPoolTest.py b/tests/Exscript/AccountPoolTest.py index 1f39fb1c..3aba0f86 100644 --- a/tests/Exscript/AccountPoolTest.py +++ b/tests/Exscript/AccountPoolTest.py @@ -1,7 +1,6 @@ import os.path import unittest - from Exscript import Account from Exscript.account import AccountPool from Exscript.util.file import get_accounts_from_file diff --git a/tests/Exscript/FileLoggerTest.py b/tests/Exscript/FileLoggerTest.py index af2f9b9a..ea226d6f 100644 --- a/tests/Exscript/FileLoggerTest.py +++ b/tests/Exscript/FileLoggerTest.py @@ -1,10 +1,11 @@ -import unittest import os.path - -from tempfile import mkdtemp +import unittest from shutil import rmtree +from tempfile import mkdtemp + +from LoggerTest import FakeJob, LoggerTest + from Exscript import FileLogger -from LoggerTest import LoggerTest, FakeJob class FakeError(Exception): diff --git a/tests/Exscript/HostTest.py b/tests/Exscript/HostTest.py index 8e11bed5..8e6085d3 100644 --- a/tests/Exscript/HostTest.py +++ b/tests/Exscript/HostTest.py @@ -1,6 +1,5 @@ import unittest - from util.urlTest import urls from Exscript import Account, Host diff --git a/tests/Exscript/KnownIssuesTest.py b/tests/Exscript/KnownIssuesTest.py index ce3a0692..e03e43f6 100644 --- a/tests/Exscript/KnownIssuesTest.py +++ b/tests/Exscript/KnownIssuesTest.py @@ -17,7 +17,6 @@ import uuid from unittest import mock - import paramiko from Exscript import Account, PrivateKey diff --git a/tests/Exscript/LexerTest.py b/tests/Exscript/LexerTest.py index 978b9935..4fbadd7b 100644 --- a/tests/Exscript/LexerTest.py +++ b/tests/Exscript/LexerTest.py @@ -1,6 +1,5 @@ import unittest - from Exscript.parselib import Lexer diff --git a/tests/Exscript/LogTest.py b/tests/Exscript/LogTest.py index 3050ad7c..33d2b423 100644 --- a/tests/Exscript/LogTest.py +++ b/tests/Exscript/LogTest.py @@ -1,7 +1,6 @@ import sys import unittest - from util.reportTest import FakeError from Exscript.logger import Log diff --git a/tests/Exscript/LogfileTest.py b/tests/Exscript/LogfileTest.py index 7854ecad..de0f0b6c 100644 --- a/tests/Exscript/LogfileTest.py +++ b/tests/Exscript/LogfileTest.py @@ -1,7 +1,5 @@ import os.path import unittest - - from shutil import rmtree from tempfile import mkdtemp diff --git a/tests/Exscript/LoggerTest.py b/tests/Exscript/LoggerTest.py index 466f5427..8ed50d5c 100644 --- a/tests/Exscript/LoggerTest.py +++ b/tests/Exscript/LoggerTest.py @@ -1,7 +1,5 @@ import sys import unittest - - from itertools import islice from LogTest import FakeError diff --git a/tests/Exscript/PrivateKeyTest.py b/tests/Exscript/PrivateKeyTest.py index ce554bb0..8bb3d4a8 100644 --- a/tests/Exscript/PrivateKeyTest.py +++ b/tests/Exscript/PrivateKeyTest.py @@ -1,6 +1,5 @@ import unittest - from Exscript import PrivateKey diff --git a/tests/Exscript/QueueTest.py b/tests/Exscript/QueueTest.py index d80086ee..8f6aafa3 100644 --- a/tests/Exscript/QueueTest.py +++ b/tests/Exscript/QueueTest.py @@ -3,7 +3,6 @@ import unittest import warnings - warnings.simplefilter('ignore', DeprecationWarning) import ctypes diff --git a/tests/Exscript/emulators/IOSEmulatorTest.py b/tests/Exscript/emulators/IOSEmulatorTest.py index ed3267c8..ff0bba37 100644 --- a/tests/Exscript/emulators/IOSEmulatorTest.py +++ b/tests/Exscript/emulators/IOSEmulatorTest.py @@ -1,6 +1,5 @@ import unittest - from Exscript.emulators import IOSEmulator from Exscript.emulators.iosemu import iosbanner diff --git a/tests/Exscript/protocols/DummyTest.py b/tests/Exscript/protocols/DummyTest.py index 43e64818..b06d0ff7 100644 --- a/tests/Exscript/protocols/DummyTest.py +++ b/tests/Exscript/protocols/DummyTest.py @@ -1,7 +1,6 @@ import re import unittest - from Exscript.emulators import VirtualDevice from Exscript.protocols import Dummy diff --git a/tests/Exscript/protocols/OsGuesserCorpusTest.py b/tests/Exscript/protocols/OsGuesserCorpusTest.py index 30ae530f..20d4843f 100644 --- a/tests/Exscript/protocols/OsGuesserCorpusTest.py +++ b/tests/Exscript/protocols/OsGuesserCorpusTest.py @@ -16,7 +16,6 @@ import os import unittest - from Exscript.protocols.osguesser import OsGuesser DIRNAME = os.path.dirname(os.path.abspath(__file__)) diff --git a/tests/Exscript/protocols/OsGuesserTest.py b/tests/Exscript/protocols/OsGuesserTest.py index ab158637..40d366e0 100644 --- a/tests/Exscript/protocols/OsGuesserTest.py +++ b/tests/Exscript/protocols/OsGuesserTest.py @@ -2,7 +2,6 @@ import re import unittest - from Exscript.protocols import drivers from Exscript.protocols.osguesser import OsGuesser diff --git a/tests/Exscript/protocols/ProtocolTest.py b/tests/Exscript/protocols/ProtocolTest.py index 33baa8ea..84f9f1eb 100644 --- a/tests/Exscript/protocols/ProtocolTest.py +++ b/tests/Exscript/protocols/ProtocolTest.py @@ -1,11 +1,9 @@ import os import re import tempfile +import time import unittest import unittest.mock - - -import time from functools import partial from Exscript import Account, PrivateKey diff --git a/tests/Exscript/protocols/SSH2Test.py b/tests/Exscript/protocols/SSH2Test.py index 5890d2de..db00f1ee 100644 --- a/tests/Exscript/protocols/SSH2Test.py +++ b/tests/Exscript/protocols/SSH2Test.py @@ -1,7 +1,6 @@ import os.path import unittest - from Exscript import PrivateKey from Exscript.protocols import SSH2 from Exscript.servers import SSHd, sshd diff --git a/tests/Exscript/protocols/TelnetTest.py b/tests/Exscript/protocols/TelnetTest.py index 80bd7f4f..c720fdb9 100644 --- a/tests/Exscript/protocols/TelnetTest.py +++ b/tests/Exscript/protocols/TelnetTest.py @@ -1,6 +1,5 @@ import unittest - from Exscript.protocols import Telnet from Exscript.servers import Telnetd diff --git a/tests/Exscript/servers/SSHdTest.py b/tests/Exscript/servers/SSHdTest.py index 551074be..e897c619 100644 --- a/tests/Exscript/servers/SSHdTest.py +++ b/tests/Exscript/servers/SSHdTest.py @@ -1,6 +1,5 @@ import unittest - from Exscript.protocols import SSH2 from Exscript.servers import SSHd diff --git a/tests/Exscript/servers/ServerTest.py b/tests/Exscript/servers/ServerTest.py index 638b55f7..fcfa9d53 100644 --- a/tests/Exscript/servers/ServerTest.py +++ b/tests/Exscript/servers/ServerTest.py @@ -1,8 +1,6 @@ import re -import unittest - - import time +import unittest from Exscript import Account from Exscript.emulators import VirtualDevice diff --git a/tests/Exscript/servers/TelnetdTest.py b/tests/Exscript/servers/TelnetdTest.py index 35868d55..9becb8ee 100644 --- a/tests/Exscript/servers/TelnetdTest.py +++ b/tests/Exscript/servers/TelnetdTest.py @@ -1,6 +1,5 @@ import unittest - from Exscript.protocols import Telnet from Exscript.servers import Telnetd diff --git a/tests/Exscript/util/bufferTest.py b/tests/Exscript/util/bufferTest.py index 2b912e49..6c4ff3fe 100644 --- a/tests/Exscript/util/bufferTest.py +++ b/tests/Exscript/util/bufferTest.py @@ -1,6 +1,4 @@ import unittest - - from functools import partial from tempfile import TemporaryFile diff --git a/tests/Exscript/util/castTest.py b/tests/Exscript/util/castTest.py index f0ff261a..7ad61da3 100644 --- a/tests/Exscript/util/castTest.py +++ b/tests/Exscript/util/castTest.py @@ -1,8 +1,7 @@ -import unittest import re +import unittest import Exscript.util.cast -import re from Exscript import Host diff --git a/tests/Exscript/util/decoratorTest.py b/tests/Exscript/util/decoratorTest.py index 887008a3..598533be 100644 --- a/tests/Exscript/util/decoratorTest.py +++ b/tests/Exscript/util/decoratorTest.py @@ -1,6 +1,4 @@ import unittest - - from functools import partial from multiprocessing import Value diff --git a/tests/Exscript/util/eventTest.py b/tests/Exscript/util/eventTest.py index 7742abdd..337a1a8f 100644 --- a/tests/Exscript/util/eventTest.py +++ b/tests/Exscript/util/eventTest.py @@ -30,6 +30,7 @@ def testConnect(self): def testListen(self): import gc + from Exscript.util.weakmethod import WeakMethod def thefunction(): diff --git a/tests/Exscript/util/fileTest.py b/tests/Exscript/util/fileTest.py index ee00c25b..ada99597 100644 --- a/tests/Exscript/util/fileTest.py +++ b/tests/Exscript/util/fileTest.py @@ -1,9 +1,9 @@ -import unittest +import base64 import os.path +import unittest +from tempfile import NamedTemporaryFile -import base64 import Exscript.util.file -from tempfile import NamedTemporaryFile account_pool = [ ('user1', 'password1'), diff --git a/tests/Exscript/util/interactTest.py b/tests/Exscript/util/interactTest.py index 6cadb279..f88f1ca3 100644 --- a/tests/Exscript/util/interactTest.py +++ b/tests/Exscript/util/interactTest.py @@ -1,6 +1,6 @@ import unittest - from tempfile import NamedTemporaryFile + import Exscript.util.interact from Exscript.util.interact import InputHistory diff --git a/tests/Exscript/util/ipv4Test.py b/tests/Exscript/util/ipv4Test.py index 0c5d4d7e..deffc7b3 100644 --- a/tests/Exscript/util/ipv4Test.py +++ b/tests/Exscript/util/ipv4Test.py @@ -1,6 +1,5 @@ import unittest - import Exscript.util.ipv4 diff --git a/tests/Exscript/util/mailTest.py b/tests/Exscript/util/mailTest.py index e75af12f..de523db9 100644 --- a/tests/Exscript/util/mailTest.py +++ b/tests/Exscript/util/mailTest.py @@ -1,7 +1,7 @@ -import unittest import tempfile - +import unittest from getpass import getuser + import Exscript.util.mail from Exscript.util.mail import Mail diff --git a/tests/Exscript/util/reportTest.py b/tests/Exscript/util/reportTest.py index 770c939a..06c4ee25 100644 --- a/tests/Exscript/util/reportTest.py +++ b/tests/Exscript/util/reportTest.py @@ -1,9 +1,7 @@ import os import sys -import unittest - - import traceback +import unittest import Exscript.util.report from Exscript import Host, Logger diff --git a/tests/Exscript/util/startTest.py b/tests/Exscript/util/startTest.py index 38069ddd..2dd63b14 100644 --- a/tests/Exscript/util/startTest.py +++ b/tests/Exscript/util/startTest.py @@ -1,8 +1,8 @@ import unittest +from multiprocessing import Value import Exscript import Exscript.util.start -from multiprocessing import Value def count_calls(job, host, conn, data, **kwargs): diff --git a/tests/Exscript/util/ttyTest.py b/tests/Exscript/util/ttyTest.py index fc5a0fed..6028ae0e 100644 --- a/tests/Exscript/util/ttyTest.py +++ b/tests/Exscript/util/ttyTest.py @@ -1,9 +1,7 @@ import os import sys -import unittest - - import tempfile +import unittest import Exscript.util.tty diff --git a/tests/Exscript/util/urlTest.py b/tests/Exscript/util/urlTest.py index bee4c3c8..9ff894ae 100644 --- a/tests/Exscript/util/urlTest.py +++ b/tests/Exscript/util/urlTest.py @@ -1,6 +1,5 @@ import unittest - from Exscript.util.url import Url urls = [ diff --git a/tests/Exscript/util/weakmethodTest.py b/tests/Exscript/util/weakmethodTest.py index 0a802612..a09c4245 100644 --- a/tests/Exscript/util/weakmethodTest.py +++ b/tests/Exscript/util/weakmethodTest.py @@ -1,6 +1,5 @@ import unittest - from Exscript.util.weakmethod import DeadMethodCalled, WeakMethod, ref diff --git a/tests/Exscript/workqueue/JobTest.py b/tests/Exscript/workqueue/JobTest.py index 4ee41c39..a4eebfc0 100644 --- a/tests/Exscript/workqueue/JobTest.py +++ b/tests/Exscript/workqueue/JobTest.py @@ -1,6 +1,4 @@ import unittest - - from multiprocessing import Pipe, get_all_start_methods from pickle import dumps, loads diff --git a/tests/Exscript/workqueue/MainLoopTest.py b/tests/Exscript/workqueue/MainLoopTest.py index bc2ab0fa..4e6d8481 100644 --- a/tests/Exscript/workqueue/MainLoopTest.py +++ b/tests/Exscript/workqueue/MainLoopTest.py @@ -1,7 +1,6 @@ import threading import unittest - from Exscript.workqueue.job import Process from Exscript.workqueue.mainloop import MainLoop from Exscript.workqueue.pipeline import Pipeline diff --git a/tests/Exscript/workqueue/PipelineTest.py b/tests/Exscript/workqueue/PipelineTest.py index 6036aa25..6515777a 100644 --- a/tests/Exscript/workqueue/PipelineTest.py +++ b/tests/Exscript/workqueue/PipelineTest.py @@ -1,8 +1,5 @@ import time import unittest - - -import time from multiprocessing import Value from threading import Thread diff --git a/tests/Exscript/workqueue/TaskTest.py b/tests/Exscript/workqueue/TaskTest.py index 5e2bc984..dc3eea5b 100644 --- a/tests/Exscript/workqueue/TaskTest.py +++ b/tests/Exscript/workqueue/TaskTest.py @@ -3,7 +3,7 @@ warnings.simplefilter('ignore', DeprecationWarning) -from Exscript.workqueue import WorkQueue, Task +from Exscript.workqueue import Task, WorkQueue from Exscript.workqueue.job import Thread diff --git a/tests/Exscript/workqueue/WorkQueueTest.py b/tests/Exscript/workqueue/WorkQueueTest.py index f16e5b14..a9e3b579 100644 --- a/tests/Exscript/workqueue/WorkQueueTest.py +++ b/tests/Exscript/workqueue/WorkQueueTest.py @@ -1,9 +1,6 @@ -import time -import unittest - - import random import time +import unittest from multiprocessing import Lock, Value from Exscript.workqueue import WorkQueue diff --git a/tools/bench_bulk.py b/tools/bench_bulk.py index 1d24f120..556050da 100755 --- a/tools/bench_bulk.py +++ b/tools/bench_bulk.py @@ -17,7 +17,6 @@ import threading import time - import paramiko from Exscript import Account From fae2abc2a3dbb5737dda0134472cca615ec9a08c Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Fri, 30 Oct 2026 18:45:31 +0100 Subject: [PATCH 009/250] Add mypy, pre-commit and GitHub Actions CI CI runs pre-commit, mypy, the tests on Python 3.12 to 3.15 and the build, and reports the API changes of pull requests. The pycrypto compatibility shims are removed: pycryptodomex is a hard dependency. --- .github/workflows/ci.yml | 85 ++++++++++ .pre-commit-config.yaml | 26 ++++ doc/index.rst | 10 +- doc/templates.rst | 2 +- doc/trouble.rst | 4 +- pyproject.toml | 16 +- src/Exscript/contrib/otel.py | 4 +- src/Exscript/protocols/ssh2.py | 17 -- src/Exscript/servers/sshd.py | 17 -- src/Exscript/util/crypt.py | 5 +- tests/Exscript/KnownIssuesTest.py | 8 +- uv.lock | 247 ++++++++++++++++++++++++++++++ 12 files changed, 388 insertions(+), 53 deletions(-) create mode 100644 .github/workflows/ci.yml create mode 100644 .pre-commit-config.yaml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..d9a34b90 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,85 @@ +name: CI + +on: + push: + branches: [master] + pull_request: + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + lint: + name: Lint and type check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + fetch-depth: 0 # hatch-vcs derives the version from tags + - uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3 + - run: uvx pre-commit run --all-files --show-diff-on-failure + - run: uv run --frozen mypy + + test: + name: Test ${{ matrix.python }} on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + # Windows, macOS and free-threading are not part of the gate until their baselines are green. + continue-on-error: ${{ matrix.experimental }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + python: ["3.12", "3.13", "3.14", "3.15"] + experimental: [false] + include: + - { os: ubuntu-latest, python: "3.14t", experimental: true } + - { os: windows-latest, python: "3.14", experimental: true } + - { os: macos-latest, python: "3.14", experimental: true } + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + fetch-depth: 0 + - uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3 + - name: Run tests + env: + PYTHON: ${{ matrix.python }} + run: uv run --frozen --python "$PYTHON" pytest --cov=Exscript --cov-report=term-missing:skip-covered + + build: + name: Build distributions + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + fetch-depth: 0 + - uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3 + - run: uv build + - run: uvx twine check --strict dist/* + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: dist + path: dist/ + + api-diff: + name: Public API changes (report only) + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + continue-on-error: true + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + fetch-depth: 0 + - uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3 + - name: Compare with the target branch + env: + BASE_REF: ${{ github.base_ref }} + run: uvx griffe check Exscript --search src --against "origin/$BASE_REF" --verbose diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..49a3d9fa --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,26 @@ +# https://pre-commit.com - run with: uvx pre-commit run --all-files +# Test data must stay byte-identical (trailing blanks and missing final newlines are intentional). +exclude: ^tests/(templates|Exscript/protocols/banners)/ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: 3e8a8703264a2f4a69428a0aa4dcb512790b2c8c # frozen: v6.0.0 + hooks: + - id: check-added-large-files + - id: check-merge-conflict + - id: check-toml + - id: check-yaml + - id: end-of-file-fixer + - id: trailing-whitespace + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: 25bcf95026b96e2dd3c0aec3090c3d6ef6ca2e6c # frozen: v0.17.3 + hooks: + - id: ruff-check + - id: ruff-format + - repo: https://github.com/astral-sh/uv-pre-commit + rev: 46787482b9fb2003c56e0f7592e312ef381829d9 # frozen: 0.13.6 + hooks: + - id: uv-lock + - repo: https://github.com/zizmorcore/zizmor-pre-commit + rev: c92afdf2c6ca336aaa8f39e2bd8d48f62dbd8e1f # frozen: v1.31.1 + hooks: + - id: zizmor diff --git a/doc/index.rst b/doc/index.rst index c69bdca2..ed00fa26 100755 --- a/doc/index.rst +++ b/doc/index.rst @@ -21,8 +21,8 @@ What is Exscript? ================= -Exscript is a Python module and a template processor for automating network -connections over protocols such as Telnet or SSH. We attempt to create the +Exscript is a Python module and a template processor for automating network +connections over protocols such as Telnet or SSH. We attempt to create the best possible set of tools for working with Telnet and SSH. Exscript also provides a set of tools and functions for sysadmins, that @@ -30,9 +30,9 @@ simplify **regular expression matching**, **reporting** by email, **logging**, or **syslog** handling, **CSV parsing**, **ip address handling**, **template processing**, and many more. -Exscript may be used to automate sessions with routers from Cisco, Juniper, -OneAccess, Huawei, or any others. If you want to configures machines -running Linux/Unix, IOS, IOS-XR, JunOS, VRP, or any other operating system +Exscript may be used to automate sessions with routers from Cisco, Juniper, +OneAccess, Huawei, or any others. If you want to configures machines +running Linux/Unix, IOS, IOS-XR, JunOS, VRP, or any other operating system that can be used with a terminal, Exscript is what you are looking for. Using Exscript with Python diff --git a/doc/templates.rst b/doc/templates.rst index 1e83e4ca..f86ed8be 100644 --- a/doc/templates.rst +++ b/doc/templates.rst @@ -33,7 +33,7 @@ filename as an argument:: When executing it from the command line, use:: - exscript -d filename=.profile my.exscript localhost + exscript -d filename=.profile my.exscript localhost Note that the -d switch allows passing variables into the template. The example executes the command ls -l .profile. You can also assign a diff --git a/doc/trouble.rst b/doc/trouble.rst index 18353ce9..0044610d 100644 --- a/doc/trouble.rst +++ b/doc/trouble.rst @@ -105,9 +105,9 @@ Using this script, the following conversation may take place: 1. router> show interface descriptions 2. Interface Status Protocol Description 3. Lo0 up up Description of my router> - 4. PO0/0 admin down down + 4. PO0/0 admin down down 5. Serial1/0 up up My WAN link - 6. router> + 6. router> Note that line 3 happens to contain the string "Router>", which looks like a prompt when it really is just a description. So after receiving diff --git a/pyproject.toml b/pyproject.toml index 053e346c..aa780ccb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -63,7 +63,8 @@ test = [ "opentelemetry-sdk>=1.12", "opentelemetry-instrumentation>=0.42b0", ] -dev = [{ include-group = "test" }] +types = ["mypy>=2.4"] +dev = [{ include-group = "test" }, { include-group = "types" }] [tool.hatch.version] source = "vcs" @@ -95,7 +96,7 @@ ignore = [ [tool.ruff.lint.per-file-ignores] "tests/**" = ["S101", "S104", "S105", "S106", "S107", "S108", "S311"] -# Security baseline: existing findings per file, to be reviewed (S1-S6); new code is checked. +# Security baseline: existing findings per file, to be reviewed; new code is checked. "demos/template/template.py" = ["S101"] "scripts/exscript" = ["S307", "S605"] "src/Exscript/emulators/iosemu.py" = ["S105"] @@ -117,6 +118,17 @@ ignore = [ "tests/Exscript/workqueue/JobTest.py" = ["S301"] "tools/bench_bulk.py" = ["S101"] +[tool.mypy] +python_version = "3.12" +files = ["src"] +ignore_missing_imports = true # paramiko, ncclient and others ship no type hints +warn_unused_ignores = true +warn_redundant_casts = true + +[[tool.mypy.overrides]] +module = ["Exscript.protocols.telnetlib"] +ignore_errors = true # vendored copy of the former standard library module + [tool.pytest.ini_options] testpaths = ["tests"] python_files = ["*Test.py"] diff --git a/src/Exscript/contrib/otel.py b/src/Exscript/contrib/otel.py index 22d3f891..9c8df7b5 100644 --- a/src/Exscript/contrib/otel.py +++ b/src/Exscript/contrib/otel.py @@ -41,7 +41,9 @@ from collections.abc import Callable, Collection from timeit import default_timer -from opentelemetry.instrumentation.instrumentor import BaseInstrumentor +from opentelemetry.instrumentation.instrumentor import ( # type: ignore[attr-defined] + BaseInstrumentor, # mypy cannot resolve it in this namespace package +) from opentelemetry.metrics import Histogram, get_meter from opentelemetry.trace import SpanKind, Tracer, get_tracer, set_span_in_context from opentelemetry.trace.span import Span diff --git a/src/Exscript/protocols/ssh2.py b/src/Exscript/protocols/ssh2.py index 7e8b1227..58dfc67b 100644 --- a/src/Exscript/protocols/ssh2.py +++ b/src/Exscript/protocols/ssh2.py @@ -32,11 +32,6 @@ from binascii import hexlify import paramiko - -try: - import Cryptodome as Crypto -except ImportError: - import Crypto from paramiko.ssh_exception import ( AuthenticationException, BadAuthenticationType, @@ -90,18 +85,6 @@ def __init__(self, **kwargs): "password", ] - # Since each protocol may be created in it's own thread, we must - # re-initialize the random number generator to make sure that - # child threads have no way of guessing the numbers of the parent. - # If we don't, PyCrypto generates an error message for security - # reasons. - try: - Crypto.Random.atfork() - except AttributeError: - # pycrypto versions that have no "Random" module also do not - # detect the missing atfork() call, so they do not raise. - pass - # Paramiko client stuff. self._system_host_keys = paramiko.HostKeys() self._host_keys = paramiko.HostKeys() diff --git a/src/Exscript/servers/sshd.py b/src/Exscript/servers/sshd.py index d2c199da..03b90e1f 100644 --- a/src/Exscript/servers/sshd.py +++ b/src/Exscript/servers/sshd.py @@ -30,11 +30,6 @@ import socket import threading -try: - import Cryptodome as Crypto -except ImportError: - import Crypto - import paramiko from paramiko import ServerInterface @@ -63,18 +58,6 @@ def __init__(self): self.event = threading.Event() self._logger = _logger - # Since each server is created in it's own thread, we must - # re-initialize the random number generator to make sure that - # child threads have no way of guessing the numbers of the parent. - # If we don't, PyCrypto generates an error message for security - # reasons. - try: - Crypto.Random.atfork() - except AttributeError: - # pycrypto versions that have no "Random" module also do not - # detect the missing atfork() call, so they do not raise. - pass - def check_channel_request(self, kind, chanid): if kind == 'session': return paramiko.OPEN_SUCCEEDED diff --git a/src/Exscript/util/crypt.py b/src/Exscript/util/crypt.py index 2f9fe22e..1f9884a5 100644 --- a/src/Exscript/util/crypt.py +++ b/src/Exscript/util/crypt.py @@ -27,10 +27,7 @@ import random import string -try: - from Cryptodome.Hash import MD4 -except ImportError: - from Crypto.Hash import MD4 +from Cryptodome.Hash import MD4 _VALIDSEEDCHARACTERS = string.ascii_letters + string.digits diff --git a/tests/Exscript/KnownIssuesTest.py b/tests/Exscript/KnownIssuesTest.py index e03e43f6..ea77eecc 100644 --- a/tests/Exscript/KnownIssuesTest.py +++ b/tests/Exscript/KnownIssuesTest.py @@ -60,7 +60,7 @@ def start_daemon(self, daemon_cls, device, **kwargs): class HostKeyTest(ServerMixin, unittest.TestCase): - @unittest.expectedFailure # S1: known_hosts is only loaded after the check; no [host]:port lookup + @unittest.expectedFailure # known_hosts is only loaded after the check; no [host]:port lookup def testKnownHostsMismatchIsRejected(self): port = self.start_daemon(SSHd, VirtualDevice(HOST), key=PrivateKey.from_file(KEYFILE)) other_key = paramiko.RSAKey.generate(2048) @@ -110,7 +110,7 @@ def testSSH2(self): class SecretHygieneTest(unittest.TestCase): - @unittest.expectedFailure # S3: Protocol.send() logs passwords at debug level 4 + @unittest.expectedFailure # Protocol.send() logs passwords at debug level 4 def testDebugLogRedactsPassword(self): conn = Dummy(device=VirtualDevice('r1', echo=False, strict=False), debug=4) with self.assertLogs('Exscript.protocols', level='DEBUG') as logs: @@ -139,11 +139,11 @@ def runSession(self, login): conn.execute('show run') conn.close(force=True) - @unittest.expectedFailure # S2: the wrapper reads args[1] for app_account -> IndexError + @unittest.expectedFailure # the wrapper reads args[1] for app_account -> IndexError def testLoginWithPositionalAccount(self): self.runSession(lambda conn, account: conn.login(account)) - @unittest.expectedFailure # S2: commands (incl. passwords) and responses are exported + @unittest.expectedFailure # commands (incl. passwords) and responses are exported def testNoSecretsOrDeviceOutputExportedByDefault(self): self.runSession(lambda conn, account: conn.login(account=account)) exported = [ diff --git a/uv.lock b/uv.lock index a835f7e4..2af54347 100644 --- a/uv.lock +++ b/uv.lock @@ -1,6 +1,74 @@ version = 1 revision = 5 requires-python = ">=3.12" +resolution-markers = [ + "python_full_version >= '3.15'", + "python_full_version < '3.15'", +] + +[[package]] +name = "ast-serialize" +version = "0.13.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/21/d5/5dc1ee8a80c3b20a0b63c61b43ba6bc53517faecd1def47bcd752d22268e/ast_serialize-0.13.1.tar.gz", hash = "sha256:401cc84e8888d4d76f2c8d296fa006ed1a5ff993b6aff90dd777c03973efd025", size = 955137, upload-time = "2026-10-17T00:04:17.183Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f3/a8/fbd136b9f70c59e75f59a8e974c84102976f1201828b4f886480ba956936/ast_serialize-0.13.1-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:3e129d44b0cffbea24a7f40117ce49e30993c3d119e584d4817ebd6f87246456", size = 897306, upload-time = "2026-10-17T00:02:52.479Z" }, + { url = "https://files.pythonhosted.org/packages/50/3f/2ad51e9823a790e7a70fd781e9595ab3abeace27f3aefcaa1939457efd72/ast_serialize-0.13.1-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:523d45bab16ddaf1399c04c4fe81bbf180f81e33c2f7bcbe76be56b0f7311ab4", size = 1236082, upload-time = "2026-10-17T00:02:54.82Z" }, + { url = "https://files.pythonhosted.org/packages/c0/5c/8e959db5cc2237f9d4c560a8e2253c98f9746883fb4774651e4cda458901/ast_serialize-0.13.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:638333fb8ae72c62753f9e794fda20290883ddc65b0d85e26e08f8bb227567ad", size = 1221242, upload-time = "2026-10-17T00:02:56.395Z" }, + { url = "https://files.pythonhosted.org/packages/d6/05/ab1c56c3b7adcac88764bba609502d6b4989e449b2dd60eb478e1d20e369/ast_serialize-0.13.1-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cc34c670747d85d4232834665e247009413eaebe47c69eaf60a5c439aaecaed9", size = 1282105, upload-time = "2026-10-17T00:02:58.275Z" }, + { url = "https://files.pythonhosted.org/packages/0d/f4/41981fcd2ddffe0760e5326be9dcbd2eda9478a8430a637e5123bd8fcb26/ast_serialize-0.13.1-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8c8ebd419811c435812d242fb2a173e57dfede198c9ac9a0a584666ebc9b9331", size = 1287905, upload-time = "2026-10-17T00:03:00.068Z" }, + { url = "https://files.pythonhosted.org/packages/6a/43/b783addc5681153707121991b7b47892f7e9c8ba58a0d03b71a3b023ba47/ast_serialize-0.13.1-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8899c038b1c15ae8b98e379057fb3ef398ceb0b21d1737a8156dd5a0923b7e51", size = 1541806, upload-time = "2026-10-17T00:03:01.503Z" }, + { url = "https://files.pythonhosted.org/packages/47/10/969db095038ee15270f65ad20d1129c5e4da605bf7e93b70ae640f1c8a9f/ast_serialize-0.13.1-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e14c4de98f965647a74e6eaf37e20207124e45451ccd36e84a6f6ebab45e0372", size = 1309946, upload-time = "2026-10-17T00:03:02.971Z" }, + { url = "https://files.pythonhosted.org/packages/06/dd/7c623d2fb0b868948952d1787e629a43b866f7d4caefd5445e20aefacec4/ast_serialize-0.13.1-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7f3e8a3eff69af640db985360fef35073fb46956dfc3e5c614cf00920d2746dd", size = 1302773, upload-time = "2026-10-17T00:03:04.378Z" }, + { url = "https://files.pythonhosted.org/packages/d4/c2/4d49469110a19a44fc6269f4fd59679bcf84df068f185f89b73bee4da19b/ast_serialize-0.13.1-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:75d7750fe98f6f35a96c43aba8d98a2e623e22c778a0c17ab9ede92eb10f5d82", size = 1299579, upload-time = "2026-10-17T00:03:05.88Z" }, + { url = "https://files.pythonhosted.org/packages/9d/9a/2d381f1f262dda15affc8ad643fba1619eb07e7d9f8bd904deffef183006/ast_serialize-0.13.1-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:43e8ddd63e80fc280c2f790157386f27f9b9f6ccb9c4f8e803b863cbfb085769", size = 1355969, upload-time = "2026-10-17T00:03:07.321Z" }, + { url = "https://files.pythonhosted.org/packages/9d/9c/2266794e6c25fa9ce51ce04a9bf4b7e330e3ac9351ba3fa14e487678acdb/ast_serialize-0.13.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:9221e09496fb942901993c1812ee87950cc0ecbb514c7c0f1f5fa6d78f701ed8", size = 1460227, upload-time = "2026-10-17T00:03:08.733Z" }, + { url = "https://files.pythonhosted.org/packages/2b/d6/14deb28e5d8f328f626a0778ea2cf98fe908f2bbcccd804cb6f449ee1782/ast_serialize-0.13.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:7a5c38bdb93669c596f94fb7204dd6ebe4fadb19058ac9a2faf2948c2a175f96", size = 1562691, upload-time = "2026-10-17T00:03:10.25Z" }, + { url = "https://files.pythonhosted.org/packages/f5/19/7a28c0e5ba54f0d3e054f847a2a549b748cb179ab433321a8dc1b39e398d/ast_serialize-0.13.1-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:acf4e9f055d75058ab01536da330648746f2f82b7697a85230547f2d5ebe895c", size = 1558669, upload-time = "2026-10-17T00:03:11.799Z" }, + { url = "https://files.pythonhosted.org/packages/18/9a/f9e2690eab58217f1d395a964a78efa7eaf813a1e38be8dfd9a8d45d0840/ast_serialize-0.13.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:87427913c325adc811e81e7c9a757b061f47ef1e29114ac479aa0953a2687a98", size = 1665071, upload-time = "2026-10-17T00:03:13.284Z" }, + { url = "https://files.pythonhosted.org/packages/8e/f3/ef5b159a0d7161b404f05e7ec21cf7735ee7edc10c000cd2cbbc7bb6b69a/ast_serialize-0.13.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:ad88921f95b92bc144d9ad059a0e16f8c1ae69f5239f2d3c6536fd2774f73dfe", size = 1473364, upload-time = "2026-10-17T00:03:14.563Z" }, + { url = "https://files.pythonhosted.org/packages/1d/87/6e8ac4a865124a12cc4f915bf1bcfac3bba426cd5f58d3712a873ee61012/ast_serialize-0.13.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:f8b17e2921e56621bcb46792bd3e0b82ca3b85de78d5936e7b8174ba1758b0aa", size = 1500566, upload-time = "2026-10-17T00:03:15.917Z" }, + { url = "https://files.pythonhosted.org/packages/f5/1e/29155f1621433008013ad5c940a03d41a676be0680c41527040472d37583/ast_serialize-0.13.1-cp314-cp314t-win32.whl", hash = "sha256:2f9a9ce8b0994f757b941f7f7fe384a6666b4162b941ebc5c248081baea18e47", size = 1128674, upload-time = "2026-10-17T00:03:17.481Z" }, + { url = "https://files.pythonhosted.org/packages/bb/64/2506da65143ea138de5822e9173e0b72320a2f70e23bd723f044033175f1/ast_serialize-0.13.1-cp314-cp314t-win_amd64.whl", hash = "sha256:5d99910e6b9a9b9360bcca2ae99aee36cc912ce058df2a57d1c33099b6cb2593", size = 1163306, upload-time = "2026-10-17T00:03:19.046Z" }, + { url = "https://files.pythonhosted.org/packages/6d/8f/c7cb22638899b0cf15908809c270560e46395e65604bc59e4b623ab9f405/ast_serialize-0.13.1-cp314-cp314t-win_arm64.whl", hash = "sha256:a789cbe14d0a0e9ec9b32a456e5abc81ad8799479c3cfd88a29e3442262aebe6", size = 1133775, upload-time = "2026-10-17T00:03:20.377Z" }, + { url = "https://files.pythonhosted.org/packages/2b/e2/b0c140fe50fa39dd249e382f7733af6f816c90021fe7959a0b026f86237f/ast_serialize-0.13.1-cp315-abi3.abi3t-macosx_10_12_x86_64.whl", hash = "sha256:23d768369b89d551077dc173da6892fad00443338417523b6732f4e8ba4a7ed5", size = 1236160, upload-time = "2026-10-17T00:03:21.945Z" }, + { url = "https://files.pythonhosted.org/packages/23/c4/a8a2ca21e3943555b48221f8fba546877a7ad39efaf6505506524967d7c4/ast_serialize-0.13.1-cp315-abi3.abi3t-macosx_11_0_arm64.whl", hash = "sha256:68f8829d114dab6c9db4a8bbb8d0e342958faee1b5e15a29022678df639960f8", size = 1222214, upload-time = "2026-10-17T00:03:23.462Z" }, + { url = "https://files.pythonhosted.org/packages/15/19/2f58f2ba64e4799b39de85c140aef09a03aa6ea53be75dd292a9d9b9d8ae/ast_serialize-0.13.1-cp315-abi3.abi3t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4e385b17d5bde892a6184003a9aff3518387908b73f8f5ea539313c07c8c0ede", size = 1282979, upload-time = "2026-10-17T00:03:24.695Z" }, + { url = "https://files.pythonhosted.org/packages/b1/af/ca6087fe84210c1ccd43e3182917bb70649fa1e69745a90c1d59ba08fea8/ast_serialize-0.13.1-cp315-abi3.abi3t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:02686b281ec9b8e9614f6f0db27e12e4cac515f336c5d05b678bc9eb73ff2af5", size = 1288596, upload-time = "2026-10-17T00:03:26.04Z" }, + { url = "https://files.pythonhosted.org/packages/b0/2c/4ef964c763329a0347125e745ec0451ba4fbaceec9efe2ac729b7cd55f17/ast_serialize-0.13.1-cp315-abi3.abi3t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a8ab2c2585b2e22e48cd4aff2136fce4eab4ced569b85f150ef9a5a5d6f34d49", size = 1545540, upload-time = "2026-10-17T00:03:27.52Z" }, + { url = "https://files.pythonhosted.org/packages/bb/85/aa961dbb75813cfa94de0ce9a16a9510c226b95f604c405421e727f44bb0/ast_serialize-0.13.1-cp315-abi3.abi3t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c652d9f5d52d8b81eff3ff2c95c9dd0f281de45f6ad5d3fa95527550c396454a", size = 1309374, upload-time = "2026-10-17T00:03:29.091Z" }, + { url = "https://files.pythonhosted.org/packages/b4/e6/f2b67c327d9f9533e41791804403ccc2dc457821195a2e51f0b587a90b90/ast_serialize-0.13.1-cp315-abi3.abi3t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:121e1d08d58b75c15491cc3003609cbad71ac2b0016043c36d30a6b2032b6605", size = 1303466, upload-time = "2026-10-17T00:03:30.451Z" }, + { url = "https://files.pythonhosted.org/packages/73/1d/33f18b81d1bb1f5d682943ea92fda8130a6593627f12f6201668d40824fd/ast_serialize-0.13.1-cp315-abi3.abi3t-manylinux_2_31_riscv64.whl", hash = "sha256:30cf89b6a6844542d116c758725d13de5b97b9410a51ee577715f761270d0c1c", size = 1300576, upload-time = "2026-10-17T00:03:32.102Z" }, + { url = "https://files.pythonhosted.org/packages/25/cb/64914144925f1a4abdd9217e717b122255c38fc5ebe4342f92052f3bacbc/ast_serialize-0.13.1-cp315-abi3.abi3t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:732aaae9e498c973fc63fd7f308009b57c4afa6ff59d0b674d894e57d0bca01a", size = 1355960, upload-time = "2026-10-17T00:03:33.676Z" }, + { url = "https://files.pythonhosted.org/packages/ee/80/bd29379fab2bce19adc0bf9177002b11b23dfe8d8fecf409998b458b4624/ast_serialize-0.13.1-cp315-abi3.abi3t-musllinux_1_2_aarch64.whl", hash = "sha256:d9c1a920b2aa0c1e016bed7721689717034ca0dba64ac3a3ff243b003d4e8dd8", size = 1459779, upload-time = "2026-10-17T00:03:35.09Z" }, + { url = "https://files.pythonhosted.org/packages/83/89/012dbe733de05ffdf8e40ca0f9fa5d417cb16ec93a74463c57c001914198/ast_serialize-0.13.1-cp315-abi3.abi3t-musllinux_1_2_armv7l.whl", hash = "sha256:847cad4c22953823f5e76d0b126c6f9da670cfa45b1df41b9324e22189d77611", size = 1563399, upload-time = "2026-10-17T00:03:37.285Z" }, + { url = "https://files.pythonhosted.org/packages/69/37/ee4bcb0358bee38744eb0f18560446572d095fe3bf6b63d6f739d08a839c/ast_serialize-0.13.1-cp315-abi3.abi3t-musllinux_1_2_i686.whl", hash = "sha256:aca07de3ca8e3d9f2b716c3c31f4f384fe4bd2d03e09912417cff7a3e08036f7", size = 1558808, upload-time = "2026-10-17T00:03:38.711Z" }, + { url = "https://files.pythonhosted.org/packages/54/1c/ff44c5355251d61ef22934c4430bb31a61de4781d25a992154aac9b8c82a/ast_serialize-0.13.1-cp315-abi3.abi3t-musllinux_1_2_ppc64le.whl", hash = "sha256:5db12d843d931b9d1f1519ce66b9cbfe72d8563b75c1204d255322254d1964dd", size = 1666324, upload-time = "2026-10-17T00:03:40.376Z" }, + { url = "https://files.pythonhosted.org/packages/c8/bb/6da4bfc8aa3403e6447607131bf3463acaf954d990c6c8b6787c1902ddfe/ast_serialize-0.13.1-cp315-abi3.abi3t-musllinux_1_2_riscv64.whl", hash = "sha256:fb8021f4dc3296de18737e3b8b34994411da725374d74a0b9eb32a738f2a23d0", size = 1474089, upload-time = "2026-10-17T00:03:41.715Z" }, + { url = "https://files.pythonhosted.org/packages/21/97/897e33bef2eefd880e6350e375cc66bc5bcd00a139be480fd655f5b3f095/ast_serialize-0.13.1-cp315-abi3.abi3t-musllinux_1_2_x86_64.whl", hash = "sha256:0d764913374ba21df8cd618045b371ed104ce55c6276de7c59e554a4f5ba26fd", size = 1501542, upload-time = "2026-10-17T00:03:43.08Z" }, + { url = "https://files.pythonhosted.org/packages/8d/4c/821211ac83870c27e4412381b7c7838336456b40575bf840e4680b2018d9/ast_serialize-0.13.1-cp315-abi3.abi3t-win32.whl", hash = "sha256:2bad302ae2427454ef9dab71bd5c3ad67188860c39d36daee38f72031edb39c4", size = 1126118, upload-time = "2026-10-17T00:03:44.616Z" }, + { url = "https://files.pythonhosted.org/packages/1f/86/2b2d5e6c4fa352874a80618e742ffd32d249dee6c119f4233b51afc6b8f4/ast_serialize-0.13.1-cp315-abi3.abi3t-win_amd64.whl", hash = "sha256:032b09d86c785c3984e975378b7861f2a68ac44e22ba7b07b93a536b2524eeb8", size = 1163766, upload-time = "2026-10-17T00:03:45.986Z" }, + { url = "https://files.pythonhosted.org/packages/77/a3/d610cfcdabaa44fd62a54ca1fc57a7dba120643798b089a14914973562de/ast_serialize-0.13.1-cp315-abi3.abi3t-win_arm64.whl", hash = "sha256:929eda0c419e622ab49df3a4c45f81c57dcc10874bbad55d0470a4a03324dac3", size = 1134043, upload-time = "2026-10-17T00:03:48.453Z" }, + { url = "https://files.pythonhosted.org/packages/f0/89/0e30f9eddf7d44ccf9d25979030754fd0457af61fa4279acb8c293a04f97/ast_serialize-0.13.1-cp315-cp315-pyemscripten_2026_5_wasm32.whl", hash = "sha256:bf53a3ba1cbe45399fc1e4ccd684b8f2a11cca450f89706b79122f478534c6f5", size = 897546, upload-time = "2026-10-17T00:03:50.093Z" }, + { url = "https://files.pythonhosted.org/packages/81/92/d705be88ff452b9ba0088b554a0cbd308fd01879da092b33fd2ad7d1aa60/ast_serialize-0.13.1-cp39-abi3-macosx_10_12_x86_64.whl", hash = "sha256:4bcf1d5f4eefecaa325a34ecf079b2741189e7d0821523f92cb343a389a67489", size = 1242416, upload-time = "2026-10-17T00:03:51.521Z" }, + { url = "https://files.pythonhosted.org/packages/60/c2/98370845a7f35cb8a25782e78284b48e01bf3128339881d48d613f304f8f/ast_serialize-0.13.1-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:1b158f1c9e3a0d386913924c57995d6df78224868a4d0f215bdd41587c97b34a", size = 1229729, upload-time = "2026-10-17T00:03:52.825Z" }, + { url = "https://files.pythonhosted.org/packages/d8/8b/2d7ffed76addd9d3fcf37fb171400942735c8912264c763561a85407f6a8/ast_serialize-0.13.1-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:77a6201b69ce42e919f2fd9a559138d5410b5d8afff7c3a6b1f500b0481ff7d8", size = 1292255, upload-time = "2026-10-17T00:03:54.148Z" }, + { url = "https://files.pythonhosted.org/packages/6b/2a/bb9e0e719d89cc950fadfcfcc9cb4bc7640419c4228c852f031ab8abf440/ast_serialize-0.13.1-cp39-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:6eb0be64c9b8f1ed26637eef345abe75ea90351c92c85a8aec7838b3af971011", size = 1298501, upload-time = "2026-10-17T00:03:55.559Z" }, + { url = "https://files.pythonhosted.org/packages/7a/32/13ce07dce48226f5bd9cfd8f708ec101a8be05aa430cda4918ce31ce8c82/ast_serialize-0.13.1-cp39-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7f2427bddfca208e84de700044fadf9aef2769a70bae740a1bdabb89ee9e1117", size = 1553758, upload-time = "2026-10-17T00:03:56.952Z" }, + { url = "https://files.pythonhosted.org/packages/e2/fb/9b51d186db0c20ca28cbaaefd50f03b573c89327b9a2627820cbce22196b/ast_serialize-0.13.1-cp39-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:88df960e67184a71874e5b2ee014f87d1254a0dbc1f903c7bd7b1ab7733ecaef", size = 1318258, upload-time = "2026-10-17T00:03:58.418Z" }, + { url = "https://files.pythonhosted.org/packages/c5/fa/da311559dd3dae9af5825cf21660775a929b970f72e3786af3f2cd667944/ast_serialize-0.13.1-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9f99786e0d62082a5b121b3437f3167562133711df22128de1e9886f57f0b226", size = 1313260, upload-time = "2026-10-17T00:04:00.194Z" }, + { url = "https://files.pythonhosted.org/packages/2b/11/97a4b9e5b4776d72f47fcd9a09a497bbf68ec8c1ae074307a2c51e410d85/ast_serialize-0.13.1-cp39-abi3-manylinux_2_31_riscv64.whl", hash = "sha256:a01b515b25a03184eff4acef25b981ae007916379682e908fc8785b342a0bf3e", size = 1309992, upload-time = "2026-10-17T00:04:01.575Z" }, + { url = "https://files.pythonhosted.org/packages/2e/c4/62f85121a80cd451ef04f550cf0f7ea0ad9ff46cc4e9a5b9ae439b6527b1/ast_serialize-0.13.1-cp39-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:29422d7bcd9e7f39411571c36cec45ce8a7a9b3ad4b2e7aa64497c15a9e397f6", size = 1362715, upload-time = "2026-10-17T00:04:02.881Z" }, + { url = "https://files.pythonhosted.org/packages/c1/fa/c5f4a545f72a7681e3295bab6e3600ee0862ec4391bed961fe9f735b67e0/ast_serialize-0.13.1-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:6ffb0f548a7dc126067fdfb7b2325ab5dfcdeecc29faf5e0f0461c96d6ebfa3d", size = 1469931, upload-time = "2026-10-17T00:04:04.181Z" }, + { url = "https://files.pythonhosted.org/packages/38/a9/65aed4f8fa7a001a7cfdc03e68e844517ab041ba6fb614b7d78a1d6ac056/ast_serialize-0.13.1-cp39-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:2876444a5f87a7c76f371af3c07c2fda563a08bc8af52568df812e8aba4bf568", size = 1570945, upload-time = "2026-10-17T00:04:05.778Z" }, + { url = "https://files.pythonhosted.org/packages/fb/8c/44b2da991cede79bfbf52089f33a5f0168945466b3d9cd38b5dcf08e7130/ast_serialize-0.13.1-cp39-abi3-musllinux_1_2_i686.whl", hash = "sha256:885a62ec6481ce58ab24c6357360d7a82023b62dd7e5e7ceccf59e9662a3272b", size = 1569978, upload-time = "2026-10-17T00:04:07.347Z" }, + { url = "https://files.pythonhosted.org/packages/46/e4/109b2a5beca700237767d5add920c2f8e1ba9bd5799b0faf143d0e9cf4ef/ast_serialize-0.13.1-cp39-abi3-musllinux_1_2_ppc64le.whl", hash = "sha256:2015fd0165e463aac3b1aadc6a1fa1e5820c14ca950821f4fe02f8a3f6a388d6", size = 1676082, upload-time = "2026-10-17T00:04:08.7Z" }, + { url = "https://files.pythonhosted.org/packages/73/e0/7df70f6dbb11b5762da5416318f08316a1c51348e0d69ec79ca5f98f836a/ast_serialize-0.13.1-cp39-abi3-musllinux_1_2_riscv64.whl", hash = "sha256:eb41e1906502ac80d24ff343f45008552a3f5b5158607959fe6a07dbfc6eb037", size = 1483207, upload-time = "2026-10-17T00:04:10.162Z" }, + { url = "https://files.pythonhosted.org/packages/1f/bc/523aa1c08bd69b5654621beb3dc1a9035715a4a63762fb8d219d6fb3eb81/ast_serialize-0.13.1-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:ca9acc67f796aa2554fa3037dccc022a4b08b38b2ad153edfac9c9bb7cb1f3c6", size = 1512140, upload-time = "2026-10-17T00:04:11.589Z" }, + { url = "https://files.pythonhosted.org/packages/66/83/6607060659998f00afdea0d8375c1e6e2b7d515ab905090d9a5a02f48851/ast_serialize-0.13.1-cp39-abi3-win32.whl", hash = "sha256:4f98da562e5901d69c7dd30885594c5438f859edf354e9ac7517d188fb026c3a", size = 1131889, upload-time = "2026-10-17T00:04:12.953Z" }, + { url = "https://files.pythonhosted.org/packages/9f/6f/bafd6490fe7b355627fabe95f4417e3d02da93102c8552550eb9a0774e1e/ast_serialize-0.13.1-cp39-abi3-win_amd64.whl", hash = "sha256:ddf7cbc9c8abf70102c4a088efe154bb86bfb825b0a10e556d2b15972cf99bd7", size = 1171153, upload-time = "2026-10-17T00:04:14.362Z" }, + { url = "https://files.pythonhosted.org/packages/11/25/4235f4b375a67a37ac595bf478a5407a744eaf7b64113c1c3a5432b8d954/ast_serialize-0.13.1-cp39-abi3-win_arm64.whl", hash = "sha256:48f11d96c9ec71119612f8e79a8c55c12f2b136716d68347433c561d27a25f43", size = 1139035, upload-time = "2026-10-17T00:04:15.767Z" }, +] [[package]] name = "bcrypt" @@ -355,6 +423,7 @@ otel = [ [package.dev-dependencies] dev = [ + { name = "mypy" }, { name = "opentelemetry-instrumentation" }, { name = "opentelemetry-sdk" }, { name = "pytest" }, @@ -366,6 +435,9 @@ test = [ { name = "pytest" }, { name = "pytest-cov" }, ] +types = [ + { name = "mypy" }, +] [package.metadata] requires-dist = [ @@ -381,6 +453,7 @@ provides-extras = ["otel"] [package.metadata.requires-dev] dev = [ + { name = "mypy", specifier = ">=2.4" }, { name = "opentelemetry-instrumentation", specifier = ">=0.42b0" }, { name = "opentelemetry-sdk", specifier = ">=1.12" }, { name = "pytest", specifier = ">=9.2" }, @@ -392,6 +465,7 @@ test = [ { name = "pytest", specifier = ">=9.2" }, { name = "pytest-cov", specifier = ">=7.1" }, ] +types = [{ name = "mypy", specifier = ">=2.4" }] [[package]] name = "iniconfig" @@ -411,6 +485,107 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/5a/de/bbc12563bbf979618d17625a4e753ff7a078523e28d870d3626daa97261a/invoke-3.0.3-py3-none-any.whl", hash = "sha256:f11327165e5cbb89b2ad1d88d3292b5113332c43b8553b494da435d6ec6f5053", size = 160958, upload-time = "2026-04-07T15:17:46.875Z" }, ] +[[package]] +name = "librt" +version = "0.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/04/f5/9dc696772d241814bacac7880bac32f2930b5a6ebc1f85317b83161a011c/librt-0.16.0.tar.gz", hash = "sha256:ac38d6d8d66bf3d744148dbbc0b8e193e195a51e364ed55e224631f5721891fc", size = 219839, upload-time = "2026-09-29T00:55:32.891Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ad/76/bbdaeb87b7c47b5c7343e90222b9bfa4e4a8a83f647e02933ad0129225b1/librt-0.16.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:fe52bf4641069e7978a14253b036cb9002def1926317e710f2e249f8a8c47742", size = 151028, upload-time = "2026-09-29T00:52:41.508Z" }, + { url = "https://files.pythonhosted.org/packages/fd/0c/ab8ed3dab0085931aec4a792c7eaac8dc6c5ff4691fda3a5360d9d8a9cd2/librt-0.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:5bcc2c4726ced915b00de0c9856a4eeabfb3fddb93e10e0b8f735b7709358b6d", size = 155283, upload-time = "2026-09-29T00:52:42.873Z" }, + { url = "https://files.pythonhosted.org/packages/eb/36/494e79d460c80c1f030661e8287c9eca5e1ad652dc2b2180b6cd42abce0a/librt-0.16.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ff7baa55f8e7c69851419e50a666015d02a74198716fd45c0125a2112e0a389f", size = 503185, upload-time = "2026-09-29T00:52:44.638Z" }, + { url = "https://files.pythonhosted.org/packages/9b/34/a8464038dd9db6e4381fa2b6eb73dc9a50888d77102c4c139304ac35cddc/librt-0.16.0-cp312-cp312-manylinux2014_i686.manylinux_2_17_i686.manylinux_2_28_i686.whl", hash = "sha256:b95d5d92ab83d39e760a52091bb1baba664f3a2351e39b1e16801e5747c2f0e9", size = 496738, upload-time = "2026-09-29T00:52:46.441Z" }, + { url = "https://files.pythonhosted.org/packages/6a/53/e0e5e334ef0c6ed27039d323819368b9ef6712be87d55ee2bf9799398afd/librt-0.16.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:b6d085d70bce51d43c5c7c36d63490770180d8779e71c49305c87b4213918de7", size = 513818, upload-time = "2026-09-29T00:52:48.068Z" }, + { url = "https://files.pythonhosted.org/packages/ad/f7/7ce72cbf19d0addd05090b339152fd0548a02562c2866a603e6e3b3da2df/librt-0.16.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:36e53948e99bbe3ffea257124cfcae1cfb01831555c9a9c903c9f9a72db7fd07", size = 531952, upload-time = "2026-09-29T00:52:49.816Z" }, + { url = "https://files.pythonhosted.org/packages/83/22/0b1bcb6a8e723c8b4fd60dfc8ae8ec6461c54073fbc8685efeb8d900d407/librt-0.16.0-cp312-cp312-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:54d11f726aae9df5a6ffbbf0a03a52449bbac84a53ef03669cb41cdfd4ae41bf", size = 524465, upload-time = "2026-09-29T00:52:51.354Z" }, + { url = "https://files.pythonhosted.org/packages/64/2e/e9c23b8b9df1813da1be205deca9606beb7ddd033972246cd426d05374a0/librt-0.16.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4323193ac0cd025f85af531df8ba91bf24d1973b401697347a6282e8fd3fcf5e", size = 543191, upload-time = "2026-09-29T00:52:53.284Z" }, + { url = "https://files.pythonhosted.org/packages/bc/e5/6a8b21b342c03ed7e230fa3afbfd2edc58e6e87ef1f0d11fa2b9a748c252/librt-0.16.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:e42f8e098b9c5396fefa05fb1cc7e33b0e08fc51da106b5de4a45fd22aac6743", size = 546979, upload-time = "2026-09-29T00:52:54.932Z" }, + { url = "https://files.pythonhosted.org/packages/84/9e/b5129023eced1be01e01c22757f53be551d463b1bb7264f787927404c1d6/librt-0.16.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:39ec1d5a14e37baf1450a6cabf03fe552340808bf1ad9d71824ab90117716459", size = 555317, upload-time = "2026-09-29T00:52:56.869Z" }, + { url = "https://files.pythonhosted.org/packages/71/89/28bba5938c725fe91f06bf93f7fa6c6b150229df53a87b454b0d5c2a796e/librt-0.16.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:d1aabe3925cbb4a08d15b7b20ba4011b53019da0c4173a25155139b7b1baed65", size = 535846, upload-time = "2026-09-29T00:52:58.475Z" }, + { url = "https://files.pythonhosted.org/packages/22/92/63773026614f888c5d4e370e395ce42ca604b89f70b3acdfedbf94851b80/librt-0.16.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:300c3ffdc459f4a779a8411ecb188e3ac0b1ff3a3a7b099642555dedae06c69b", size = 573700, upload-time = "2026-09-29T00:53:00.127Z" }, + { url = "https://files.pythonhosted.org/packages/66/8f/347d4677eefb57cd9f8e01d95a1dcee9a91b4e44e664173c32ff6f3752e5/librt-0.16.0-cp312-cp312-win32.whl", hash = "sha256:c17194318e4c0c0348b36f36c2ec7534436fe0a4c15582403162a4f08c80797a", size = 106158, upload-time = "2026-09-29T00:53:02.031Z" }, + { url = "https://files.pythonhosted.org/packages/f0/2c/5193dc81127cd5ddfad031391b046bf32dda219b38463ab872407ca30646/librt-0.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:25a58a19ea8d83b68209f04912df765e9260635ef77646542ed4b4abe6bc7940", size = 126176, upload-time = "2026-09-29T00:53:03.445Z" }, + { url = "https://files.pythonhosted.org/packages/ff/3d/9668a400c8dd81d162eba38b33fa49fa6205f1a64493570a13fbb815c3ee/librt-0.16.0-cp312-cp312-win_arm64.whl", hash = "sha256:f7be7cf555bc30ec12622e9447299cc4a9b8ff307548b634794353db0c2065dc", size = 116572, upload-time = "2026-09-29T00:53:04.815Z" }, + { url = "https://files.pythonhosted.org/packages/46/cd/ae5e0e9dba45d1399aa04a5395bcc0bead40d9fa06dc903634a7b4d7473d/librt-0.16.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:c5e6144e68b577f157519f2ba88ca20e3ed61c29b00e5cdfa76cd2d45acf059a", size = 151036, upload-time = "2026-09-29T00:53:06.284Z" }, + { url = "https://files.pythonhosted.org/packages/41/5a/48a16e323c5f9447a94cce7b59babf60fa04e62c3365ecf060c77ed8b320/librt-0.16.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:33f41443a1f4e1f099331b3d8120e409fbff84b9760bc1cc9ea496f37ddaa5cc", size = 155211, upload-time = "2026-09-29T00:53:07.72Z" }, + { url = "https://files.pythonhosted.org/packages/3f/29/0f59299eb4251a409b2e690ad4b7d9f8a676db829d7817ec961f32b44f7e/librt-0.16.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7e510b7770bee609617a3374a96548eb114cae048023e3f049ee449e7ff2db32", size = 502587, upload-time = "2026-09-29T00:53:09.191Z" }, + { url = "https://files.pythonhosted.org/packages/de/ba/d6fb4ef8d1537c396079d72289f16be7cd35a366e5065c51253fea2760b6/librt-0.16.0-cp313-cp313-manylinux2014_i686.manylinux_2_17_i686.manylinux_2_28_i686.whl", hash = "sha256:efc49c462d4516b8a58b00b490078fa64689fd1fe66970cc190131d7afb8027e", size = 496146, upload-time = "2026-09-29T00:53:11.016Z" }, + { url = "https://files.pythonhosted.org/packages/52/fc/8c50dd4d7cc97c0ee8f252c8a3104980f234391cf1519b554e8b9de08b60/librt-0.16.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:92caf82ebef5e12d21c72242b70d1e92536f1711cf2a727a4c276de4b4469087", size = 513336, upload-time = "2026-09-29T00:53:12.655Z" }, + { url = "https://files.pythonhosted.org/packages/a9/59/16c409c56f708eda2db9a0553662845d45e3871c77d70a240dae3f3bdc56/librt-0.16.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:17bac7f7a16b328fff77e440287693eb017abde913595b5827ebccbc21ecd8a6", size = 531689, upload-time = "2026-09-29T00:53:14.39Z" }, + { url = "https://files.pythonhosted.org/packages/88/82/d34772a6c29d1446dcca6e64d74062efd508523ab351aa16625a4689d5cc/librt-0.16.0-cp313-cp313-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5b976054553670829985ed767feb78fb6bcede0175327c4844dd5c281c1be659", size = 524537, upload-time = "2026-09-29T00:53:16.064Z" }, + { url = "https://files.pythonhosted.org/packages/77/8f/24c5631313746131ccee53bc91fdc8374f9cf25e0082a1fee9c93bb98acc/librt-0.16.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:0058f9d68721094105917254c72ac0569117bb7b13b9769cf45d26d89f9d21cd", size = 543231, upload-time = "2026-09-29T00:53:17.939Z" }, + { url = "https://files.pythonhosted.org/packages/f2/cb/5f8e0d41dbd8b499c2265e939c31acc9ba59845565bf99539ad1c06aebcf/librt-0.16.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:30b7beaf3f4487b7d8adef1f158b49067cb4d5a19fa7a3bf31a4e7a820e435c5", size = 546499, upload-time = "2026-09-29T00:53:19.666Z" }, + { url = "https://files.pythonhosted.org/packages/be/61/063052de441d1385f59cea4223f184bf9e5d125de1ae3239b490aa1e486e/librt-0.16.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:468df902df016a06eb0e40b0747dc8d14e47d7a38b18b63b1fb167d85cb94d63", size = 555261, upload-time = "2026-09-29T00:53:21.292Z" }, + { url = "https://files.pythonhosted.org/packages/14/11/a2ada0529372268d6401afa9d457a095b68cd7753532b6f7f33049a19b43/librt-0.16.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:aea7b1f2b125dad5de85f049136651bff256c883c65e6b9209b2da0a1ac3cdef", size = 536029, upload-time = "2026-09-29T00:53:23.07Z" }, + { url = "https://files.pythonhosted.org/packages/23/9d/5bb6d38853382986dca702fc7e06c8256d30f5fa0676d882773b744610dc/librt-0.16.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:a8afb6557920860b7a3a596eb804cf37e09e7cf8a803db2478c202acc72d8c2e", size = 573748, upload-time = "2026-09-29T00:53:24.696Z" }, + { url = "https://files.pythonhosted.org/packages/99/f6/0025cde35ff7f607684dc775a2b2d732cce2561cec050a6ee1fb2e1fc6fe/librt-0.16.0-cp313-cp313-pyemscripten_2025_0_wasm32.whl", hash = "sha256:77c7a2b4fe2c1369e0d5aa1cade26740a7b14be32fbc9a5535d617d20065c39d", size = 79861, upload-time = "2026-09-29T00:53:26.089Z" }, + { url = "https://files.pythonhosted.org/packages/93/93/303b8592909bd583f83f02818ffbea3f7647ca1e22b1cd5465d04b8145fe/librt-0.16.0-cp313-cp313-win32.whl", hash = "sha256:02d89c813d5ff74b17df72d3a34819d132cd168e56b81bf755b809bd9e46b8c4", size = 106236, upload-time = "2026-09-29T00:53:27.43Z" }, + { url = "https://files.pythonhosted.org/packages/cf/24/80bbb463c60ed18e29cb26aba386ddb76609580e4e7160710e550587018c/librt-0.16.0-cp313-cp313-win_amd64.whl", hash = "sha256:14ed6ebe3e4f85f326d7920011ad30ff49ed9334e62cf88caef9ba973d9e3a92", size = 126180, upload-time = "2026-09-29T00:53:28.7Z" }, + { url = "https://files.pythonhosted.org/packages/43/80/b1a6fbdd7da825cdd55c71aa81eb6cfa82c513c360774152eacb50b4a771/librt-0.16.0-cp313-cp313-win_arm64.whl", hash = "sha256:83d4041a3d9b2fd053a8a4e1f22878b3e5833e2712956382d5c048d791454e91", size = 116557, upload-time = "2026-09-29T00:53:30.012Z" }, + { url = "https://files.pythonhosted.org/packages/1e/93/9e0cf7da129a93c3dc7f45bc3cd4a660f2aaa995aa8a6f95c2583ef41239/librt-0.16.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:931a0bb0fcac88f263e269e46eb30ba8e21402cd3c62ca40cb97034c0693fab1", size = 149844, upload-time = "2026-09-29T00:53:31.391Z" }, + { url = "https://files.pythonhosted.org/packages/8f/26/8a90d2a8f2b2e471bb486b7aec117b8ae622715ed6c39853aec48ea20073/librt-0.16.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1bc17e54e5305f8d40b7ca203671ff5a9e59c1d0f8ea0f625dcca53a3984de11", size = 154111, upload-time = "2026-09-29T00:53:32.718Z" }, + { url = "https://files.pythonhosted.org/packages/35/ce/67abb46258da4d3e42ff5b141db6f38c59357bef84c7979f183b22f924f1/librt-0.16.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:877698bf6bca5721d8be345f2fe09778e40ecadea8b58c73075f2b1a53666bf2", size = 494234, upload-time = "2026-09-29T00:53:34.466Z" }, + { url = "https://files.pythonhosted.org/packages/12/f9/ea7162414a16f8f1bbd3b493ad6d22b926c5471916e078350bdbca8c4e5f/librt-0.16.0-cp314-cp314-manylinux2014_i686.manylinux_2_17_i686.manylinux_2_28_i686.whl", hash = "sha256:5981c011b306781ce561e18e14230a14524a3d8109b97553666c942c18f31a96", size = 491173, upload-time = "2026-09-29T00:53:36.124Z" }, + { url = "https://files.pythonhosted.org/packages/b1/09/9b3e869060dd33f9989b80ba4fea306f6db8cecefcdfe6d7346ef0603f65/librt-0.16.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:afced3dfc17cd805ecf7a3d77996a71cf5f2c75aa66eb0c21a9930f4fc992f86", size = 505547, upload-time = "2026-09-29T00:53:37.686Z" }, + { url = "https://files.pythonhosted.org/packages/50/07/79007d2165f649ea93e08c0962d1d62c70af9cde77965255095bf9d96f9a/librt-0.16.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ca8052401c55d7511dda6760719fda7618067e83535d7d0010096d216c34b667", size = 523057, upload-time = "2026-09-29T00:53:39.301Z" }, + { url = "https://files.pythonhosted.org/packages/61/0c/8fbaff66d0ba376d8864653f5acce1569bae27e89648671f26f7eca67ab8/librt-0.16.0-cp314-cp314-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:1e511762a074005bb0aa569166779834e75e438370226930d0ce1866d4b6a33b", size = 515158, upload-time = "2026-09-29T00:53:41.012Z" }, + { url = "https://files.pythonhosted.org/packages/df/2e/23ff0dece76f07a4124413a57682efa0bbeb5765ac0122bc0955513f82ca/librt-0.16.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:f1e8591bd8a5a628cd7f07954c6a1592359a878bf032957a8e9057a41d644311", size = 534183, upload-time = "2026-09-29T00:53:42.451Z" }, + { url = "https://files.pythonhosted.org/packages/26/c4/e11dea21d9a29486eba78887380374189d472734fa32cc50cb37ca44d3d0/librt-0.16.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:a4aaefb4ba6c07e1aeebb2795c8958148f1d6f9af3b555b53d23d766edb6d67a", size = 540663, upload-time = "2026-09-29T00:53:44.272Z" }, + { url = "https://files.pythonhosted.org/packages/44/75/e873ae158a8b7f5359be33e7fd6c1fbe02d9a78a3e89a77de6b0e837f476/librt-0.16.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:d92db7a0f6aee44f1baee94750457e8d2d1c6ccea41842de6268d34e8dc7eddd", size = 545908, upload-time = "2026-09-29T00:53:45.812Z" }, + { url = "https://files.pythonhosted.org/packages/ba/36/8939d3f6a93e11bd9592e6fe28d2b44f1c2dc4bed6e22e72359a91e18ffb/librt-0.16.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:378dfaffb38e59c24a87cde5713cd865d51ff7383fa12947f3907f306ea1ca55", size = 523302, upload-time = "2026-09-29T00:53:47.596Z" }, + { url = "https://files.pythonhosted.org/packages/71/14/35309f44a077f0f42ade0e2e7cd88c0cea760c661af205c01ea90d3c0e1f/librt-0.16.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:3e0c39bdc85370422e8b637be76eb1fd07d30967551b03e62267dd156f553152", size = 565911, upload-time = "2026-09-29T00:53:49.278Z" }, + { url = "https://files.pythonhosted.org/packages/78/0c/df6255b94967f3159ebc46f08d8e783c12da6ee269ddb74b2efed63c640c/librt-0.16.0-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:1b384b90ab79a7bc30b566895809a636e0666f21f3cf12b54823d025b7e83839", size = 79119, upload-time = "2026-09-29T00:53:50.938Z" }, + { url = "https://files.pythonhosted.org/packages/6b/44/d30d5a5461378c9d33f36736c6791c3b4c4ba4b1ffe0da7350aedcb2c9a0/librt-0.16.0-cp314-cp314-win32.whl", hash = "sha256:52327da75a94012e7f932f913d20d3876bed3c102be00e6c3e8600ff7bdd58a7", size = 100140, upload-time = "2026-09-29T00:53:52.205Z" }, + { url = "https://files.pythonhosted.org/packages/c2/98/769712f356a1e897df3581bb0c3100375d054a000de26099360ea65b5111/librt-0.16.0-cp314-cp314-win_amd64.whl", hash = "sha256:3f0b8114c44b2ac06ff5dacd08e07e8e807ff4f46083f2a1602685122559be41", size = 120464, upload-time = "2026-09-29T00:53:53.495Z" }, + { url = "https://files.pythonhosted.org/packages/bf/d3/ae2abccc8bdc8b063c1613a77686e17b74e9d3d60cfe6f12c63fa2821b9f/librt-0.16.0-cp314-cp314-win_arm64.whl", hash = "sha256:8caf96a4ef8fb27d0ac0d1ad8337d26a240acd4a02fe4345d0a8f264753e8f99", size = 110889, upload-time = "2026-09-29T00:53:54.817Z" }, + { url = "https://files.pythonhosted.org/packages/e4/26/0737d4be058dd6376eade7dd8b380d4b869b6394cee929393a5a431c45bb/librt-0.16.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:953107e2f68d0f3512c48f898b0dbf0ce5cc52bba0f318d847c985dc555ee4cc", size = 159838, upload-time = "2026-09-29T00:53:56.22Z" }, + { url = "https://files.pythonhosted.org/packages/01/96/9bc96531d7c620e9949af904470f02e3fa8f35129ab8e8f281c51eaa3788/librt-0.16.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:ad37d5b9abd49c9a655dcda7ea52a8a752884062ef1ee71ae17c2f2a0f81fe6a", size = 162018, upload-time = "2026-09-29T00:53:57.532Z" }, + { url = "https://files.pythonhosted.org/packages/0e/fa/b0289dcb186eb3f97221ba00da5f4bd3ba7fa5e99752d48aa8d336615334/librt-0.16.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2c4aa329c17bd1aaea4f6e89335d8ccd494b3a5830b6654462273e50e11023f0", size = 703983, upload-time = "2026-09-29T00:53:59.024Z" }, + { url = "https://files.pythonhosted.org/packages/d9/ab/05ebbbde7530fc5eeb58fbd1581522a9f64f132fa703c4c595eed6e14760/librt-0.16.0-cp314-cp314t-manylinux2014_i686.manylinux_2_17_i686.manylinux_2_28_i686.whl", hash = "sha256:0ead24d2562a49473dddd9efef8581f020007eb0054389c3ee3ffad38b1ca4c9", size = 683847, upload-time = "2026-09-29T00:54:00.671Z" }, + { url = "https://files.pythonhosted.org/packages/a3/75/f52aeecd4dbadbddf80725ba7de126d8bd5d0eb66247eae17a81ed90dd4d/librt-0.16.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:02118f56a9c36ddd07dfd9b919d9ecc117ba20a90987d56aa4c429fa34509188", size = 698045, upload-time = "2026-09-29T00:54:02.247Z" }, + { url = "https://files.pythonhosted.org/packages/e6/55/fa277a835cd6eb42380591ceb85f48c5b4d2b2d7e2cb9869e1e17d24d237/librt-0.16.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4e29522c62e28595ff7e324c6834ade51127707f0e255b18d1c1cf03d39c1048", size = 724640, upload-time = "2026-09-29T00:54:04.078Z" }, + { url = "https://files.pythonhosted.org/packages/25/e4/2cf64354f3fde8ebd591b3b48f96510bee24ac5f7d1e567adbbe210abdbd/librt-0.16.0-cp314-cp314t-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3ff4b2367926b69c6215635902cccb04048e73094e9862900d27cb2c6bbff143", size = 731748, upload-time = "2026-09-29T00:54:05.741Z" }, + { url = "https://files.pythonhosted.org/packages/f8/c9/c180af3e94e01aa529fa93d7733fec2abc47f222345400cf21d5481d5f8a/librt-0.16.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:c6f1b27bf1632a7e016af9f145f82be95e1edd7721a646505c21059257cb5a04", size = 754960, upload-time = "2026-09-29T00:54:07.38Z" }, + { url = "https://files.pythonhosted.org/packages/95/d6/01073aa78c58f356b10d9c57b3fe9abb143df338142bcd316df417b89db0/librt-0.16.0-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:5696d7f52e7b37217cb3a8f92c744fe835942602fdd4c1a8bc4741d3bfdce15e", size = 747000, upload-time = "2026-09-29T00:54:09.06Z" }, + { url = "https://files.pythonhosted.org/packages/f7/d8/1de3783908658d697a8cfc00582f61299ffba7796d7260c112e4700b1109/librt-0.16.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:6072e92dd876ff6ceeb6cf371e35e51f479349837391341f479b08df4564242b", size = 749569, upload-time = "2026-09-29T00:54:10.702Z" }, + { url = "https://files.pythonhosted.org/packages/b7/32/e817f66c96d6caa8bb8435ff93c4c220624d59efc506be59ab98fcd01d0c/librt-0.16.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:39ca4f2f2fe05de8e63493da592d84311adabe5bef52b193851981da9816b302", size = 729483, upload-time = "2026-09-29T00:54:12.25Z" }, + { url = "https://files.pythonhosted.org/packages/7c/c9/23992ccd2b9d22798fdd0f61353183a47414e651da782ad83eb680f50833/librt-0.16.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:f9807485a908f00355820f18e91e045ffdcdc5adb68aaec40a1e2b88c5f7bba1", size = 776584, upload-time = "2026-09-29T00:54:13.837Z" }, + { url = "https://files.pythonhosted.org/packages/67/3b/e8af957f08e6e2e8e566b099e43d2748418aa565df6ee1d9d3331209fdfe/librt-0.16.0-cp314-cp314t-win32.whl", hash = "sha256:94be5cb7bca4df6201f4183e9e4fa2086c655283d20b38cd84500a69057575a7", size = 104402, upload-time = "2026-09-29T00:54:15.568Z" }, + { url = "https://files.pythonhosted.org/packages/ca/1c/946e6443d7cd32347a086043395e421e52fd603c9163d2ec970ceab8eed6/librt-0.16.0-cp314-cp314t-win_amd64.whl", hash = "sha256:d46ca272b251d033dd4527b0dec5f261a28a52bd5fa0f99c117b0a1f8588cc2d", size = 126227, upload-time = "2026-09-29T00:54:17.165Z" }, + { url = "https://files.pythonhosted.org/packages/c2/a3/bc4f9959d3c62bcbf9e5fd3470a8bbd8bf33224ed4c25d7fba173201b8ac/librt-0.16.0-cp314-cp314t-win_arm64.whl", hash = "sha256:b9d6d4b14e92d876f8026b54c20c445f36425214c1081dc76f74e40db386b82b", size = 116493, upload-time = "2026-09-29T00:54:18.567Z" }, + { url = "https://files.pythonhosted.org/packages/b6/4b/10fdb42dfab4c1533e1570e686b18e86ff4328b406361b39fb3016667638/librt-0.16.0-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:6fe436af2eaf630474f491af5d032cbe45f93fcff5c3b9fe4ab194a7255b20ff", size = 149829, upload-time = "2026-09-29T00:54:19.933Z" }, + { url = "https://files.pythonhosted.org/packages/8e/30/a90ca13f1d3d91af1680000a4038536907018fbd51764767287a05d28b8b/librt-0.16.0-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:8ff5d26c529336be9bd7ae04483235d77778ee7d6444a95353102b542601ce81", size = 154522, upload-time = "2026-09-29T00:54:21.306Z" }, + { url = "https://files.pythonhosted.org/packages/5c/dd/bcf364eacfa070bb1fc88d503111ae7177914197ba1fa717c748926e7930/librt-0.16.0-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:909d8e3c1faee44cb762b1c519ff8613dcc5ceae5c99987a00917b5a31fd1d6a", size = 497593, upload-time = "2026-09-29T00:54:22.798Z" }, + { url = "https://files.pythonhosted.org/packages/18/c1/2c4e81e347bdabfe8346bfc6dc37ae5e154d61c88e30848ec0606025a5e0/librt-0.16.0-cp315-cp315-manylinux2014_i686.manylinux_2_17_i686.manylinux_2_28_i686.whl", hash = "sha256:6d4a64283ee61824b5790de882bc68e2d9d7a5143537cb7a966f7354f71646d4", size = 480459, upload-time = "2026-09-29T00:54:24.364Z" }, + { url = "https://files.pythonhosted.org/packages/ab/d7/fef2a3cb8400701be496f6e459876f650b3451f407e30cb243de571ae615/librt-0.16.0-cp315-cp315-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:5810ba811297fdf37a1531a57667cb8ace0842013ca8606bf9eb7c24cf4be154", size = 507814, upload-time = "2026-09-29T00:54:25.997Z" }, + { url = "https://files.pythonhosted.org/packages/e1/6f/53762927a32e9dc9eb1d1c1f3528da281290c86d96929a8af652f671266e/librt-0.16.0-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e56aaf8c167548dc8e5d6f3bd0f48dcdd299a23c73be3f744aab79d99e9c7f5d", size = 525114, upload-time = "2026-09-29T00:54:27.747Z" }, + { url = "https://files.pythonhosted.org/packages/6c/67/0b9d031f303c4e8c691a9a8ef9d272f13b530c11cc563b819df621b6a348/librt-0.16.0-cp315-cp315-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:8f36c58e33b304b525c6c9c5076399c6ebf1109e17b9051a05a407b091b9215b", size = 520465, upload-time = "2026-09-29T00:54:29.331Z" }, + { url = "https://files.pythonhosted.org/packages/ae/d5/2056a3a85864e882eb17a203a10ddb26fa748bc9718ec67e79059ab46cae/librt-0.16.0-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:242e00b3d4fa37c3d3c1ca5f5c9adb7d909ddb1eac9c41f2787320d00caa0af2", size = 537257, upload-time = "2026-09-29T00:54:30.915Z" }, + { url = "https://files.pythonhosted.org/packages/54/57/e0d79790c163cbc0909e209a6f62e30bb713b647f905a176cdc64848fd4d/librt-0.16.0-cp315-cp315-musllinux_1_2_i686.whl", hash = "sha256:0253721561787b8df8443eb347b7a6461015354e5bdd37ee38a41fef220d2bb0", size = 527442, upload-time = "2026-09-29T00:54:32.478Z" }, + { url = "https://files.pythonhosted.org/packages/aa/50/1c0c95aba7af51f4752ea34fbf2eb79b36e7cae3a72536248e8c735de735/librt-0.16.0-cp315-cp315-musllinux_1_2_ppc64le.whl", hash = "sha256:3e483a8d69ede8067db70c0e83007423b6925de6fd53afed01d66160f2e9398c", size = 548139, upload-time = "2026-09-29T00:54:34.183Z" }, + { url = "https://files.pythonhosted.org/packages/98/91/a8a43dd5138d4f55f88846b8f0c85454a4fad69952cebfcff9948f831290/librt-0.16.0-cp315-cp315-musllinux_1_2_riscv64.whl", hash = "sha256:69ba927445cfaaffb4081003ef5224c55a5c2ab67ef956f416ef744916e44121", size = 529708, upload-time = "2026-09-29T00:54:35.761Z" }, + { url = "https://files.pythonhosted.org/packages/2d/41/d5226881ab2b7c20d9d587b37bdd4a0ec8775a96d00ca87ac9f385587db4/librt-0.16.0-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:d6a365f2ab45a984d0e00eee0dd17f599ceab8cadab6ea07b6111c8132fc0e42", size = 567781, upload-time = "2026-09-29T00:54:37.409Z" }, + { url = "https://files.pythonhosted.org/packages/bb/bf/2345ba57a626e8c78c4ddcc724636a8df5a593fe46c1ee76bbf477e32b0d/librt-0.16.0-cp315-cp315-pyemscripten_2026_5_wasm32.whl", hash = "sha256:f01f3805f2dae4781c0c34b440e31740d082950bdaf89a6f601ad589a28af57a", size = 79177, upload-time = "2026-09-29T00:54:38.792Z" }, + { url = "https://files.pythonhosted.org/packages/1e/40/99e77936cc9207f629b077bf7cbc5e2f0827cddd7c29792f04ef881bd2c3/librt-0.16.0-cp315-cp315-win32.whl", hash = "sha256:b0e3e721c75d2e79a76d4422c79d7ba705fe1bbafec907037fe7a657a480a0e3", size = 100127, upload-time = "2026-09-29T00:54:40.251Z" }, + { url = "https://files.pythonhosted.org/packages/56/1e/801fe26bc622061b9dfd010e166d94142cb774b733217e6d98d1c0cf2638/librt-0.16.0-cp315-cp315-win_amd64.whl", hash = "sha256:bc02954b1295de798bbdb0b4e2d8a28c2117de8b5c73dcbeb27dc32572dfb971", size = 120480, upload-time = "2026-09-29T00:54:41.722Z" }, + { url = "https://files.pythonhosted.org/packages/bb/a9/d533983055bd36e112627384c2c038845d1df882540b8bcefb566475640b/librt-0.16.0-cp315-cp315-win_arm64.whl", hash = "sha256:c5db585d43449a5f54303d4b2774e45e1babd975cfe1630a3d708c0b80c3e560", size = 110915, upload-time = "2026-09-29T00:54:43.052Z" }, + { url = "https://files.pythonhosted.org/packages/55/fe/d62238fa9c653b0e0613290349467cb65711b5800e8e474f45e942a0ad95/librt-0.16.0-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:f06c689cb14afd9b612727553a5ec5a40febf113ca41c4413a2b0b334285884b", size = 159714, upload-time = "2026-09-29T00:54:44.497Z" }, + { url = "https://files.pythonhosted.org/packages/f3/ac/f31efe7818700be72ba4f9af8a80fa67c39808c8d26dacc55dc1f6f35172/librt-0.16.0-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:13b4e8aba90b0b1c82474e9844aa9ffe7ad3faa484350e1da64cb8188d903134", size = 162086, upload-time = "2026-09-29T00:54:45.968Z" }, + { url = "https://files.pythonhosted.org/packages/d9/16/4d7487bf86a9d7e8e18f37ba5538789233ea693637379b75701bd35ec9de/librt-0.16.0-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5a269c46ae327d8e6f8c1f85f7516cb52c0fa48127565a1105a4f4a05ff2a0b4", size = 712727, upload-time = "2026-09-29T00:54:47.486Z" }, + { url = "https://files.pythonhosted.org/packages/77/74/50cd550ccc1a517b9ed62347625c01ddf8c4afad66490312677c011458f6/librt-0.16.0-cp315-cp315t-manylinux2014_i686.manylinux_2_17_i686.manylinux_2_28_i686.whl", hash = "sha256:a33e0dae1f8592146a4764d54ce842b278732d21a84e17c3bbe6b1bc158a2248", size = 681106, upload-time = "2026-09-29T00:54:49.126Z" }, + { url = "https://files.pythonhosted.org/packages/df/5d/7293f712975ee6fdd2251411fd9ef1c62bc99c7b83ecbe62dbc999b1fab7/librt-0.16.0-cp315-cp315t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:47ada6ea32636492c61aa8ad27ae3b9404bfe7a97e3ba946d1984236cc741da0", size = 705921, upload-time = "2026-09-29T00:54:50.905Z" }, + { url = "https://files.pythonhosted.org/packages/67/f7/8aab946f11d59d1bece9ffc65d994b200c789a8ca5a95c17e17e609f912a/librt-0.16.0-cp315-cp315t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c43bd6e642d8a248c114327f98dd25ac5a7cb5aa168ef02f0559b91874df16b8", size = 731786, upload-time = "2026-09-29T00:54:52.584Z" }, + { url = "https://files.pythonhosted.org/packages/43/76/1c42ab31e7cb8384ebf6d3af607213c495222474f4940443ae7639ab7685/librt-0.16.0-cp315-cp315t-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c3d1bb7841a816ace6449bb26d3f9560dbfa20e71c568d23f0f62bf1e68f50b1", size = 744964, upload-time = "2026-09-29T00:54:54.217Z" }, + { url = "https://files.pythonhosted.org/packages/6f/2e/4b19982d933d2dfced671e840b219e4c1cd3f507df6e6dabb51bbd4e3850/librt-0.16.0-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:3931f7a3db322e7f44e02a280e3949326ce9579ad388ee8d691dc7c76da9fb70", size = 765305, upload-time = "2026-09-29T00:54:55.815Z" }, + { url = "https://files.pythonhosted.org/packages/b8/1b/e872583de2dcb3ac7746e7a2321aeeb168274f3952a87dc66e05ddb29faf/librt-0.16.0-cp315-cp315t-musllinux_1_2_i686.whl", hash = "sha256:f4462528b6000afe8f16907b5c7c2553abf1df005ba5140e6eb394541c3624c3", size = 744959, upload-time = "2026-09-29T00:54:57.532Z" }, + { url = "https://files.pythonhosted.org/packages/10/de/a18c6bcfb297af2674233e90b3c3661c3a0af0f1434a0c966d2ef708a835/librt-0.16.0-cp315-cp315t-musllinux_1_2_ppc64le.whl", hash = "sha256:80039ba9b6a7d5f1a0175a4cca6bbefead87bd854c80abad1cb30afe47a830db", size = 758138, upload-time = "2026-09-29T00:54:59.484Z" }, + { url = "https://files.pythonhosted.org/packages/22/1c/0df1d732539c297bb1a093e1fe3204d2faf76a9022cf4e1d1c2fce059790/librt-0.16.0-cp315-cp315t-musllinux_1_2_riscv64.whl", hash = "sha256:7a1d272724b581bb6bc769dfdafed6da2ecc9886ba2450311de55a4ac2e1e9cd", size = 744268, upload-time = "2026-09-29T00:55:01.179Z" }, + { url = "https://files.pythonhosted.org/packages/a2/f7/ccaf31331f20c91a5bd9bd48ffc3f743f9c81bb5720cc7b2a720ca204f0a/librt-0.16.0-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:fbe4fb8c5445f7496d7f7f6bb0807875d09d47e6771ffa175fb2df2895fb86ba", size = 785783, upload-time = "2026-09-29T00:55:02.938Z" }, + { url = "https://files.pythonhosted.org/packages/c1/ea/7421d9e6db894cd6cd788b604b3394a163ded6b942052b6f96c23ccf08e1/librt-0.16.0-cp315-cp315t-win32.whl", hash = "sha256:375bfe6b572a8f6cfc398709356046173bf27e64c4c5edaf5f7062f051fb4bf9", size = 104355, upload-time = "2026-09-29T00:55:04.533Z" }, + { url = "https://files.pythonhosted.org/packages/85/6d/7c31a506eb847bc58aeb2402d58e17605e821f68ab5df5e66fe274a6df41/librt-0.16.0-cp315-cp315t-win_amd64.whl", hash = "sha256:bd3150023d3dc2bc70f3784e59ffa1140d56ddba3d8125b3d6f9f85221279bfc", size = 126134, upload-time = "2026-09-29T00:55:06.017Z" }, + { url = "https://files.pythonhosted.org/packages/36/69/7a5d10ac409c4da0355e054a14371871da9b5557fcc42772cd00181c6cce/librt-0.16.0-cp315-cp315t-win_arm64.whl", hash = "sha256:8ceafb70f2a4f0826f11031942e59c0728fd98da112dc346d4352bde1e486866", size = 116391, upload-time = "2026-09-29T00:55:07.484Z" }, +] + [[package]] name = "lxml" version = "6.1.3" @@ -521,6 +696,69 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f8/b7/44edd7de434181c582892e68d1ffe6775ca403ce14aea07cb5a218a936cf/lxml-6.1.3-cp315-cp315t-win_arm64.whl", hash = "sha256:5a721a98c649855963811b59b55755b30566e7f7fc40bdc9803d66dee9f811cf", size = 3822836, upload-time = "2026-09-02T14:51:42.471Z" }, ] +[[package]] +name = "mypy" +version = "2.4.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "ast-serialize" }, + { name = "librt", marker = "platform_python_implementation != 'PyPy'" }, + { name = "mypy-extensions" }, + { name = "pathspec" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/4a/3c/78ecece71ee4b5445389a4ccd11627cfb9c69e4de554d11060171f376305/mypy-2.4.1.tar.gz", hash = "sha256:63b590965691e540a335fa74dadf31a7069142f6b72e234689bbd244edf55623", size = 4066728, upload-time = "2026-10-28T19:23:13.286Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c0/1a/0be9dc6af5e0e04128f18a5e36b4ca44bfb0085857d37aa6a7de1ac2b0cb/mypy-2.4.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d63737426a048a27705a4eedb828669f4bc0873612a1b46b0b9fe892b2ebaaaf", size = 14381321, upload-time = "2026-10-28T19:21:54.223Z" }, + { url = "https://files.pythonhosted.org/packages/0d/a6/7907e36711a8a8c703f961f076fab61793383694140bd1fd87589f3f68d4/mypy-2.4.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7344869cad05f073b8cae2826774fa0161e3eb2f5c685b76813574008d77d750", size = 14466373, upload-time = "2026-10-28T19:22:25.211Z" }, + { url = "https://files.pythonhosted.org/packages/5d/b8/7313212b29de570c25c9859d7c8e4ef2251a6d34338a1e20031d94297721/mypy-2.4.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a20ff6cd2af7a670c64f089bfa5fbcebe8ff3bf896c5e6ac7c9862c7960e4494", size = 15553666, upload-time = "2026-10-28T19:22:36.511Z" }, + { url = "https://files.pythonhosted.org/packages/23/8f/92966b6082ff49da9142e1e527eb9af4359a9a61632a1cfc313da96b37fe/mypy-2.4.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:9ed1195502cca32be5ef756b1693477f4991db2f2e34eaa90cc4ad6bea4f9531", size = 15802641, upload-time = "2026-10-28T19:21:13.421Z" }, + { url = "https://files.pythonhosted.org/packages/b6/b4/874f6fd54b005020ad63e2cf7a59b178d16a771d478849b29b842cb0cf14/mypy-2.4.1-cp312-cp312-win_amd64.whl", hash = "sha256:4c194c5d878130fca1ae662dfb1feb6274afbab0b40032c80147377e0ea803d5", size = 11383067, upload-time = "2026-10-28T19:22:03.516Z" }, + { url = "https://files.pythonhosted.org/packages/b0/41/5ae8e6528c2faa96e4a921a999d2c7ac7c565bee184b4282bf62ffa76b5d/mypy-2.4.1-cp312-cp312-win_arm64.whl", hash = "sha256:71eda06e5bae51976ce869fda429159c9cec678bd02f25bb197f0265c8f991be", size = 10946882, upload-time = "2026-10-28T19:21:39.51Z" }, + { url = "https://files.pythonhosted.org/packages/6a/65/cd19bc06b67d2df4828220245fd98569ba40fac00c60ba48bef1817d8ac3/mypy-2.4.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:3f1613bfe6726169a5680ec3dad5fe1f6d96285c12aded51ad432eadedd39d45", size = 14214179, upload-time = "2026-10-28T19:22:27.49Z" }, + { url = "https://files.pythonhosted.org/packages/c9/db/4d54ca4f49a0113d0c956980272aefdc6eec5458f7a72b350785b9ef2249/mypy-2.4.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:680ba5d9f0e3fe0c746b3d14df15f2a718e4bc308a1145616a1ea691ce7c354b", size = 14376217, upload-time = "2026-10-28T19:21:08.019Z" }, + { url = "https://files.pythonhosted.org/packages/a2/45/089b4724b85f0f408941be5298f817ee89181c1ce031a73d60ebb150c2c7/mypy-2.4.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1fb70621535c22fd4b1a0450d37ed1d6c5eb39656a1cebfb189084c12de54ba7", size = 15467623, upload-time = "2026-10-28T19:21:21.555Z" }, + { url = "https://files.pythonhosted.org/packages/27/4c/15edbc64b0b1412bfe94c74192b3b56554ab642b85a31bb06cef6313101f/mypy-2.4.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4af5ad558033bf71a44a1be149ed61633522824f84c1b6bc2445d3fe2d2a5296", size = 15729867, upload-time = "2026-10-28T19:21:37.557Z" }, + { url = "https://files.pythonhosted.org/packages/76/5b/7d160fb685994b65b0ca86d352f3afb5424f96a9e6914719f7b861fb6701/mypy-2.4.1-cp313-cp313-win_amd64.whl", hash = "sha256:acdd82cb3a2eb10720e87706a128d75785d005a8c47e36a4973221524f41fc56", size = 11390802, upload-time = "2026-10-28T19:22:18.081Z" }, + { url = "https://files.pythonhosted.org/packages/9a/ea/453d27f8d531f57c5f14aa73094eaa48e3fb77f680bdede2045cb64fa58b/mypy-2.4.1-cp313-cp313-win_arm64.whl", hash = "sha256:133886ef1a7a8cf673df55d865e41580c6c46aa9b8e31e1325dfe315c038f745", size = 10950427, upload-time = "2026-10-28T19:22:59.003Z" }, + { url = "https://files.pythonhosted.org/packages/7d/75/282d43d6fea7be984ac63e75ffd8eee146ba2f3b077bb1ca5bf3f973e0c4/mypy-2.4.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c45485061ba08f700acc12cfc8b177e6997e6bc353605babfc0abb081e079456", size = 14233251, upload-time = "2026-10-28T19:22:44.977Z" }, + { url = "https://files.pythonhosted.org/packages/ac/a8/9021263cc7e761451c7d8131bdfab3c0a5c2d67aa2d694670019e1ab86a7/mypy-2.4.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:727890a32f1858c09549eb55a8b234fe990d460450b28f03e8213ef925f1247c", size = 14394953, upload-time = "2026-10-28T19:21:29.492Z" }, + { url = "https://files.pythonhosted.org/packages/16/10/4192a5cea09ebbc41c534f57bd0d8f0fb42a8e810632cda6e37f072c59fd/mypy-2.4.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:beb254fec368c4e7f7f82a5f310c6470b893ec88a90b66426da971c6826b0453", size = 15442824, upload-time = "2026-10-28T19:22:42.068Z" }, + { url = "https://files.pythonhosted.org/packages/99/e6/d3b0a7474aa4f696594a47a218b6796d9c4346fc81ed288904278cddd9bd/mypy-2.4.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ec416d5d2ae689e681f5727e328723f18d8b5dc8c903a146b7851dc5727aad8a", size = 15717227, upload-time = "2026-10-28T19:22:54.026Z" }, + { url = "https://files.pythonhosted.org/packages/48/6a/2ec530dacd4f77ca50d0fa05b854098f2d75e5d18c587d2612b44f10d68c/mypy-2.4.1-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:0e95ad0bec03ced9a73b42514208abd24c789e67f0e1eba476e276dd7a2dea3a", size = 7898637, upload-time = "2026-10-28T19:21:41.709Z" }, + { url = "https://files.pythonhosted.org/packages/cc/56/de9d7bad12e3868d5f1f4279aed20b1b568b2c5e166f2d3a05b211dca0b1/mypy-2.4.1-cp314-cp314-win_amd64.whl", hash = "sha256:51f742d4154824837ca6e92aa81f00eec222700a3c337bd258e5562a3880d75a", size = 11576656, upload-time = "2026-10-28T19:22:13.232Z" }, + { url = "https://files.pythonhosted.org/packages/b8/12/4fa9883b9b59e5c96ff62845ad019dc47786031c45d67eaccf452cffffd8/mypy-2.4.1-cp314-cp314-win_arm64.whl", hash = "sha256:f44dd4c5ab22df21206dd926c98381ebdbf5bfd696df37dd38ded2d23143fcb6", size = 11174403, upload-time = "2026-10-28T19:23:01.346Z" }, + { url = "https://files.pythonhosted.org/packages/3f/f2/827e356fc4f74c60b8da42663336e035b49d948a3c11224b0fc7543086b1/mypy-2.4.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:2875280687e64740e6ddcef43bd4c5852ad45ff34549740f6aea3f0373870b93", size = 15713427, upload-time = "2026-10-28T19:21:26.757Z" }, + { url = "https://files.pythonhosted.org/packages/a2/fa/e58a4fcae1d238189eac758036c0921992a86e3e6209190efd96d61fc5d4/mypy-2.4.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ea0714d2bccaf6d010c970e1d3ba2c9c2add79d2030aad2ef03167d788948276", size = 15938481, upload-time = "2026-10-28T19:22:34.371Z" }, + { url = "https://files.pythonhosted.org/packages/a1/54/a20f881801b88900923f2be1541a4b438ad0f0883e03836641fdc3d7adf4/mypy-2.4.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:63c6ff0e7ea546943001a91639c9f0c64b60d239268fa5d29e4a6c87f82c2373", size = 17243006, upload-time = "2026-10-28T19:22:39.425Z" }, + { url = "https://files.pythonhosted.org/packages/de/1c/ff84a6133a6e89c390bff6b25d9bad6598e7891b205f4b30b496906d636d/mypy-2.4.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:a5b5da107dc2c93fb0a22f7a45dbaea5dc131afc7056b5468d78e819f9a138e6", size = 17557728, upload-time = "2026-10-28T19:21:43.607Z" }, + { url = "https://files.pythonhosted.org/packages/15/4a/effa47d39b8fadf679c08e7c78559e523f9769a42a75662d2d15b2b9cdd6/mypy-2.4.1-cp314-cp314t-win_amd64.whl", hash = "sha256:c3a4d0c47053cc90035dd3a2c321112b593ad42935d0cbfd747288e1564fa8ca", size = 12370826, upload-time = "2026-10-28T19:22:29.779Z" }, + { url = "https://files.pythonhosted.org/packages/46/9f/b214ececb1a97fbb61ab3dfd48ae3cffd92ae2c148af4c9dc94c6c56772b/mypy-2.4.1-cp314-cp314t-win_arm64.whl", hash = "sha256:0c5bf267645351887b72903fd2f5fbddcf85234e0e6c4ad94e2fcfb4f0c3f971", size = 11851083, upload-time = "2026-10-28T19:23:11.288Z" }, + { url = "https://files.pythonhosted.org/packages/57/5b/e44a2edb0b25a9f1429de770e1d0238af23302671dd40c77ac5efeb0c559/mypy-2.4.1-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:db599d03e7d60d15f2e0a11665c18bc52a88699fcab990fdf8cecd5be8ac801c", size = 14243144, upload-time = "2026-10-28T19:21:10.975Z" }, + { url = "https://files.pythonhosted.org/packages/59/d4/5c99dc3a28e11d2f0fcbb02df63d2d3e574dedc0f9f69d50901df305a7a6/mypy-2.4.1-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c6c7210e40dd423af7d65cbe428b5906d11dc60659337bdae4e5fdf02ce32ed7", size = 14409316, upload-time = "2026-10-28T19:22:15.515Z" }, + { url = "https://files.pythonhosted.org/packages/c3/c9/3108b38f4c30eabd80194f5d9771a87e27196c5e0ddb5bf4b99c6ed1e97e/mypy-2.4.1-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a2d4a51db92059eecc2c33b2a5194b28585ebfd917292c015fe87a77dbff6cd2", size = 15445573, upload-time = "2026-10-28T19:22:05.635Z" }, + { url = "https://files.pythonhosted.org/packages/dd/12/84a53eceaf11c46dfd7931fb6b8902d01fb33787d3a991e81c461458150b/mypy-2.4.1-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:01b7718552d017f88d1151f3cc9d6556d778a90bd58060a2c3d87d070f9099fb", size = 15732989, upload-time = "2026-10-28T19:23:06.512Z" }, + { url = "https://files.pythonhosted.org/packages/65/ba/bf017dc2abdf83b2f74e29c43834b7af67dc1f1b20202f02dae60b4781a0/mypy-2.4.1-cp315-cp315-pyemscripten_2026_5_wasm32.whl", hash = "sha256:3467b1d13000812ab85dc2a3834b9379ec77798ca1bcabe434e23fb21fead692", size = 7898932, upload-time = "2026-10-28T19:21:59.366Z" }, + { url = "https://files.pythonhosted.org/packages/76/05/0c22b1f0390120a8592d31209cb6b9f932b6a0b426fe643e550de15139e3/mypy-2.4.1-cp315-cp315-win_amd64.whl", hash = "sha256:080cf64a60b90a73435542feecf4171904cc5710658165b40c6869115d4aa100", size = 11577577, upload-time = "2026-10-28T19:22:01.302Z" }, + { url = "https://files.pythonhosted.org/packages/73/fb/ac01e173629697ddac2f11e51efd10347d673f9a7311b03f4aba710ebccd/mypy-2.4.1-cp315-cp315-win_arm64.whl", hash = "sha256:f9118a3f50de264b614861bb11814833a742138ac3c8e2e49a860b6977fb2704", size = 11173540, upload-time = "2026-10-28T19:21:46.444Z" }, + { url = "https://files.pythonhosted.org/packages/5b/e4/b9b9c8eb09e70927ee65b1456cfed35710b6e4ff0b07f364ceaf2202307c/mypy-2.4.1-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:9d7dafd56179d296651df9e1844b91c9517c379398f017a2a435d077a28943c1", size = 15706623, upload-time = "2026-10-28T19:22:20.528Z" }, + { url = "https://files.pythonhosted.org/packages/26/74/1ceb1ca884add0565e618d013dd24333039100dc0fcc42661561f2e1cae5/mypy-2.4.1-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d923cec43316f0e6736b8298fd00abeaaf348a091c20e7296b3aebceecb63eca", size = 15944095, upload-time = "2026-10-28T19:21:04.779Z" }, + { url = "https://files.pythonhosted.org/packages/2b/68/27bcf9b4efa22733b6050485ae37845b94c9b8851abe4fc60632d905c32d/mypy-2.4.1-cp315-cp315t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d6e081b466e88d4343adffe11aff0f02df34b5b9070b4b700053a0b7294c0be1", size = 17223065, upload-time = "2026-10-28T19:23:03.724Z" }, + { url = "https://files.pythonhosted.org/packages/1a/a3/ed5724b317bf585c29f3c8be2792e97d45925d65a9c8af40ceeb267072e6/mypy-2.4.1-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:70196470ac300f444a302a738541101893c2c57804078b33690286a7899149e9", size = 17560589, upload-time = "2026-10-28T19:21:34.539Z" }, + { url = "https://files.pythonhosted.org/packages/42/2f/005185e6ab4752fd6cc2d1a6d1421fe21979770afe2490c00c555f34a913/mypy-2.4.1-cp315-cp315t-win_amd64.whl", hash = "sha256:de95648e0a6d46fa9929b956ed2c87c5dc9933d4d59945e6390e9d753c74e7c1", size = 12353324, upload-time = "2026-10-28T19:22:51.481Z" }, + { url = "https://files.pythonhosted.org/packages/a1/d6/20420cc9c780087139662534d128c2f3274361f8bb7e9cf0d5495a846ed5/mypy-2.4.1-cp315-cp315t-win_arm64.whl", hash = "sha256:ef8d5be90032261197ac388d945d8ff4f476c95880798a1911760af64ddfe71f", size = 11845599, upload-time = "2026-10-28T19:22:23.136Z" }, + { url = "https://files.pythonhosted.org/packages/21/5c/6e8010f8f2c4ad817364592d4b3883633417269b913fc9d5174fdeb7e565/mypy-2.4.1-py3-none-any.whl", hash = "sha256:1784648d6ec8c2b5465e3f26241b845a0b3cb4e56c993c64abe16989e7698999", size = 2801286, upload-time = "2026-10-28T19:21:19.516Z" }, +] + +[[package]] +name = "mypy-extensions" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a2/6e/371856a3fb9d31ca8dac321cda606860fa4548858c0cc45d9d1d4ca2628b/mypy_extensions-1.1.0.tar.gz", hash = "sha256:52e68efc3284861e772bbcd66823fde5ae21fd2fdb51c62a211403730b916558", size = 6343, upload-time = "2025-04-22T14:54:24.164Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505", size = 4963, upload-time = "2025-04-22T14:54:22.983Z" }, +] + [[package]] name = "ncclient" version = "0.7.1" @@ -612,6 +850,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/13/45/fa0e8d70cb25d2dae06bc238e580a84834f6ca53a4e2408cccc19a78db7b/paramiko-5.1.0-py3-none-any.whl", hash = "sha256:d9fe6d67a9d0bbf3eb85b051d26ab850b678c02d060f32915b944e5ca6d0d9e1", size = 211623, upload-time = "2026-10-18T02:38:49.903Z" }, ] +[[package]] +name = "pathspec" +version = "1.1.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5a/82/42f767fc1c1143d6fd36efb827202a2d997a375e160a71eb2888a925aac1/pathspec-1.1.1.tar.gz", hash = "sha256:17db5ecd524104a120e173814c90367a96a98d07c45b2e10c2f3919fff91bf5a", size = 135180, upload-time = "2026-04-27T01:46:08.907Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f1/d9/7fb5aa316bc299258e68c73ba3bddbc499654a07f151cba08f6153988714/pathspec-1.1.1-py3-none-any.whl", hash = "sha256:a00ce642f577bf7f473932318056212bc4f8bfdf53128c78bbd5af0b9b20b189", size = 57328, upload-time = "2026-04-27T01:46:07.06Z" }, +] + [[package]] name = "pluggy" version = "1.7.0" From 86d7cf6a9142cd8022c38b05146c6f6a98118fb3 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Fri, 30 Oct 2026 23:40:27 +0100 Subject: [PATCH 010/250] Use the MIT license header in the exscript CLI The package is MIT licensed; the script still carried the GPL version 2 header from 2007-2010. --- scripts/exscript | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/scripts/exscript b/scripts/exscript index b389bd70..7366c715 100755 --- a/scripts/exscript +++ b/scripts/exscript @@ -1,20 +1,27 @@ #!/usr/bin/env python3 """TODO: Add missing doctring.""" -# Copyright (C) 2007-2010 Samuel Abels. +# Copyright (C) 2007-2010 Samuel Abels +# The MIT License (MIT) # -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2, as -# published by the Free Software Foundation. +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files +# (the "Software"), to deal in the Software without restriction, +# including without limitation the rights to use, copy, modify, merge, +# publish, distribute, sublicense, and/or sell copies of the Software, +# and to permit persons to whom the Software is furnished to do so, +# subject to the following conditions: # -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. # -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. import argparse import getpass import importlib.util From 25b38c66cb94509fad4dbff499f8135424ffd1dc Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Sat, 31 Oct 2026 10:26:32 +0100 Subject: [PATCH 011/250] Add record/replay tests with self-collected recordings tools/record_session.py records Telnet, SSH and NETCONF sessions without Exscript and never stores passwords; tools/sanitize_recording.py removes addresses and secrets. ReplayTest replays the recordings over real sockets and compares the results with golden files. --- .gitattributes | 3 + tests/Exscript/protocols/ReplayTest.py | 175 ++++++ tests/Exscript/protocols/replay.py | 499 ++++++++++++++++++ tests/recordings/README.md | 108 ++++ tests/recordings/commands/busybox-ash.txt | 7 + tests/recordings/commands/frr-vtysh.txt | 15 + tests/recordings/commands/ios-basic.txt | 28 + tests/recordings/commands/ios-pager.txt | 6 + tests/recordings/commands/ios_xr-basic.txt | 23 + tests/recordings/commands/junos-basic.txt | 19 + tests/recordings/commands/junos-pager.txt | 6 + tests/recordings/commands/linux-bash.txt | 9 + tests/recordings/commands/netconf-junos.xml | 11 + tests/recordings/commands/netconf.xml | 9 + tests/recordings/commands/netopeer2.xml | 11 + tests/recordings/frr/frr-10.4-vtysh-ssh.jsonl | 31 ++ .../alpine-3.22-bash-ssh-vt100.golden.json | 6 + .../linux/alpine-3.22-bash-ssh-vt100.jsonl | 19 + .../linux/alpine-3.22-bash-ssh.golden.json | 71 +++ .../linux/alpine-3.22-bash-ssh.jsonl | 19 + .../linux/busybox-1.37-ash-telnet.golden.json | 6 + .../linux/busybox-1.37-ash-telnet.jsonl | 25 + .../netopeer2/netopeer2-netconf.golden.json | 55 ++ .../netopeer2/netopeer2-netconf.jsonl | 17 + tools/record_session.py | 390 ++++++++++++++ tools/sanitize_recording.py | 284 ++++++++++ 26 files changed, 1852 insertions(+) create mode 100644 tests/Exscript/protocols/ReplayTest.py create mode 100644 tests/Exscript/protocols/replay.py create mode 100644 tests/recordings/README.md create mode 100644 tests/recordings/commands/busybox-ash.txt create mode 100644 tests/recordings/commands/frr-vtysh.txt create mode 100644 tests/recordings/commands/ios-basic.txt create mode 100644 tests/recordings/commands/ios-pager.txt create mode 100644 tests/recordings/commands/ios_xr-basic.txt create mode 100644 tests/recordings/commands/junos-basic.txt create mode 100644 tests/recordings/commands/junos-pager.txt create mode 100644 tests/recordings/commands/linux-bash.txt create mode 100644 tests/recordings/commands/netconf-junos.xml create mode 100644 tests/recordings/commands/netconf.xml create mode 100644 tests/recordings/commands/netopeer2.xml create mode 100644 tests/recordings/frr/frr-10.4-vtysh-ssh.jsonl create mode 100644 tests/recordings/linux/alpine-3.22-bash-ssh-vt100.golden.json create mode 100644 tests/recordings/linux/alpine-3.22-bash-ssh-vt100.jsonl create mode 100644 tests/recordings/linux/alpine-3.22-bash-ssh.golden.json create mode 100644 tests/recordings/linux/alpine-3.22-bash-ssh.jsonl create mode 100644 tests/recordings/linux/busybox-1.37-ash-telnet.golden.json create mode 100644 tests/recordings/linux/busybox-1.37-ash-telnet.jsonl create mode 100644 tests/recordings/netopeer2/netopeer2-netconf.golden.json create mode 100644 tests/recordings/netopeer2/netopeer2-netconf.jsonl create mode 100755 tools/record_session.py create mode 100755 tools/sanitize_recording.py diff --git a/.gitattributes b/.gitattributes index a2282ca7..197bb161 100644 --- a/.gitattributes +++ b/.gitattributes @@ -3,3 +3,6 @@ # Device captures and expected outputs must stay byte-identical (CRLF is intentional). tests/Exscript/protocols/banners/** -text tests/templates/** -text +# Recorded sessions and their expected results are generated; collapse them in diffs. +tests/recordings/*/*.jsonl linguist-generated +tests/recordings/*/*.golden.json linguist-generated diff --git a/tests/Exscript/protocols/ReplayTest.py b/tests/Exscript/protocols/ReplayTest.py new file mode 100644 index 00000000..eb55591d --- /dev/null +++ b/tests/Exscript/protocols/ReplayTest.py @@ -0,0 +1,175 @@ +""" +Replays the recorded sessions in tests/recordings and compares the results with +golden files. Set EXSCRIPT_UPDATE_GOLDEN=1 to rewrite the golden files after an +intended change, and review the diff. +""" + +import glob +import hashlib +import json +import os +import re +import unittest + +from Exscript import Account +from Exscript.protocols import SSH2, Netconf, Telnet +from Exscript.protocols.exception import TimeoutException + +from .replay import CHUNK_MODES, Recording, ReplayServer, operation, rpc_body + +RECORDINGS = os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, 'recordings') +UPDATE_GOLDEN = os.environ.get('EXSCRIPT_UPDATE_GOLDEN') == '1' +ADAPTERS = {'ssh': SSH2, 'telnet': Telnet, 'netconf': Netconf} +# Recordings that Exscript does not handle like the recorded device yet, and why. +KNOWN_ISSUES = { + 'linux/alpine-3.22-bash-ssh': 'bash prompts are not guessed as shell', + 'linux/alpine-3.22-bash-ssh-vt100': 'escape sequences before the prompt make login() time out', + 'linux/busybox-1.37-ash-telnet': '"router3 login:" is guessed as junos, whose prompt does not match', + 'netopeer2/netopeer2-netconf': 'the unanchored isam prompt regex matches NETCONF replies', +} +# Recordings whose results depend on read timing until the issue is fixed; their tests are skipped. +TIMING_ISSUES = { + 'frr/frr-10.4-vtysh-ssh': 'route codes like "S>" at the end of a read match the prompt', +} +_message_id = re.compile(r'message-id="[^"]*"') +_results = {} + + +def summarize(command, response, error=None): + """Condenses a command and its response for the golden file.""" + response = _message_id.sub('message-id="*"', response or '') + lines = response.splitlines() + return { + 'command': command[:120], + 'error': error, + 'lines': len(lines), + 'sha256': hashlib.sha256(response.encode('utf-8')).hexdigest(), + 'first': lines[0][:120] if lines else '', + 'last': lines[-1][:120] if lines else '', + } + + +def run_cli(conn, session, account): + """Runs the recorded commands that login() did not send.""" + sends = [ + e for e in session.remaining() if e['dir'] == 'send' and e.get('phase') != 'negotiation' + ] + results = [] + for index, event in enumerate(sends): + if event.get('secret'): + continue # sent by app_authorize() below + command = event['data'].rstrip('\r\n') + try: + if index + 1 < len(sends) and sends[index + 1].get('secret'): + conn.send(event['data']) + conn.app_authorize(account) + elif command != event['data']: + conn.execute(command) + else: # keys without a line end, e.g. for a pager + conn.send(command) + continue + except Exception as exc: + results.append(summarize(command, conn.response, type(exc).__name__)) + if isinstance(exc, (TimeoutException, EOFError)): + break + continue + results.append(summarize(command, conn.response)) + return results + + +def run_netconf(conn, session): + """Runs the recorded RPCs; close() sends close-session.""" + results = [] + for rpc, _, _ in session.remaining(): + if operation(rpc) == b'close-session': + continue + body = rpc_body(rpc).decode('utf-8') + try: + conn.execute(body) + except Exception as exc: + results.append(summarize(body, conn.response, type(exc).__name__)) + if isinstance(exc, (TimeoutException, EOFError)): + break + continue + results.append(summarize(body, conn.response)) + return results + + +def replay(path, chunks): + """Logs in to a replayed recording, runs its commands, and returns a summary and server errors.""" + if (path, chunks) in _results: + return _results[path, chunks] + recording = Recording(path) + options = {'timeout': 2, 'connect_timeout': 2, 'connect_rate_limit': 0} + if recording.header.get('terminal'): + options['termtype'] = recording.header['terminal']['type'] + if recording.protocol != 'telnet': + methods = (recording.header.get('ssh') or {}).get('auth_methods') or [] + options['verify_fingerprint'] = False + options['preferred_authentications'] = [ + method for method in ('keyboard-interactive', 'password') if method in methods + ] or ['password'] + account = Account(recording.login_name() or 'user', 'password', 'secret') + conn = ADAPTERS[recording.protocol](**options) + result = {'login': 'ok', 'driver': None, 'os': None, 'commands': []} + with ReplayServer(recording, chunks) as server: + try: + conn.connect('127.0.0.1', server.port) + conn.login(account) + except Exception as exc: + result['login'] = type(exc).__name__ + else: + if recording.protocol == 'netconf': + result['commands'] = run_netconf(conn, server.sessions[-1]) + else: + result['commands'] = run_cli(conn, server.sessions[-1], account) + result['driver'] = conn.get_driver().name + result['os'] = conn.guess_os() + conn.close(force=recording.protocol != 'netconf' or result['login'] != 'ok') + _results[path, chunks] = result, [repr(error) for error in server.errors] + return _results[path, chunks] + + +class ReplayTest(unittest.TestCase): + """The test methods are generated per recording below.""" + + maxDiff = None + + +def _truth_test(path): + def test(self): + result = replay(path, 'recorded')[0] + self.assertEqual('ok', result['login']) + expected_os = Recording(path).header['device'].get('os') + if expected_os: + self.assertEqual(expected_os, result['os']) + + return test + + +def _golden_test(path, chunks): + def test(self): + result, server_errors = replay(path, chunks) + self.assertEqual([], server_errors) + golden = path.removesuffix('.jsonl') + '.golden.json' + if UPDATE_GOLDEN and chunks == 'recorded': + with open(golden, 'w', encoding='ascii', newline='\n') as fp: + fp.write(json.dumps(result, indent=2) + '\n') + if not os.path.exists(golden): + self.skipTest('no golden file; run with EXSCRIPT_UPDATE_GOLDEN=1') + with open(golden, encoding='ascii') as fp: + self.assertEqual(json.load(fp), result) + + return test + + +for _path in sorted(glob.glob(os.path.join(RECORDINGS, '*', '*.jsonl'))): + _key = os.path.relpath(_path, RECORDINGS).replace(os.sep, '/').removesuffix('.jsonl') + _name = re.sub(r'\W', '_', _key) + _tests = {'truth': _truth_test(_path)} | {c: _golden_test(_path, c) for c in CHUNK_MODES} + for _suffix, _test in _tests.items(): + if _key in TIMING_ISSUES: + _test = unittest.skip(TIMING_ISSUES[_key])(_test) + elif _suffix == 'truth' and _key in KNOWN_ISSUES: + _test = unittest.expectedFailure(_test) + setattr(ReplayTest, f'test_{_name}_{_suffix}', _test) diff --git a/tests/Exscript/protocols/replay.py b/tests/Exscript/protocols/replay.py new file mode 100644 index 00000000..1e964884 --- /dev/null +++ b/tests/Exscript/protocols/replay.py @@ -0,0 +1,499 @@ +""" +Replays recorded device sessions for the protocol tests. + +See ``tests/recordings/README.md`` for the recording format. The server sends +the recorded device output and waits wherever the client sent something during +the recording: + +- a client line that matches the recording continues the replay; +- a recorded secret matches any line; +- any other line is echoed and answered with the current prompt. + +Each connection replays the recording from the start, because some drivers +reconnect between SSH authentication methods. NETCONF replies are matched by +operation name and get the message-id of the client's request. +""" + +import contextlib +import json +import random +import re +import socket +import threading +import time +from xml.sax.saxutils import quoteattr + +import paramiko + +EOM = b']]>]]>' +NETCONF_BASE_11 = b'urn:ietf:params:netconf:base:1.1' +IAC, SB, SE = 255, 250, 240 +CHUNK_MODES = ('recorded', 'small') + +_frame_header = re.compile(rb'\n#(#|\d+)\n') +_rpc_tag = re.compile(rb'<(?:[\w.-]+:)?rpc\b[^>]*>') +_reply_tag = re.compile(rb'<(?:[\w.-]+:)?rpc-reply\b[^>]*>') +_operation = re.compile(rb'<(?:[\w.-]+:)?rpc\b[^>]*>\s*<(?:[\w.-]+:)?([\w.-]+)') +_message_id = re.compile(rb'\bmessage-id\s*=\s*(["\'])(.*?)\1') +_ok_reply = b'' +_error_reply = ( + b'' + b'applicationoperation-not-supported' + b'errornot in the recording' + b'' +) + + +def data(event): + """Returns the bytes of an event.""" + return event['data'].encode('latin-1') + + +def operation(message): + """Returns the operation name of a NETCONF RPC, e.g. b'get-config'.""" + match = _operation.search(message) + return match.group(1) if match else None + + +def rpc_body(message): + """Returns the content of the element of a NETCONF RPC.""" + start = _rpc_tag.search(message).end() + return message[start : message.rindex(b' len(stream): + break + message += stream[match.end() : end] + sizes.append(end - match.end()) + pos = end + return messages, stream[start:] + + +def encode_message(message, chunked, sizes=()): + """Frames a NETCONF message, with the given chunk sizes if chunked (last chunk takes the rest).""" + if not chunked: + return message + EOM + chunks, pos = [], 0 + for size in sizes[:-1]: + if pos + size >= len(message): + break + chunks.append(message[pos : pos + size]) + pos += size + chunks.append(message[pos:]) + return b''.join(b'\n#%d\n%s' % (len(chunk), chunk) for chunk in chunks) + b'\n##\n' + + +class Recording: + """A recorded session: the header dict and the list of event dicts.""" + + def __init__(self, path): + with open(path, encoding='utf-8') as fp: + items = [json.loads(line) for line in fp if line.strip()] + header = items[0] + if header.get('format') != 'exscript-recording' or header.get('version') != 1: + raise ValueError(f'{path}: not an exscript recording of version 1') + self.path = path + self.header = header + self.events = items[1:] + self.protocol = header['protocol'] + + def login_name(self): + """Returns the user name that was sent at a login prompt, if any.""" + for event in self.events: + if event['dir'] == 'send' and event.get('phase') == 'login' and not event.get('secret'): + return event['data'].rstrip('\r\n') + return None + + def netconf_exchange(self): + """Returns the server hello and the (rpc, reply, reply chunk sizes) tuples.""" + received = b''.join(data(e) for e in self.events if e['dir'] == 'recv') + sent = b''.join(data(e) for e in self.events if e['dir'] == 'send') + hello, received = received.split(EOM, 1) + client_hello, sent = sent.split(EOM, 1) + chunked = NETCONF_BASE_11 in hello and NETCONF_BASE_11 in client_hello + rpcs = decode_messages(sent, chunked)[0] + replies = decode_messages(received, chunked)[0] + return hello, [(rpc, reply, sizes) for (rpc, _), (reply, sizes) in zip(rpcs, replies)] + + +class ReplayServer: + """Serves a recording on 127.0.0.1 at a free port; use it as a context manager. + + :param chunks: 'recorded' sends the output as stored in the recording; + 'small' splits it into random pieces of 1 to 64 bytes (bulk data in + the middle of large output in larger pieces). + """ + + def __init__(self, recording, chunks='recorded', seed=0): + if chunks not in CHUNK_MODES: + raise ValueError(f'unknown chunk mode: {chunks}') + self.recording = recording + self.chunks = chunks + self.seed = seed + self.sessions = [] + self.errors = [] + self._closing = threading.Event() + self._sockets = [] + self._transports = [] + self._host_key = paramiko.ECDSAKey.generate() if recording.protocol != 'telnet' else None + self._listener = socket.create_server(('127.0.0.1', 0)) + self._listener.settimeout(0.1) + self.port = self._listener.getsockname()[1] + self._thread = threading.Thread(target=self._accept, daemon=True) + + def __enter__(self): + self._thread.start() + return self + + def __exit__(self, *exc_info): + self.close() + + def close(self): + self._closing.set() + self._thread.join() + self._listener.close() + for transport in self._transports: + transport.close() + for sock in self._sockets: + try: + sock.shutdown(socket.SHUT_RDWR) + except OSError: + pass + sock.close() + + def _accept(self): + while not self._closing.is_set(): + try: + sock, _ = self._listener.accept() + except TimeoutError: + continue + except OSError as exc: + self.errors.append(exc) + return + sock.settimeout(None) + sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + self._sockets.append(sock) + threading.Thread(target=self._serve, args=(sock,), daemon=True).start() + + def _serve(self, sock): + if self.recording.protocol == 'telnet': + self.start_session(CliReplay(self, TelnetStream(sock))) + return + transport = paramiko.Transport(sock) + self._transports.append(transport) + server_version = (self.recording.header.get('ssh') or {}).get('server_version') + if server_version: + transport.local_version = server_version + transport.add_server_key(self._host_key) + try: + transport.start_server(server=_SSHServer(self)) + except (OSError, EOFError, paramiko.SSHException): + pass # the client went away during the handshake + + def start_session(self, session): + self.sessions.append(session) + threading.Thread(target=session.run, daemon=True).start() + + +class _SSHServer(paramiko.ServerInterface): + def __init__(self, server): + self.server = server + ssh = server.recording.header.get('ssh') or {} + methods = ssh.get('auth_methods') or [] + self.auth_none = 'none' in methods + self.methods = [method for method in methods if method != 'none'] + if not {'password', 'keyboard-interactive'} & set(self.methods): + self.methods.append('password') # the tests log in with a password + self.banner = ssh.get('banner') + self.interactive = ssh.get('interactive') or { + 'title': '', + 'instructions': '', + 'prompts': [['Password: ', False]], + } + + def get_banner(self): + return (self.banner, 'en-US') if self.banner else (None, None) + + def get_allowed_auths(self, username): + return ','.join(self.methods) + + def check_auth_none(self, username): + return paramiko.AUTH_SUCCESSFUL if self.auth_none else paramiko.AUTH_FAILED + + def check_auth_password(self, username, password): + return paramiko.AUTH_SUCCESSFUL + + def check_auth_interactive(self, username, submethods): + query = self.interactive + prompts = [tuple(prompt) for prompt in query['prompts']] + return paramiko.InteractiveQuery(query['title'], query['instructions'], *prompts) + + def check_auth_interactive_response(self, responses): + return paramiko.AUTH_SUCCESSFUL + + def check_channel_request(self, kind, chanid): + if kind == 'session': + return paramiko.OPEN_SUCCEEDED + return paramiko.OPEN_FAILED_ADMINISTRATIVELY_PROHIBITED + + def check_channel_pty_request( + self, channel, term, width, height, pixelwidth, pixelheight, modes + ): + recorded = (self.server.recording.header.get('terminal') or {}).get('type') + term = term.decode() if isinstance(term, bytes) else term + if recorded and term != recorded: + self.server.errors.append( + f'the client requested terminal {term!r}, recorded was {recorded!r}' + ) + return True + + def check_channel_window_change_request(self, channel, width, height, pixelwidth, pixelheight): + return True + + def check_channel_shell_request(self, channel): + if self.server.recording.protocol != 'ssh': + return False + self.server.start_session(CliReplay(self.server, channel)) + return True + + def check_channel_subsystem_request(self, channel, name): + if self.server.recording.protocol != 'netconf' or name != 'netconf': + return False + self.server.start_session(NetconfReplay(self.server, channel)) + return True + + +class TelnetStream: + """A socket that drops Telnet commands from the received data.""" + + def __init__(self, sock): + self.sock = sock + self._pending = b'' + + def sendall(self, data): + self.sock.sendall(data) + + def close(self): + self.sock.close() + + def recv(self, size): + while chunk := self.sock.recv(size): + text, self._pending = _strip_telnet_commands(self._pending + chunk) + if text: + return text + return b'' + + +def _strip_telnet_commands(data): + text, pos = bytearray(), 0 + while pos < len(data): + if data[pos] != IAC: + text.append(data[pos]) + pos += 1 + continue + if pos + 1 >= len(data): + break + command = data[pos + 1] + if command == IAC: + text.append(IAC) + pos += 2 + elif command in (251, 252, 253, 254): # WILL, WONT, DO, DONT + if pos + 2 >= len(data): + break + pos += 3 + elif command == SB: + end = data.find(bytes([IAC, SE]), pos + 2) + if end < 0: + break + pos = end + 2 + else: + pos += 2 + return bytes(text), data[pos:] + + +class _Replay: + """The replay of one connection.""" + + def __init__(self, server, stream): + self.server = server + self.stream = stream + self.cursor = 0 + self.unrecorded = [] + self._input = b'' + self._random = random.Random(server.seed) + + def run(self): + try: + self.replay() + except (OSError, EOFError): + pass # the client closed the connection + except Exception as exc: + self.server.errors.append(exc) + raise + finally: + with contextlib.suppress(OSError, EOFError): + self.stream.close() + + def replay(self): + raise NotImplementedError + + def write(self, data): + if self.server.chunks == 'recorded': + self.stream.sendall(data) + return + pos = 0 + while pos < len(data): + bulk = pos >= 4096 and len(data) - pos > 512 + size = self._random.randint(512, 4096) if bulk else self._random.randint(1, 64) + self.stream.sendall(data[pos : pos + size]) + pos += size + time.sleep(0.001) + + def fill(self): + chunk = self.stream.recv(65536) + if not chunk: + raise EOFError + self._input += chunk + + +class CliReplay(_Replay): + """Replays a Telnet or SSH shell session.""" + + def __init__(self, server, stream): + super().__init__(server, stream) + self.events = server.recording.events + self._prompt = b'' + self._skip_lf = False + + def remaining(self): + """Returns the events that the client has not reached yet.""" + return self.events[self.cursor :] + + def replay(self): + while self._send_output(): + expected = self.events[self.cursor] if self.cursor < len(self.events) else None + line = self._read(expected) + if expected is not None and ( + expected.get('secret') or _strip(line) == _strip(data(expected)) + ): + self.cursor += 1 + continue + self.unrecorded.append(line) + self.write(_strip(line) + b'\r\n' + self._prompt) + + def _send_output(self): + """Sends the output up to the next client input; returns False at a recorded close.""" + while self.cursor < len(self.events): + event = self.events[self.cursor] + if event['dir'] == 'close': + return False + if event['dir'] == 'recv': + output = data(event) + self.write(output) + last_line = output.rsplit(b'\n', 1)[-1] + if last_line.strip(): + self._prompt = last_line + elif event.get('phase') != 'negotiation': + return True + self.cursor += 1 + return True + + def _read(self, expected): + """Reads a line, or as many bytes as a recorded input without a line end.""" + size = None + if expected is not None and not expected.get('secret'): + recorded = data(expected) + if not recorded.endswith((b'\r', b'\n')): + size = len(recorded) + while True: + if self._skip_lf and self._input: + if self._input[:1] in (b'\n', b'\x00'): + self._input = self._input[1:] + self._skip_lf = False + if size is not None and len(self._input) >= size: + line, self._input = self._input[:size], self._input[size:] + return line + match = re.search(rb'[\r\n]', self._input) + if size is None and match: + line, self._input = self._input[: match.end()], self._input[match.end() :] + self._skip_lf = line.endswith(b'\r') + return line + self.fill() + + +def _strip(line): + return line.rstrip(b'\r\n\x00') + + +class NetconfReplay(_Replay): + """Replays a NETCONF session on an SSH subsystem channel.""" + + def __init__(self, server, stream): + super().__init__(server, stream) + self.hello, self.exchange = server.recording.netconf_exchange() + + def remaining(self): + """Returns the (rpc, reply, sizes) tuples that the client has not reached yet.""" + return self.exchange[self.cursor :] + + def replay(self): + self.write(self.hello + EOM) + while EOM not in self._input: + self.fill() + client_hello, self._input = self._input.split(EOM, 1) + chunked = NETCONF_BASE_11 in self.hello and NETCONF_BASE_11 in client_hello + while True: + rpc = self._read_message(chunked) + name = operation(rpc) + sizes = [] + if ( + self.cursor < len(self.exchange) + and operation(self.exchange[self.cursor][0]) == name + ): + _, reply, sizes = self.exchange[self.cursor] + self.cursor += 1 + elif name == b'close-session': + reply = _ok_reply + else: + self.unrecorded.append(rpc) + reply = _error_reply + tag = _rpc_tag.search(rpc) + message_id = tag and _message_id.search(tag.group()) + if message_id: + reply = _set_message_id(reply, message_id.group(2)) + self.write(encode_message(reply, chunked, sizes)) + if name == b'close-session': + return + + def _read_message(self, chunked): + while not (messages := decode_messages(self._input, chunked)[0]): + self.fill() + message, sizes = messages[0] + self._input = self._input[len(encode_message(message, chunked, sizes)) :] + return message + + +def _set_message_id(reply, message_id): + tag = _reply_tag.search(reply) + if tag is None: + return reply + attribute = b'message-id=' + quoteattr(message_id.decode('latin-1')).encode('latin-1') + if _message_id.search(tag.group()): + new_tag = _message_id.sub(lambda _: attribute, tag.group(), count=1) + else: + new_tag = tag.group()[:-1] + b' ' + attribute + b'>' + return reply[: tag.start()] + new_tag + reply[tag.end() :] diff --git a/tests/recordings/README.md b/tests/recordings/README.md new file mode 100644 index 00000000..57653426 --- /dev/null +++ b/tests/recordings/README.md @@ -0,0 +1,108 @@ +# Recorded device sessions + +`tests/Exscript/protocols/ReplayTest.py` replays every `*/*.jsonl` file in this directory against +the real Telnet, SSH and NETCONF adapters and compares the results with the `.golden.json` file next +to it. The replay server (`tests/Exscript/protocols/replay.py`) runs on 127.0.0.1 and sends the +recorded device output, so the tests need no devices. + +Each recording runs twice: with the output as recorded, and split into small random pieces. Both +runs must give the same result. A third test checks the device's ground truth: login succeeds and +`guess_os()` returns `device.os`. + +## Adding a recording + +1. Write a command file (see `commands/`). For CLI sessions, put one command per line; empty lines + and lines starting with `#` are skipped. `!secret` sends a second secret that the recorder asks + for (e.g. after `enable`), and `!send TEXT` sends keys without a line end (`!send \x20` answers a + pager). For NETCONF, separate the RPC bodies with lines of `---`. + +2. Record the session. The recorder asks for the password and never stores it: + + ```sh + uv run tools/record_session.py ssh 192.0.2.1 -u admin -c cmds.txt -o raw.jsonl \ + --os ios --os-version 15.9 --platform C2960X + ``` + + - The protocol is `ssh`, `telnet` or `netconf`. + - `--os` is the ground truth: the Exscript driver name that `guess_os()` should return + (`unknown` if no driver fits). + - Keep the default terminal type `dumb`. It is Exscript's default, and the replay test uses the + recorded type. + - Use `--auth password` for SSH servers that drop the connection after a failed + keyboard-interactive attempt. + - See `--help` for the other options. + +3. Sanitize the recording, and replace host names, user names and anything else identifying: + + ```sh + uv run tools/sanitize_recording.py raw.jsonl tests/recordings/ios/ios-15.9-c2960x-ssh.jsonl \ + --replace core-rtr-01=router1 --replace jdoe=admin + ``` + + The sanitizer replaces addresses, MACs, secrets and serial numbers, and prints every replacement + and the prompts it saw. It cannot know every identifier: read the sanitized file before + committing it. Never commit a raw recording. + +4. Create the golden file, review it, and commit both files: + + ```sh + EXSCRIPT_UPDATE_GOLDEN=1 uv run pytest tests/Exscript/protocols/ReplayTest.py + ``` + + If Exscript does not handle the device correctly yet, add the recording to `KNOWN_ISSUES` in + `ReplayTest.py` with the reason. If its results depend on read timing, add it to + `TIMING_ISSUES` instead. + +Name files `/---[-].jsonl`. The directory groups +recordings by device family. + +## Format (version 1) + +A recording is a JSON Lines file. Bytes are stored as strings with one character per byte +(latin-1). + +The first line is the header: + +```json +{"format": "exscript-recording", "version": 1, "protocol": "ssh", "date": "2026-10-31", + "sanitized": true, "device": {"os": "shell", "version": "Alpine Linux 3.22", "platform": "docker"}, + "ssh": {"server_version": "SSH-2.0-OpenSSH_10.0", "banner": null, + "auth_methods": ["publickey", "password", "keyboard-interactive"], "interactive": null, + "host_key_type": "ssh-ed25519"}, + "terminal": {"type": "dumb", "width": 80, "height": 24}} +``` + +- `ssh` (SSH and NETCONF only) holds the server's version string, the pre-login banner, the + authentication methods it offered, and the keyboard-interactive query it sent, if any. +- `terminal` (CLI sessions only) holds the terminal that the recorder requested. +- `incomplete: true` marks a recording that ended with an error. + +Each following line is an event: + +```json +{"t": 2.1, "dir": "send", "data": "show version\r", "phase": "command"} +{"t": 2.103, "dir": "recv", "data": "show version\r\nCisco IOS Software, ..."} +{"t": 4.073, "dir": "send", "data": "", "secret": true, "phase": "login"} +{"t": 28.161, "dir": "close", "data": ""} +``` + +- `t` is the time in seconds since the connection was opened. +- `dir` is `recv` (device output), `send` (client input) or `close` (the device closed the + connection). +- `phase` of a `send` event is `negotiation` (Telnet options), `login`, `command` or `logout`. +- A `send` with `secret: true` is a password whose data was not stored. + +For NETCONF, the data is the raw content of the `netconf` SSH subsystem, including the framing. + +## Replay rules + +- **Matching input.** The server sends the output up to the next recorded `send` and waits for a + line from the client. A matching line continues the replay. A recorded secret matches any line. + Any other line is echoed and answered with the last prompt, so `login()` may send commands that + were not recorded, such as `enable`. +- **Reconnects.** Every connection replays the recording from the start, because some drivers + reconnect between SSH authentication methods. +- **NETCONF.** An RPC gets the next recorded reply if the operation name matches. The reply gets + the client's message-id and keeps its recorded chunk sizes. An unrecorded `close-session` gets + ``; any other unrecorded RPC gets an `rpc-error`. +- **Authentication.** The SSH server accepts any password or keyboard-interactive answer. diff --git a/tests/recordings/commands/busybox-ash.txt b/tests/recordings/commands/busybox-ash.txt new file mode 100644 index 00000000..03dc871d --- /dev/null +++ b/tests/recordings/commands/busybox-ash.txt @@ -0,0 +1,7 @@ +# BusyBox ash over Telnet (R1). +uname -sr +cat /etc/passwd +ls -l / +seq 1 5000 +ls /nonexistent +exit diff --git a/tests/recordings/commands/frr-vtysh.txt b/tests/recordings/commands/frr-vtysh.txt new file mode 100644 index 00000000..10e6de68 --- /dev/null +++ b/tests/recordings/commands/frr-vtysh.txt @@ -0,0 +1,15 @@ +# FRRouting vtysh over SSH (R1), an IOS-like CLI. +terminal length 0 +show version +show running-config +show interface brief +show ip route +show nonexistent +show ip +sh i +configure terminal +interface lo +description loopback for the Exscript tests +end +show running-config +exit diff --git a/tests/recordings/commands/ios-basic.txt b/tests/recordings/commands/ios-basic.txt new file mode 100644 index 00000000..14c9a0dc --- /dev/null +++ b/tests/recordings/commands/ios-basic.txt @@ -0,0 +1,28 @@ +# Cisco IOS and IOS XE (R2, lab devices only), with --os ios. +# Log in as a user that lands in user EXEC (">"); drop the enable and !secret lines if you land in "#". +# Changes the running config: creates and removes Loopback999. +enable +!secret +terminal length 0 +terminal width 0 +show version +show running-config +show interfaces +show ip route +show ip interface brief +show inventory +show nonexistent +show ip +sh i +configure terminal +interface Loopback999 +description Exscript test, safe to remove +end +show running-config interface Loopback999 +configure terminal +no interface Loopback999 +end +clear counters +!send \r +show this-command-is-much-longer-than-the-terminal-width | include something-that-does-not-match-anything-at-all +exit diff --git a/tests/recordings/commands/ios-pager.txt b/tests/recordings/commands/ios-pager.txt new file mode 100644 index 00000000..2b022469 --- /dev/null +++ b/tests/recordings/commands/ios-pager.txt @@ -0,0 +1,6 @@ +# Cisco IOS, IOS XE and IOS XR (R2): output with the pager on. +show running-config +!send \x20 +!send \x20 +!send q +exit diff --git a/tests/recordings/commands/ios_xr-basic.txt b/tests/recordings/commands/ios_xr-basic.txt new file mode 100644 index 00000000..44077c7c --- /dev/null +++ b/tests/recordings/commands/ios_xr-basic.txt @@ -0,0 +1,23 @@ +# Cisco IOS XR, cXR and eXR (R2, lab devices only), with --os ios_xr. +# Changes the running config: creates and removes Loopback999 (two commits). +terminal length 0 +terminal width 0 +show version +show running-config +show interfaces brief +show route +show inventory +show nonexistent +show ip +configure terminal +interface Loopback999 +description Exscript test, safe to remove +commit +end +show running-config interface Loopback999 +configure terminal +no interface Loopback999 +commit +end +show this-command-is-much-longer-than-the-terminal-width | include something-that-does-not-match-anything-at-all +exit diff --git a/tests/recordings/commands/junos-basic.txt b/tests/recordings/commands/junos-basic.txt new file mode 100644 index 00000000..316a7069 --- /dev/null +++ b/tests/recordings/commands/junos-basic.txt @@ -0,0 +1,19 @@ +# Juniper Junos and Junos Evolved (R2, lab devices only), with --os junos. +# Enters configuration mode but commits nothing (commit check, then rollback 0). +set cli screen-length 0 +set cli screen-width 0 +show version +show configuration +show interfaces terse +show route +show chassis hardware +show nonexistent +show i +configure +set interfaces lo0 description "Exscript test" +show | compare +commit check +rollback 0 +exit configuration-mode +show this-command-is-much-longer-than-the-terminal-width | match something-that-does-not-match-anything-at-all +exit diff --git a/tests/recordings/commands/junos-pager.txt b/tests/recordings/commands/junos-pager.txt new file mode 100644 index 00000000..ead18383 --- /dev/null +++ b/tests/recordings/commands/junos-pager.txt @@ -0,0 +1,6 @@ +# Juniper Junos (R2): output with the pager on. +show configuration +!send \x20 +!send \x20 +!send q +exit diff --git a/tests/recordings/commands/linux-bash.txt b/tests/recordings/commands/linux-bash.txt new file mode 100644 index 00000000..53cb965f --- /dev/null +++ b/tests/recordings/commands/linux-bash.txt @@ -0,0 +1,9 @@ +# Linux shell over SSH (R1): Alpine container with bash. +uname -sr +cat /etc/os-release +ls -l --color=always / +printf 'umlauts: \303\244\303\266\303\274, euro sign: \342\202\254\n' +echo the quick brown fox jumps over the lazy dog, the quick brown fox jumps over the lazy dog, and again +seq 1 20000 +ls /nonexistent +exit diff --git a/tests/recordings/commands/netconf-junos.xml b/tests/recordings/commands/netconf-junos.xml new file mode 100644 index 00000000..a13c5127 --- /dev/null +++ b/tests/recordings/commands/netconf-junos.xml @@ -0,0 +1,11 @@ + +--- + +--- + +--- + +--- + +--- + diff --git a/tests/recordings/commands/netconf.xml b/tests/recordings/commands/netconf.xml new file mode 100644 index 00000000..954197b1 --- /dev/null +++ b/tests/recordings/commands/netconf.xml @@ -0,0 +1,9 @@ + +--- + +--- + +--- + +--- + diff --git a/tests/recordings/commands/netopeer2.xml b/tests/recordings/commands/netopeer2.xml new file mode 100644 index 00000000..85ef03bb --- /dev/null +++ b/tests/recordings/commands/netopeer2.xml @@ -0,0 +1,11 @@ + +--- + +--- + +--- + +--- + +--- + diff --git a/tests/recordings/frr/frr-10.4-vtysh-ssh.jsonl b/tests/recordings/frr/frr-10.4-vtysh-ssh.jsonl new file mode 100644 index 00000000..86386868 --- /dev/null +++ b/tests/recordings/frr/frr-10.4-vtysh-ssh.jsonl @@ -0,0 +1,31 @@ +{"format": "exscript-recording", "version": 1, "protocol": "ssh", "date": "2026-10-31", "sanitized": true, "device": {"os": "unknown", "version": "FRRouting 10.4.1", "platform": "docker quay.io/frrouting/frr:10.4.1"}, "ssh": {"server_version": "SSH-2.0-OpenSSH_9.7", "banner": null, "auth_methods": ["publickey", "password", "keyboard-interactive"], "interactive": null, "host_key_type": "ssh-ed25519"}, "terminal": {"type": "dumb", "width": 80, "height": 24}} +{"t": 0.089, "dir": "recv", "data": "Welcome to Alpine!\r\n\r\nThe Alpine Wiki contains a large amount of how-to guides and general\r\ninformation about administrating Alpine systems.\r\nSee .\r\n\r\nYou can setup the system with the command: setup-alpine\r\n\r\nYou may change this message by editing /etc/motd.\r\n\r\n\r\nHello, this is FRRouting (version 10.4.1_git).\r\nCopyright 1996-2005 Kunihiro Ishiguro, et al.\r\n\r\nrouter2# "} +{"t": 2.098, "dir": "send", "data": "terminal length 0\r", "phase": "command"} +{"t": 2.1, "dir": "recv", "data": "terminal length 0\r\nrouter2# "} +{"t": 4.101, "dir": "send", "data": "show version\r", "phase": "command"} +{"t": 4.11, "dir": "recv", "data": "show version\r\nFRRouting 10.4.1_git (router2) on Linux(6.18.54.1-microsoft-standard-WSL2).\r\nCopyright 1996-2005 Kunihiro Ishiguro, et al.\r\nconfigured with:\r\n '--prefix=/usr' '--sysconfdir=/etc' '--localstatedir=/var' '--sbindir=/usr/lib/frr' '--libdir=/usr/lib' '--enable-rpki' '--enable-vtysh' '--enable-multipath=64' '--enable-vty-group=frrvty' '--enable-user=frr' '--enable-group=frr' '--enable-pcre2posix' '--enable-scripting' 'CC=gcc' 'CXX=g++'\r\nrouter2# "} +{"t": 6.111, "dir": "send", "data": "show running-config\r", "phase": "command"} +{"t": 6.113, "dir": "recv", "data": "show running-config\r\nBuilding configuration...\r\n\r\nCurrent configuration:\r\n!\r\nfrr version 10.4.1_git\r\nfrr defaults traditional\r\nhostname router2\r\ndomainname localdomain\r\n!\r\nip route 192.0.2.0/24 Null0\r\nip route 198.51.100.0/24 Null0\r\nipv6 route 2001:db8:1::/48 Null0\r\n!\r\nend\r\nrouter2# "} +{"t": 8.114, "dir": "send", "data": "show interface brief\r", "phase": "command"} +{"t": 8.116, "dir": "recv", "data": "show interface brief\r\nInterface Status VRF Addresses\r\n--------- ------ --- ---------\r\neth0 up default 192.0.2.1/16\r\nlo up default \r\n\r\nrouter2# "} +{"t": 10.117, "dir": "send", "data": "show ip route\r", "phase": "command"} +{"t": 10.119, "dir": "recv", "data": "show ip route\r\nCodes: K - kernel route, C - connected, L - local, S - static,\r\n R - RIP, O - OSPF, I - IS-IS, B - BGP, E - EIGRP, N - NHRP,\r\n T - Table, v - VNC, V - VNC-Direct, A - Babel, F - PBR,\r\n f - OpenFabric, t - Table-Direct,\r\n > - selected route, * - FIB route, q - queued, r - rejected, b - backup\r\n t - trapped, o - offload failure\r\n\r\nIPv4 unicast VRF default:\r\nK>* 0.0.0.0/0 [0/0] via 192.0.2.2, eth0, weight 1, 00:11:08\r\nC>* 192.0.2.3/16 is directly connected, eth0, weight 1, 00:11:08\r\nL>* 192.0.2.1/32 is directly connected, eth0, weight 1, 00:11:08\r\nS>* 192.0.2.0/24 [1/0] unreachable (blackhole), weight 1, 00:00:26\r\nS>* 198.51.100.0/24 [1/0] unreachable (blackhole), weight 1, 00:00:26\r\nrouter2# "} +{"t": 12.119, "dir": "send", "data": "show nonexistent\r", "phase": "command"} +{"t": 12.121, "dir": "recv", "data": "show nonexistent\r\n% Unknown command: show nonexistent\r\nrouter2# "} +{"t": 14.121, "dir": "send", "data": "show ip\r", "phase": "command"} +{"t": 14.127, "dir": "recv", "data": "show ip\r\n% Command incomplete: show ip\r\nrouter2# "} +{"t": 16.128, "dir": "send", "data": "sh i\r", "phase": "command"} +{"t": 16.129, "dir": "recv", "data": "sh i\r\nInterface eth0 is up, line protocol is up\r\n Link ups: 0 last: (never)\r\n Link downs: 0 last: (never)\r\n vrf: default\r\n index 2 metric 0 mtu 1500 speed 10000 txqlen 0\r\n flags: \r\n MPLS Not specified by CLI\r\n Multicast config is Not specified by CLI\r\n Shutdown config is Not specified by CLI\r\n Type: Ethernet\r\n HWaddr: 02:00:00:00:00:01\r\n inet 192.0.2.1/16\r\n Interface Type VETH\r\n Interface Slave Type None\r\n protodown: off \r\n Parent ifindex: 18\r\nInterface lo is up, line protocol is up\r\n Link ups: 0 last: (never)\r\n Link downs: 0 last: (never)\r\n vrf: default\r\n index 1 metric 0 mtu 65536 speed 0 txqlen 1000\r\n flags: \r\n MPLS Not specified by CLI\r\n Multicast config is Not specified by CLI\r\n Shutdown config is Not specified by CLI\r\n Type: Loopback\r\n Interface Type Other\r\n Interface Slave Type None\r\n protodown: off \r\nrouter2# "} +{"t": 18.132, "dir": "send", "data": "configure terminal\r", "phase": "command"} +{"t": 18.137, "dir": "recv", "data": "configure terminal\r\nrouter2(config)# "} +{"t": 20.137, "dir": "send", "data": "interface lo\r", "phase": "command"} +{"t": 20.139, "dir": "recv", "data": "interface lo\r\nrouter2(config-if)# "} +{"t": 22.141, "dir": "send", "data": "description loopback for the Exscript tests\r", "phase": "command"} +{"t": 22.15, "dir": "recv", "data": "description loopback for the Exscript tests\r\nrouter2(config-if)# "} +{"t": 24.151, "dir": "send", "data": "end\r", "phase": "command"} +{"t": 24.153, "dir": "recv", "data": "end\r\nrouter2# "} +{"t": 26.154, "dir": "send", "data": "show running-config\r", "phase": "command"} +{"t": 26.157, "dir": "recv", "data": "show running-config\r\nBuilding configuration...\r\n\r\nCurrent configuration:\r\n!\r\nfrr version 10.4.1_git\r\nfrr defaults traditional\r\nhostname router2\r\ndomainname localdomain\r\n!\r\nip route 192.0.2.0/24 Null0\r\nip route 198.51.100.0/24 Null0\r\nipv6 route 2001:db8:1::/48 Null0\r\n!\r\ninterface lo\r\n description loopback for the Exscript tests\r\nexit\r\n!\r\nend\r\nrouter2# "} +{"t": 28.158, "dir": "send", "data": "exit\r", "phase": "command"} +{"t": 28.16, "dir": "recv", "data": "exit\r\n"} +{"t": 28.161, "dir": "close", "data": ""} diff --git a/tests/recordings/linux/alpine-3.22-bash-ssh-vt100.golden.json b/tests/recordings/linux/alpine-3.22-bash-ssh-vt100.golden.json new file mode 100644 index 00000000..5976248f --- /dev/null +++ b/tests/recordings/linux/alpine-3.22-bash-ssh-vt100.golden.json @@ -0,0 +1,6 @@ +{ + "login": "TimeoutException", + "driver": "generic", + "os": "unknown", + "commands": [] +} diff --git a/tests/recordings/linux/alpine-3.22-bash-ssh-vt100.jsonl b/tests/recordings/linux/alpine-3.22-bash-ssh-vt100.jsonl new file mode 100644 index 00000000..4e9fa001 --- /dev/null +++ b/tests/recordings/linux/alpine-3.22-bash-ssh-vt100.jsonl @@ -0,0 +1,19 @@ +{"format": "exscript-recording", "version": 1, "protocol": "ssh", "date": "2026-10-31", "sanitized": true, "device": {"os": "shell", "version": "Alpine Linux 3.22", "platform": "docker alpine:3.22, bash 5.2"}, "ssh": {"server_version": "SSH-2.0-OpenSSH_10.0", "banner": null, "auth_methods": ["publickey", "password", "keyboard-interactive"], "interactive": null, "host_key_type": "ssh-ed25519"}, "terminal": {"type": "vt100", "width": 80, "height": 24}} +{"t": 0.09, "dir": "recv", "data": "Welcome to router1, an Exscript test host.\r\n\u001b[?2004hrouter1:~$ "} +{"t": 2.1, "dir": "send", "data": "uname -sr\r", "phase": "command"} +{"t": 2.103, "dir": "recv", "data": "uname -sr\r\n\u001b[?2004l\rLinux 6.18.54.1-microsoft-standard-WSL2\r\n\u001b[?2004hrouter1:~$ "} +{"t": 4.108, "dir": "send", "data": "cat /etc/os-release\r", "phase": "command"} +{"t": 4.111, "dir": "recv", "data": "cat /etc/os-release\r\n\u001b[?2004l\rNAME=\"Alpine Linux\"\r\nID=alpine\r\nVERSION_ID=3.22.7\r\nPRETTY_NAME=\"Alpine Linux v3.22\"\r\nHOME_URL=\"https://alpinelinux.org/\"\r\nBUG_REPORT_URL=\"https://gitlab.alpinelinux.org/alpine/aports/-/issues\"\r\n\u001b[?2004hrouter1:~$ "} +{"t": 6.112, "dir": "send", "data": "ls -l --color=always /\r", "phase": "command"} +{"t": 6.116, "dir": "recv", "data": "ls -l --color=always /\r\n\u001b[?2004l\rtotal 68\r\ndrwxr-xr-x 1 root root 4096 Oct 31 08:37 \u001b[1;34mbin\u001b[m\r\ndrwxr-xr-x 5 root root 340 Oct 31 08:37 \u001b[1;34mdev\u001b[m\r\ndrwxr-xr-x 1 root root 4096 Oct 31 08:38 \u001b[1;34metc\u001b[m\r\ndrwxr-xr-x 1 root root 4096 Oct 31 08:38 \u001b[1;34mhome\u001b[m\r\ndrwxr-xr-x 1 root root 4096 Oct 21 19:59 \u001b[1;34mlib\u001b[m\r\ndrwxr-xr-x 5 root root 4096 Oct 21 19:59 \u001b[1;34mmedia\u001b[m\r\ndrwxr-xr-x 2 root root 4096 Oct 21 19:59 \u001b[1;34mmnt\u001b[m\r\ndrwxr-xr-x 2 root root 4096 Oct 21 19:59 \u001b[1;34mopt\u001b[m\r\ndr-xr-xr-x 399 root root 0 Oct 31 08:37 \u001b[1;34mproc\u001b[m\r\ndrwx------ 2 root root 4096 Oct 21 19:59 \u001b[1;34mroot\u001b[m\r\ndrwxr-xr-x 1 root root 4096 Oct 31 08:38 \u001b[1;34mrun\u001b[m\r\ndrwxr-xr-x 2 root root 4096 Oct 21 19:59 \u001b[1;34msbin\u001b[m\r\ndrwxr-xr-x 2 root root 4096 Oct 21 19:59 \u001b[1;34msrv\u001b[m\r\ndr-xr-xr-x 12 root root 0 Oct 30 12:05 \u001b[1;34msys\u001b[m\r\ndrwxrwxrwt 2 root root 4096 Oct 21 19:59 \u001b[1;34mtmp\u001b[m\r\ndrwxr-xr-x 1 root root 4096 Oct 21 19:59 \u001b[1;34musr\u001b[m\r\ndrwxr-xr-x 11 root root 4096 Oct 21 19:59 \u001b[1;34mvar\u001b[m\r\n\u001b[?2004hrouter1:~$ "} +{"t": 8.119, "dir": "send", "data": "printf 'umlauts: \\303\\244\\303\\266\\303\\274, euro sign: \\342\\202\\254\\n'\r", "phase": "command"} +{"t": 8.121, "dir": "recv", "data": "printf 'umlauts: \\303\\244\\303\\266\\303\\274, euro sign: \\342\\202\\254\\n' \r\u001b[A\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u001b[C\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u001b[C\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u001b[C\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u001b[C\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u001b[C\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u001b[C\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u001b[C\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u001b[C\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u001b[C\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u001b[C\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u001b[C\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u001b[C\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u001b[C\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u001b[C\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u001b[C\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u001b[C\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u001b[C\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u001b[C\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u001b[C\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u001b[C\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u001b[C\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u001b[C\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u001b[C\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u001b[C\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u001b[C\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u001b[C\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u001b[C\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u001b[C\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u001b[C\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u001b[C\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u001b[C\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u001b[C\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u001b[C\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u001b[C\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u001b[C\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u001b[C\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u001b[C\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u001b[C\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u001b[C\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u001b[C\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u001b[C\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u001b[C\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u001b[C\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u001b[C\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u001b[C\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u001b[C\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u001b[C\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u001b[C\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u001b[C\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u001b[C\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u001b[C\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u001b[C\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u001b[C\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u001b[C\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u001b[C\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u001b[C\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u001b[C\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u001b[C\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u001b[C\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u001b[C\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u001b[C\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u001b[C\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u001b[C\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u001b[C\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u001b[C\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u001b[C\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u001b[C\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u001b[C\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u001b[C\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u001b[C\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u001b[C\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u001b[C\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u001b[C\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u001b[C\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u001b[C\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u001b[C\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u001b[C\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u001b[C\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u001b[C\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u001b[K\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000'\r\n\u001b[?2004l\rumlauts: \u00c3\u00a4\u00c3\u00b6\u00c3\u00bc, euro sign: \u00e2\u0082\u00ac\r\n\u001b[?2004hrouter1:~$ "} +{"t": 10.123, "dir": "send", "data": "echo the quick brown fox jumps over the lazy dog, the quick brown fox jumps over the lazy dog, and again\r", "phase": "command"} +{"t": 10.125, "dir": "recv", "data": "echo the quick brown fox jumps over the lazy dog, the quick brown fox jumps over the lazy dog, and again\r\n\u001b[?2004l\rthe quick brown fox jumps over the lazy dog, the quick brown fox jumps over the lazy dog, and again\r\n\u001b[?2004hrouter1:~$ "} +{"t": 12.126, "dir": "send", "data": "seq 1 20000\r", "phase": "command"} +{"t": 12.128, "dir": "recv", "data": "seq 1 20000\r\n\u001b[?2004l\r1\r\n2\r\n3\r\n4\r\n5\r\n6\r\n7\r\n8\r\n9\r\n10\r\n11\r\n12\r\n13\r\n14\r\n15\r\n16\r\n17\r\n18\r\n19\r\n20\r\n21\r\n22\r\n23\r\n24\r\n25\r\n26\r\n27\r\n28\r\n29\r\n30\r\n31\r\n32\r\n33\r\n34\r\n35\r\n36\r\n37\r\n38\r\n39\r\n40\r\n41\r\n42\r\n43\r\n44\r\n45\r\n46\r\n47\r\n48\r\n49\r\n50\r\n51\r\n52\r\n53\r\n54\r\n55\r\n56\r\n57\r\n58\r\n59\r\n60\r\n61\r\n62\r\n63\r\n64\r\n65\r\n66\r\n67\r\n68\r\n69\r\n70\r\n71\r\n72\r\n73\r\n74\r\n75\r\n76\r\n77\r\n78\r\n79\r\n80\r\n81\r\n82\r\n83\r\n84\r\n85\r\n86\r\n87\r\n88\r\n89\r\n90\r\n91\r\n92\r\n93\r\n94\r\n95\r\n96\r\n97\r\n98\r\n99\r\n100\r\n101\r\n102\r\n103\r\n104\r\n105\r\n106\r\n107\r\n108\r\n109\r\n110\r\n111\r\n112\r\n113\r\n114\r\n115\r\n116\r\n117\r\n118\r\n119\r\n120\r\n121\r\n122\r\n123\r\n124\r\n125\r\n126\r\n127\r\n128\r\n129\r\n130\r\n131\r\n132\r\n133\r\n134\r\n135\r\n136\r\n137\r\n138\r\n139\r\n140\r\n141\r\n142\r\n143\r\n144\r\n145\r\n146\r\n147\r\n148\r\n149\r\n150\r\n151\r\n152\r\n153\r\n154\r\n155\r\n156\r\n157\r\n158\r\n159\r\n160\r\n161\r\n162\r\n163\r\n164\r\n165\r\n166\r\n167\r\n168\r\n169\r\n170\r\n171\r\n172\r\n173\r\n174\r\n175\r\n176\r\n177\r\n178\r\n179\r\n180\r\n181\r\n182\r\n183\r\n184\r\n185\r\n186\r\n187\r\n188\r\n189\r\n190\r\n191\r\n192\r\n193\r\n194\r\n195\r\n196\r\n197\r\n198\r\n199\r\n200\r\n201\r\n202\r\n203\r\n204\r\n205\r\n206\r\n207\r\n208\r\n209\r\n210\r\n211\r\n212\r\n213\r\n214\r\n215\r\n216\r\n217\r\n218\r\n219\r\n220\r\n221\r\n222\r\n223\r\n224\r\n225\r\n226\r\n227\r\n228\r\n229\r\n230\r\n231\r\n232\r\n233\r\n234\r\n235\r\n236\r\n237\r\n238\r\n239\r\n240\r\n241\r\n242\r\n243\r\n244\r\n245\r\n246\r\n247\r\n248\r\n249\r\n250\r\n251\r\n252\r\n253\r\n254\r\n255\r\n256\r\n257\r\n258\r\n259\r\n260\r\n261\r\n262\r\n263\r\n264\r\n265\r\n266\r\n267\r\n268\r\n269\r\n270\r\n271\r\n272\r\n273\r\n274\r\n275\r\n276\r\n277\r\n278\r\n279\r\n280\r\n281\r\n282\r\n283\r\n284\r\n285\r\n286\r\n287\r\n288\r\n289\r\n290\r\n291\r\n292\r\n293\r\n294\r\n295\r\n296\r\n297\r\n298\r\n299\r\n300\r\n301\r\n302\r\n303\r\n304\r\n305\r\n306\r\n307\r\n308\r\n309\r\n310\r\n311\r\n312\r\n313\r\n314\r\n315\r\n316\r\n317\r\n318\r\n319\r\n320\r\n321\r\n322\r\n323\r\n324\r\n325\r\n326\r\n327\r\n328\r\n329\r\n330\r\n331\r\n332\r\n333\r\n334\r\n335\r\n336\r\n337\r\n338\r\n339\r\n340\r\n341\r\n342\r\n343\r\n344\r\n345\r\n346\r\n347\r\n348\r\n349\r\n350\r\n351\r\n352\r\n353\r\n354\r\n355\r\n356\r\n357\r\n358\r\n359\r\n360\r\n361\r\n362\r\n363\r\n364\r\n365\r\n366\r\n367\r\n368\r\n369\r\n370\r\n371\r\n372\r\n373\r\n374\r\n375\r\n376\r\n377\r\n378\r\n379\r\n380\r\n381\r\n382\r\n383\r\n384\r\n385\r\n386\r\n387\r\n388\r\n389\r\n390\r\n391\r\n392\r\n393\r\n394\r\n395\r\n396\r\n397\r\n398\r\n399\r\n400\r\n401\r\n402\r\n403\r\n404\r\n405\r\n406\r\n407\r\n408\r\n409\r\n410\r\n411\r\n412\r\n413\r\n414\r\n415\r\n416\r\n417\r\n418\r\n419\r\n420\r\n421\r\n422\r\n423\r\n424\r\n425\r\n426\r\n427\r\n428\r\n429\r\n430\r\n431\r\n432\r\n433\r\n434\r\n435\r\n436\r\n437\r\n438\r\n439\r\n440\r\n441\r\n442\r\n443\r\n444\r\n445\r\n446\r\n447\r\n448\r\n449\r\n450\r\n451\r\n452\r\n453\r\n454\r\n455\r\n456\r\n457\r\n458\r\n459\r\n460\r\n461\r\n462\r\n463\r\n464\r\n465\r\n466\r\n467\r\n468\r\n469\r\n470\r\n471\r\n472\r\n473\r\n474\r\n475\r\n476\r\n477\r\n478\r\n479\r\n480\r\n481\r\n482\r\n483\r\n484\r\n485\r\n486\r\n487\r\n488\r\n489\r\n490\r\n491\r\n492\r\n493\r\n494\r\n495\r\n496\r\n497\r\n498\r\n499\r\n500\r\n501\r\n502\r\n503\r\n504\r\n505\r\n506\r\n507\r\n508\r\n509\r\n510\r\n511\r\n512\r\n513\r\n514\r\n515\r\n516\r\n517\r\n518\r\n519\r\n520\r\n521\r\n522\r\n523\r\n524\r\n525\r\n526\r\n527\r\n528\r\n529\r\n530\r\n531\r\n532\r\n533\r\n534\r\n535\r\n536\r\n537\r\n538\r\n539\r\n540\r\n541\r\n542\r\n543\r\n544\r\n545\r\n546\r\n547\r\n548\r\n549\r\n550\r\n551\r\n552\r\n553\r\n554\r\n555\r\n556\r\n557\r\n558\r\n559\r\n560\r\n561\r\n562\r\n563\r\n564\r\n565\r\n566\r\n567\r\n568\r\n569\r\n570\r\n571\r\n572\r\n573\r\n574\r\n575\r\n576\r\n577\r\n578\r\n579\r\n580\r\n581\r\n582\r\n583\r\n584\r\n585\r\n586\r\n587\r\n588\r\n589\r\n590\r\n591\r\n592\r\n593\r\n594\r\n595\r\n596\r\n597\r\n598\r\n599\r\n600\r\n601\r\n602\r\n603\r\n604\r\n605\r\n606\r\n607\r\n608\r\n609\r\n610\r\n611\r\n612\r\n613\r\n614\r\n615\r\n616\r\n617\r\n618\r\n619\r\n620\r\n621\r\n622\r\n623\r\n624\r\n625\r\n626\r\n627\r\n628\r\n629\r\n630\r\n631\r\n632\r\n633\r\n634\r\n635\r\n636\r\n637\r\n638\r\n639\r\n640\r\n641\r\n642\r\n643\r\n644\r\n645\r\n646\r\n647\r\n648\r\n649\r\n650\r\n651\r\n652\r\n653\r\n654\r\n655\r\n656\r\n657\r\n658\r\n659\r\n660\r\n661\r\n662\r\n663\r\n664\r\n665\r\n666\r\n667\r\n668\r\n669\r\n670\r\n671\r\n672\r\n673\r\n674\r\n675\r\n676\r\n677\r\n678\r\n679\r\n680\r\n681\r\n682\r\n683\r\n684\r\n685\r\n686\r\n687\r\n688\r\n689\r\n690\r\n691\r\n692\r\n693\r\n694\r\n695\r\n696\r\n697\r\n698\r\n699\r\n700\r\n701\r\n702\r\n703\r\n704\r\n705\r\n706\r\n707\r\n708\r\n709\r\n710\r\n711\r\n712\r\n713\r\n714\r\n715\r\n716\r\n717\r\n718\r\n719\r\n720\r\n721\r\n722\r\n723\r\n724\r\n725\r\n726\r\n727\r\n728\r\n729\r\n730\r\n731\r\n732\r\n733\r\n734\r\n735\r\n736\r\n737\r\n738\r\n739\r\n740\r\n741\r\n742\r\n743\r\n744\r\n745\r\n746\r\n747\r\n748\r\n749\r\n750\r\n751\r\n752\r\n753\r\n754\r\n755\r\n756\r\n757\r\n758\r\n759\r\n760\r\n761\r\n762\r\n763\r\n764\r\n765\r\n766\r\n767\r\n768\r\n769\r\n770\r\n771\r\n772\r\n773\r\n774\r\n775\r\n776\r\n777\r\n778\r\n779\r\n780\r\n781\r\n782\r\n783\r\n784\r\n785\r\n786\r\n787\r\n788\r\n789\r\n790\r\n791\r\n792\r\n793\r\n794\r\n795\r\n796\r\n797\r\n798\r\n799\r\n800\r\n801\r\n802\r\n803\r\n804\r\n805\r\n806\r\n807\r\n808\r\n809\r\n810\r\n811\r\n812\r\n813\r\n814\r\n815\r\n816\r\n817\r\n818\r\n819\r\n820\r\n821\r\n822\r\n823\r\n824\r\n825\r\n826\r\n827\r\n828\r\n829\r\n830\r\n831\r\n832\r\n833\r\n834\r\n835\r\n836\r\n837\r\n838\r\n839\r\n840\r\n841\r\n842\r\n843\r\n844\r\n845\r\n846\r\n847\r\n848\r\n849\r\n850\r\n851\r\n852\r\n853\r\n854\r\n855\r\n856\r\n857\r\n858\r\n859\r\n860\r\n861\r\n862\r\n863\r\n864\r\n865\r\n866\r\n867\r\n868\r\n869\r\n870\r\n871\r\n872\r\n873\r\n874\r\n875\r\n876\r\n877\r\n878\r\n879\r\n880\r\n881\r\n882\r\n883\r\n884\r\n885\r\n886\r\n887\r\n888\r\n889\r\n890\r\n891\r\n892\r\n893\r\n894\r\n895\r\n896\r\n897\r\n898\r\n899\r\n900\r\n901\r\n902\r\n903\r\n904\r\n905\r\n906\r\n907\r\n908\r\n909\r\n910\r\n911\r\n912\r\n913\r\n914\r\n915\r\n916\r\n917\r\n918\r\n919\r\n920\r\n921\r\n922\r\n923\r\n924\r\n925\r\n926\r\n927\r\n928\r\n929\r\n930\r\n931\r\n932\r\n933\r\n934\r\n935\r\n936\r\n937\r\n938\r\n939\r\n940\r\n941\r\n942\r\n943\r\n944\r\n945\r\n946\r\n947\r\n948\r\n949\r\n950\r\n951\r\n952\r\n953\r\n954\r\n955\r\n956\r\n957\r\n958\r\n959\r\n960\r\n961\r\n962\r\n963\r\n964\r\n965\r\n966\r\n967\r\n968\r\n969\r\n970\r\n971\r\n972\r\n973\r\n974\r\n975\r\n976\r\n977\r\n978\r\n979\r\n980\r\n981\r\n982\r\n983\r\n984\r\n985\r\n986\r\n987\r\n988\r\n989\r\n990\r\n991\r\n992\r\n993\r\n994\r\n995\r\n996\r\n997\r\n998\r\n999\r\n1000\r\n1001\r\n1002\r\n1003\r\n1004\r\n1005\r\n1006\r\n1007\r\n1008\r\n1009\r\n1010\r\n1011\r\n1012\r\n1013\r\n1014\r\n1015\r\n1016\r\n1017\r\n1018\r\n1019\r\n1020\r\n1021\r\n1022\r\n1023\r\n1024\r\n1025\r\n1026\r\n1027\r\n1028\r\n1029\r\n1030\r\n1031\r\n1032\r\n1033\r\n1034\r\n1035\r\n1036\r\n1037\r\n1038\r\n1039\r\n1040\r\n1041\r\n1042\r\n1043\r\n1044\r\n1045\r\n1046\r\n1047\r\n1048\r\n1049\r\n1050\r\n1051\r\n1052\r\n1053\r\n1054\r\n1055\r\n1056\r\n1057\r\n1058\r\n1059\r\n1060\r\n1061\r\n1062\r\n1063\r\n1064\r\n1065\r\n1066\r\n1067\r\n1068\r\n1069\r\n1070\r\n1071\r\n1072\r\n1073\r\n1074\r\n1075\r\n1076\r\n1077\r\n1078\r\n1079\r\n1080\r\n1081\r\n1082\r\n1083\r\n1084\r\n1085\r\n1086\r\n1087\r\n1088\r\n1089\r\n1090\r\n1091\r\n1092\r\n1093\r\n1094\r\n1095\r\n1096\r\n1097\r\n1098\r\n1099\r\n1100\r\n1101\r\n1102\r\n1103\r\n1104\r\n1105\r\n1106\r\n1107\r\n1108\r\n1109\r\n1110\r\n1111\r\n1112\r\n1113\r\n1114\r\n1115\r\n1116\r\n1117\r\n1118\r\n1119\r\n1120\r\n1121\r\n1122\r\n1123\r\n1124\r\n1125\r\n1126\r\n1127\r\n1128\r\n1129\r\n1130\r\n1131\r\n1132\r\n1133\r\n1134\r\n1135\r\n1136\r\n1137\r\n1138\r\n1139\r\n1140\r\n1141\r\n1142\r\n1143\r\n1144\r\n1145\r\n1146\r\n1147\r\n1148\r\n1149\r\n1150\r\n1151\r\n1152\r\n1153\r\n1154\r\n1155\r\n1156\r\n1157\r\n1158\r\n1159\r\n1160\r\n1161\r\n1162\r\n1163\r\n1164\r\n1165\r\n1166\r\n1167\r\n1168\r\n1169\r\n1170\r\n1171\r\n1172\r\n1173\r\n1174\r\n1175\r\n1176\r\n1177\r\n1178\r\n1179\r\n1180\r\n1181\r\n1182\r\n1183\r\n1184\r\n1185\r\n1186\r\n1187\r\n1188\r\n1189\r\n1190\r\n1191\r\n1192\r\n1193\r\n1194\r\n1195\r\n1196\r\n1197\r\n1198\r\n1199\r\n1200\r\n1201\r\n1202\r\n1203\r\n1204\r\n1205\r\n1206\r\n1207\r\n1208\r\n1209\r\n1210\r\n1211\r\n1212\r\n1213\r\n1214\r\n1215\r\n1216\r\n1217\r\n1218\r\n1219\r\n1220\r\n1221\r\n1222\r\n1223\r\n1224\r\n1225\r\n1226\r\n1227\r\n1228\r\n1229\r\n1230\r\n1231\r\n1232\r\n1233\r\n1234\r\n1235\r\n1236\r\n1237\r\n1238\r\n1239\r\n1240\r\n1241\r\n1242\r\n1243\r\n1244\r\n1245\r\n1246\r\n1247\r\n1248\r\n1249\r\n1250\r\n1251\r\n1252\r\n1253\r\n1254\r\n1255\r\n1256\r\n1257\r\n1258\r\n1259\r\n1260\r\n1261\r\n1262\r\n1263\r\n1264\r\n1265\r\n1266\r\n1267\r\n1268\r\n1269\r\n1270\r\n1271\r\n1272\r\n1273\r\n1274\r\n1275\r\n1276\r\n1277\r\n1278\r\n1279\r\n1280\r\n1281\r\n1282\r\n1283\r\n1284\r\n1285\r\n1286\r\n1287\r\n1288\r\n1289\r\n1290\r\n1291\r\n1292\r\n1293\r\n1294\r\n1295\r\n1296\r\n1297\r\n1298\r\n1299\r\n1300\r\n1301\r\n1302\r\n1303\r\n1304\r\n1305\r\n1306\r\n1307\r\n1308\r\n1309\r\n1310\r\n1311\r\n1312\r\n1313\r\n1314\r\n1315\r\n1316\r\n1317\r\n1318\r\n1319\r\n1320\r\n1321\r\n1322\r\n1323\r\n1324\r\n1325\r\n1326\r\n1327\r\n1328\r\n1329\r\n1330\r\n1331\r\n1332\r\n1333\r\n1334\r\n1335\r\n1336\r\n1337\r\n1338\r\n1339\r\n1340\r\n1341\r\n1342\r\n1343\r\n1344\r\n1345\r\n1346\r\n1347\r\n1348\r\n1349\r\n1350\r\n1351\r\n1352\r\n1353\r\n1354\r\n1355\r\n1356\r\n1357\r\n1358\r\n1359\r\n1360\r\n1361\r\n1362\r\n1363\r\n1364\r\n1365\r\n1366\r\n1367\r\n1368\r\n1369\r\n1370\r\n1371\r\n1372\r\n1373\r\n1374\r\n1375\r\n1376\r\n1377\r\n1378\r\n1379\r\n1380\r\n1381\r\n1382\r\n1383\r\n1384\r\n1385\r\n1386\r\n1387\r\n1388\r\n1389\r\n1390\r\n1391\r\n1392\r\n1393\r\n1394\r\n1395\r\n1396\r\n1397\r\n1398\r\n1399\r\n1400\r\n1401\r\n1402\r\n1403\r\n1404\r\n1405\r\n1406\r\n1407\r\n1408\r\n1409\r\n1410\r\n1411\r\n1412\r\n1413\r\n1414\r\n1415\r\n1416\r\n1417\r\n1418\r\n1419\r\n1420\r\n1421\r\n1422\r\n1423\r\n1424\r\n1425\r\n1426\r\n1427\r\n1428\r\n1429\r\n1430\r\n1431\r\n1432\r\n1433\r\n1434\r\n1435\r\n1436\r\n1437\r\n1438\r\n1439\r\n1440\r\n1441\r\n1442\r\n1443\r\n1444\r\n1445\r\n1446\r\n1447\r\n1448\r\n1449\r\n1450\r\n1451\r\n1452\r\n1453\r\n1454\r\n1455\r\n1456\r\n1457\r\n1458\r\n1459\r\n1460\r\n1461\r\n1462\r\n1463\r\n1464\r\n1465\r\n1466\r\n1467\r\n1468\r\n1469\r\n1470\r\n1471\r\n1472\r\n1473\r\n1474\r\n1475\r\n1476\r\n1477\r\n1478\r\n1479\r\n1480\r\n1481\r\n1482\r\n1483\r\n1484\r\n1485\r\n1486\r\n1487\r\n1488\r\n1489\r\n1490\r\n1491\r\n1492\r\n1493\r\n1494\r\n1495\r\n1496\r\n1497\r\n1498\r\n1499\r\n1500\r\n1501\r\n1502\r\n1503\r\n1504\r\n1505\r\n1506\r\n1507\r\n1508\r\n1509\r\n1510\r\n1511\r\n1512\r\n1513\r\n1514\r\n1515\r\n1516\r\n1517\r\n1518\r\n1519\r\n1520\r\n1521\r\n1522\r\n1523\r\n1524\r\n1525\r\n1526\r\n1527\r\n1528\r\n1529\r\n1530\r\n1531\r\n1532\r\n1533\r\n1534\r\n1535\r\n1536\r\n1537\r\n1538\r\n1539\r\n1540\r\n1541\r\n1542\r\n1543\r\n1544\r\n1545\r\n1546\r\n1547\r\n1548\r\n1549\r\n1550\r\n1551\r\n1552\r\n1553\r\n1554\r\n1555\r\n1556\r\n1557\r\n1558\r\n1559\r\n1560\r\n1561\r\n1562\r\n1563\r\n1564\r\n1565\r\n1566\r\n1567\r\n1568\r\n1569\r\n1570\r\n1571\r\n1572\r\n1573\r\n1574\r\n1575\r\n1576\r\n1577\r\n1578\r\n1579\r\n1580\r\n1581\r\n1582\r\n1583\r\n1584\r\n1585\r\n1586\r\n1587\r\n1588\r\n1589\r\n1590\r\n1591\r\n1592\r\n1593\r\n1594\r\n1595\r\n1596\r\n1597\r\n1598\r\n1599\r\n1600\r\n1601\r\n1602\r\n1603\r\n1604\r\n1605\r\n1606\r\n1607\r\n1608\r\n1609\r\n1610\r\n1611\r\n1612\r\n1613\r\n1614\r\n1615\r\n1616\r\n1617\r\n1618\r\n1619\r\n1620\r\n1621\r\n1622\r\n1623\r\n1624\r\n1625\r\n1626\r\n1627\r\n1628\r\n1629\r\n1630\r\n1631\r\n1632\r\n1633\r\n1634\r\n1635\r\n1636\r\n1637\r\n1638\r\n1639\r\n1640\r\n1641\r\n1642\r\n1643\r\n1644\r\n1645\r\n1646\r\n1647\r\n1648\r\n1649\r\n1650\r\n1651\r\n1652\r\n1653\r\n1654\r\n1655\r\n1656\r\n1657\r\n1658\r\n1659\r\n1660\r\n1661\r\n1662\r\n1663\r\n1664\r\n1665\r\n1666\r\n1667\r\n1668\r\n1669\r\n1670\r\n1671\r\n1672\r\n1673\r\n1674\r\n1675\r\n1676\r\n1677\r\n1678\r\n1679\r\n1680\r\n1681\r\n1682\r\n1683\r\n1684\r\n1685\r\n1686\r\n1687\r\n1688\r\n1689\r\n1690\r\n1691\r\n1692\r\n1693\r\n1694\r\n1695\r\n1696\r\n1697\r\n1698\r\n1699\r\n1700\r\n1701\r\n1702\r\n1703\r\n1704\r\n1705\r\n1706\r\n1707\r\n1708\r\n1709\r\n1710\r\n1711\r\n1712\r\n1713\r\n1714\r\n1715\r\n1716\r\n1717\r\n1718\r\n1719\r\n1720\r\n1721\r\n1722\r\n1723\r\n1724\r\n1725\r\n1726\r\n1727\r\n1728\r\n1729\r\n1730\r\n1731\r\n1732\r\n1733\r\n1734\r\n1735\r\n1736\r\n1737\r\n1738\r\n1739\r\n1740\r\n1741\r\n1742\r\n1743\r\n1744\r\n1745\r\n1746\r\n1747\r\n1748\r\n1749\r\n1750\r\n1751\r\n1752\r\n1753\r\n1754\r\n1755\r\n1756\r\n1757\r\n1758\r\n1759\r\n1760\r\n1761\r\n1762\r\n1763\r\n1764\r\n1765\r\n1766\r\n1767\r\n1768\r\n1769\r\n1770\r\n1771\r\n1772\r\n1773\r\n1774\r\n1775\r\n1776\r\n1777\r\n1778\r\n1779\r\n1780\r\n1781\r\n1782\r\n1783\r\n1784\r\n1785\r\n1786\r\n1787\r\n1788\r\n1789\r\n1790\r\n1791\r\n1792\r\n1793\r\n1794\r\n1795\r\n1796\r\n1797\r\n1798\r\n1799\r\n1800\r\n1801\r\n1802\r\n1803\r\n1804\r\n1805\r\n1806\r\n1807\r\n1808\r\n1809\r\n1810\r\n1811\r\n1812\r\n1813\r\n1814\r\n1815\r\n1816\r\n1817\r\n1818\r\n1819\r\n1820\r\n1821\r\n1822\r\n1823\r\n1824\r\n1825\r\n1826\r\n1827\r\n1828\r\n1829\r\n1830\r\n1831\r\n1832\r\n1833\r\n1834\r\n1835\r\n1836\r\n1837\r\n1838\r\n1839\r\n1840\r\n1841\r\n1842\r\n1843\r\n1844\r\n1845\r\n1846\r\n1847\r\n1848\r\n1849\r\n1850\r\n1851\r\n1852\r\n1853\r\n1854\r\n1855\r\n1856\r\n1857\r\n1858\r\n1859\r\n1860\r\n1861\r\n1862\r\n1863\r\n1864\r\n1865\r\n1866\r\n1867\r\n1868\r\n1869\r\n1870\r\n1871\r\n1872\r\n1873\r\n1874\r\n1875\r\n1876\r\n1877\r\n1878\r\n1879\r\n1880\r\n1881\r\n1882\r\n1883\r\n1884\r\n1885\r\n1886\r\n1887\r\n1888\r\n1889\r\n1890\r\n1891\r\n1892\r\n1893\r\n1894\r\n1895\r\n1896\r\n1897\r\n1898\r\n1899\r\n1900\r\n1901\r\n1902\r\n1903\r\n1904\r\n1905\r\n1906\r\n1907\r\n1908\r\n1909\r\n1910\r\n1911\r\n1912\r\n1913\r\n1914\r\n1915\r\n1916\r\n1917\r\n1918\r\n1919\r\n1920\r\n1921\r\n1922\r\n1923\r\n1924\r\n1925\r\n1926\r\n1927\r\n1928\r\n1929\r\n1930\r\n1931\r\n1932\r\n1933\r\n1934\r\n1935\r\n1936\r\n1937\r\n1938\r\n1939\r\n1940\r\n1941\r\n1942\r\n1943\r\n1944\r\n1945\r\n1946\r\n1947\r\n1948\r\n1949\r\n1950\r\n1951\r\n1952\r\n1953\r\n1954\r\n1955\r\n1956\r\n1957\r\n1958\r\n1959\r\n1960\r\n1961\r\n1962\r\n1963\r\n1964\r\n1965\r\n1966\r\n1967\r\n1968\r\n1969\r\n1970\r\n1971\r\n1972\r\n1973\r\n1974\r\n1975\r\n1976\r\n1977\r\n1978\r\n1979\r\n1980\r\n1981\r\n1982\r\n1983\r\n1984\r\n1985\r\n1986\r\n1987\r\n1988\r\n1989\r\n1990\r\n1991\r\n1992\r\n1993\r\n1994\r\n1995\r\n1996\r\n1997\r\n1998\r\n1999\r\n2000\r\n2001\r\n2002\r\n2003\r\n2004\r\n2005\r\n2006\r\n2007\r\n2008\r\n2009\r\n2010\r\n2011\r\n2012\r\n2013\r\n2014\r\n2015\r\n2016\r\n2017\r\n2018\r\n2019\r\n2020\r\n2021\r\n2022\r\n2023\r\n2024\r\n2025\r\n2026\r\n2027\r\n2028\r\n2029\r\n2030\r\n2031\r\n2032\r\n2033\r\n2034\r\n2035\r\n2036\r\n2037\r\n2038\r\n2039\r\n2040\r\n2041\r\n2042\r\n2043\r\n2044\r\n2045\r\n2046\r\n2047\r\n2048\r\n2049\r\n2050\r\n2051\r\n2052\r\n2053\r\n2054\r\n2055\r\n2056\r\n2057\r\n2058\r\n2059\r\n2060\r\n2061\r\n2062\r\n2063\r\n2064\r\n2065\r\n2066\r\n2067\r\n2068\r\n2069\r\n2070\r\n2071\r\n2072\r\n2073\r\n2074\r\n2075\r\n2076\r\n2077\r\n2078\r\n2079\r\n2080\r\n2081\r\n2082\r\n2083\r\n2084\r\n2085\r\n2086\r\n2087\r\n2088\r\n2089\r\n2090\r\n2091\r\n2092\r\n2093\r\n2094\r\n2095\r\n2096\r\n2097\r\n2098\r\n2099\r\n2100\r\n2101\r\n2102\r\n2103\r\n2104\r\n2105\r\n2106\r\n2107\r\n2108\r\n2109\r\n2110\r\n2111\r\n2112\r\n2113\r\n2114\r\n2115\r\n2116\r\n2117\r\n2118\r\n2119\r\n2120\r\n2121\r\n2122\r\n2123\r\n2124\r\n2125\r\n2126\r\n2127\r\n2128\r\n2129\r\n2130\r\n2131\r\n2132\r\n2133\r\n2134\r\n2135\r\n2136\r\n2137\r\n2138\r\n2139\r\n2140\r\n2141\r\n2142\r\n2143\r\n2144\r\n2145\r\n2146\r\n2147\r\n2148\r\n2149\r\n2150\r\n2151\r\n2152\r\n2153\r\n2154\r\n2155\r\n2156\r\n2157\r\n2158\r\n2159\r\n2160\r\n2161\r\n2162\r\n2163\r\n2164\r\n2165\r\n2166\r\n2167\r\n2168\r\n2169\r\n2170\r\n2171\r\n2172\r\n2173\r\n2174\r\n2175\r\n2176\r\n2177\r\n2178\r\n2179\r\n2180\r\n2181\r\n2182\r\n2183\r\n2184\r\n2185\r\n2186\r\n2187\r\n2188\r\n2189\r\n2190\r\n2191\r\n2192\r\n2193\r\n2194\r\n2195\r\n2196\r\n2197\r\n2198\r\n2199\r\n2200\r\n2201\r\n2202\r\n2203\r\n2204\r\n2205\r\n2206\r\n2207\r\n2208\r\n2209\r\n2210\r\n2211\r\n2212\r\n2213\r\n2214\r\n2215\r\n2216\r\n2217\r\n2218\r\n2219\r\n2220\r\n2221\r\n2222\r\n2223\r\n2224\r\n2225\r\n2226\r\n2227\r\n2228\r\n2229\r\n2230\r\n2231\r\n2232\r\n2233\r\n2234\r\n2235\r\n2236\r\n2237\r\n2238\r\n2239\r\n2240\r\n2241\r\n2242\r\n2243\r\n2244\r\n2245\r\n2246\r\n2247\r\n2248\r\n2249\r\n2250\r\n2251\r\n2252\r\n2253\r\n2254\r\n2255\r\n2256\r\n2257\r\n2258\r\n2259\r\n2260\r\n2261\r\n2262\r\n2263\r\n2264\r\n2265\r\n2266\r\n2267\r\n2268\r\n2269\r\n2270\r\n2271\r\n2272\r\n2273\r\n2274\r\n2275\r\n2276\r\n2277\r\n2278\r\n2279\r\n2280\r\n2281\r\n2282\r\n2283\r\n2284\r\n2285\r\n2286\r\n2287\r\n2288\r\n2289\r\n2290\r\n2291\r\n2292\r\n2293\r\n2294\r\n2295\r\n2296\r\n2297\r\n2298\r\n2299\r\n2300\r\n2301\r\n2302\r\n2303\r\n2304\r\n2305\r\n2306\r\n2307\r\n2308\r\n2309\r\n2310\r\n2311\r\n2312\r\n2313\r\n2314\r\n2315\r\n2316\r\n2317\r\n2318\r\n2319\r\n2320\r\n2321\r\n2322\r\n2323\r\n2324\r\n2325\r\n2326\r\n2327\r\n2328\r\n2329\r\n2330\r\n2331\r\n2332\r\n2333\r\n2334\r\n2335\r\n2336\r\n2337\r\n2338\r\n2339\r\n2340\r\n2341\r\n2342\r\n2343\r\n2344\r\n2345\r\n2346\r\n2347\r\n2348\r\n2349\r\n2350\r\n2351\r\n2352\r\n2353\r\n2354\r\n2355\r\n2356\r\n2357\r\n2358\r\n2359\r\n2360\r\n2361\r\n2362\r\n2363\r\n2364\r\n2365\r\n2366\r\n2367\r\n2368\r\n2369\r\n2370\r\n2371\r\n2372\r\n2373\r\n2374\r\n2375\r\n2376\r\n2377\r\n2378\r\n2379\r\n2380\r\n2381\r\n2382\r\n2383\r\n2384\r\n2385\r\n2386\r\n2387\r\n2388\r\n2389\r\n2390\r\n2391\r\n2392\r\n2393\r\n2394\r\n2395\r\n2396\r\n2397\r\n2398\r\n2399\r\n2400\r\n2401\r\n2402\r\n2403\r\n2404\r\n2405\r\n2406\r\n2407\r\n2408\r\n2409\r\n2410\r\n2411\r\n2412\r\n2413\r\n2414\r\n2415\r\n2416\r\n2417\r\n2418\r\n2419\r\n2420\r\n2421\r\n2422\r\n2423\r\n2424\r\n2425\r\n2426\r\n2427\r\n2428\r\n2429\r\n2430\r\n2431\r\n2432\r\n2433\r\n2434\r\n2435\r\n2436\r\n2437\r\n2438\r\n2439\r\n2440\r\n2441\r\n2442\r\n2443\r\n2444\r\n2445\r\n2446\r\n2447\r\n2448\r\n2449\r\n2450\r\n2451\r\n2452\r\n2453\r\n2454\r\n2455\r\n2456\r\n2457\r\n2458\r\n2459\r\n2460\r\n2461\r\n2462\r\n2463\r\n2464\r\n2465\r\n2466\r\n2467\r\n2468\r\n2469\r\n2470\r\n2471\r\n2472\r\n2473\r\n2474\r\n2475\r\n2476\r\n2477\r\n2478\r\n2479\r\n2480\r\n2481\r\n2482\r\n2483\r\n2484\r\n2485\r\n2486\r\n2487\r\n2488\r\n2489\r\n2490\r\n2491\r\n2492\r\n2493\r\n2494\r\n2495\r\n2496\r\n2497\r\n2498\r\n2499\r\n2500\r\n2501\r\n2502\r\n2503\r\n2504\r\n2505\r\n2506\r\n2507\r\n2508\r\n2509\r\n2510\r\n2511\r\n2512\r\n2513\r\n2514\r\n2515\r\n2516\r\n2517\r\n2518\r\n2519\r\n2520\r\n2521\r\n2522\r\n2523\r\n2524\r\n2525\r\n2526\r\n2527\r\n2528\r\n2529\r\n2530\r\n2531\r\n2532\r\n2533\r\n2534\r\n2535\r\n2536\r\n2537\r\n2538\r\n2539\r\n2540\r\n2541\r\n2542\r\n2543\r\n2544\r\n2545\r\n2546\r\n2547\r\n2548\r\n2549\r\n2550\r\n2551\r\n2552\r\n2553\r\n2554\r\n2555\r\n2556\r\n2557\r\n2558\r\n2559\r\n2560\r\n2561\r\n2562\r\n2563\r\n2564\r\n2565\r\n2566\r\n2567\r\n2568\r\n2569\r\n2570\r\n2571\r\n2572\r\n2573\r\n2574\r\n2575\r\n2576\r\n2577\r\n2578\r\n2579\r\n2580\r\n2581\r\n2582\r\n2583\r\n2584\r\n2585\r\n2586\r\n2587\r\n2588\r\n2589\r\n2590\r\n2591\r\n2592\r\n2593\r\n2594\r\n2595\r\n2596\r\n2597\r\n2598\r\n2599\r\n2600\r\n2601\r\n2602\r\n2603\r\n2604\r\n2605\r\n2606\r\n2607\r\n2608\r\n2609\r\n2610\r\n2611\r\n2612\r\n2613\r\n2614\r\n2615\r\n2616\r\n2617\r\n2618\r\n2619\r\n2620\r\n2621\r\n2622\r\n2623\r\n2624\r\n2625\r\n2626\r\n2627\r\n2628\r\n2629\r\n2630\r\n2631\r\n2632\r\n2633\r\n2634\r\n2635\r\n2636\r\n2637\r\n2638\r\n2639\r\n2640\r\n2641\r\n2642\r\n2643\r\n2644\r\n2645\r\n2646\r\n2647\r\n2648\r\n2649\r\n2650\r\n2651\r\n2652\r\n2653\r\n2654\r\n2655\r\n2656\r\n2657\r\n2658\r\n2659\r\n2660\r\n2661\r\n2662\r\n2663\r\n2664\r\n2665\r\n2666\r\n2667\r\n2668\r\n2669\r\n2670\r\n2671\r\n2672\r\n2673\r\n2674\r\n2675\r\n2676\r\n2677\r\n2678\r\n2679\r\n2680\r\n2681\r\n2682\r\n2683\r\n2684\r\n2685\r\n2686\r\n2687\r\n2688\r\n2689\r\n2690\r\n2691\r\n2692\r\n2693\r\n2694\r\n2695\r\n2696\r\n2697\r\n2698\r\n2699\r\n2700\r\n2701\r\n2702\r\n2703\r\n2704\r\n2705\r\n2706\r\n2707\r\n2708\r\n2709\r\n2710\r\n2711\r\n2712\r\n2713\r\n2714\r\n2715\r\n2716\r\n2717\r\n2718\r\n2719\r\n2720\r\n2721\r\n2722\r\n2723\r\n2724\r\n2725\r\n2726\r\n2727\r\n2728\r\n2729\r\n2730\r\n2731\r\n2732\r\n2733\r\n2734\r\n2735\r\n2736\r\n2737\r\n2738\r\n2739\r\n2740\r\n2741\r\n2742\r\n2743\r\n2744\r\n2745\r\n2746\r\n2747\r\n2748\r\n2749\r\n2750\r\n2751\r\n2752\r\n2753\r\n2754\r\n2755\r\n2756\r\n2757\r\n2758\r\n2759\r\n2760\r\n2761\r\n2762\r\n2763\r\n2764\r\n2765\r\n2766\r\n2767\r\n2768\r\n2769\r\n2770\r\n2771\r\n2772\r\n2773\r\n2774\r\n2775\r\n2776\r\n2777\r\n2778\r\n2779\r\n2780\r\n2781\r\n2782\r\n2783\r\n2784\r\n2785\r\n2786\r\n2787\r\n2788\r\n2789\r\n2790\r\n2791\r\n2792\r\n2793\r\n2794\r\n2795\r\n2796\r\n2797\r\n2798\r\n2799\r\n2800\r\n2801\r\n2802\r\n2803\r\n2804\r\n2805\r\n2806\r\n2807\r\n2808\r\n2809\r\n2810\r\n2811\r\n2812\r\n2813\r\n2814\r\n2815\r\n2816\r\n2817\r\n2818\r\n2819\r\n2820\r\n2821\r\n2822\r\n2823\r\n2824\r\n2825\r\n2826\r\n2827\r\n2828\r\n2829\r\n2830\r\n2831\r\n2832\r\n2833\r\n2834\r\n2835\r\n2836\r\n2837\r\n2838\r\n2839\r\n2840\r\n2841\r\n2842\r\n2843\r\n2844\r\n2845\r\n2846\r\n2847\r\n2848\r\n2849\r\n2850\r\n2851\r\n2852\r\n2853\r\n2854\r\n2855\r\n2856\r\n2857\r\n2858\r\n2859\r\n2860\r\n2861\r\n2862\r\n2863\r\n2864\r\n2865\r\n2866\r\n2867\r\n2868\r\n2869\r\n2870\r\n2871\r\n2872\r\n2873\r\n2874\r\n2875\r\n2876\r\n2877\r\n2878\r\n2879\r\n2880\r\n2881\r\n2882\r\n2883\r\n2884\r\n2885\r\n2886\r\n2887\r\n2888\r\n2889\r\n2890\r\n2891\r\n2892\r\n2893\r\n2894\r\n2895\r\n2896\r\n2897\r\n2898\r\n2899\r\n2900\r\n2901\r\n2902\r\n2903\r\n2904\r\n2905\r\n2906\r\n2907\r\n2908\r\n2909\r\n2910\r\n2911\r\n2912\r\n2913\r\n2914\r\n2915\r\n2916\r\n2917\r\n2918\r\n2919\r\n2920\r\n2921\r\n2922\r\n2923\r\n2924\r\n2925\r\n2926\r\n2927\r\n2928\r\n2929\r\n2930\r\n2931\r\n2932\r\n2933\r\n2934\r\n2935\r\n2936\r\n2937\r\n2938\r\n2939\r\n2940\r\n2941\r\n2942\r\n2943\r\n2944\r\n2945\r\n2946\r\n2947\r\n2948\r\n2949\r\n2950\r\n2951\r\n2952\r\n2953\r\n2954\r\n2955\r\n2956\r\n2957\r\n2958\r\n2959\r\n2960\r\n2961\r\n2962\r\n2963\r\n2964\r\n2965\r\n2966\r\n2967\r\n2968\r\n2969\r\n2970\r\n2971\r\n2972\r\n2973\r\n2974\r\n2975\r\n2976\r\n2977\r\n2978\r\n2979\r\n2980\r\n2981\r\n2982\r\n2983\r\n2984\r\n2985\r\n2986\r\n2987\r\n2988\r\n2989\r\n2990\r\n2991\r\n2992\r\n2993\r\n2994\r\n2995\r\n2996\r\n2997\r\n2998\r\n2999\r\n3000\r\n3001\r\n3002\r\n3003\r\n3004\r\n3005\r\n3006\r\n3007\r\n3008\r\n3009\r\n3010\r\n3011\r\n3012\r\n3013\r\n3014\r\n3015\r\n3016\r\n3017\r\n3018\r\n3019\r\n3020\r\n3021\r\n3022\r\n3023\r\n3024\r\n3025\r\n3026\r\n3027\r\n3028\r\n3029\r\n3030\r\n3031\r\n3032\r\n3033\r\n3034\r\n3035\r\n3036\r\n3037\r\n3038\r\n3039\r\n3040\r\n3041\r\n3042\r\n3043\r\n3044\r\n3045\r\n3046\r\n3047\r\n3048\r\n3049\r\n3050\r\n3051\r\n3052\r\n3053\r\n3054\r\n3055\r\n3056\r\n3057\r\n3058\r\n3059\r\n3060\r\n3061\r\n3062\r\n3063\r\n3064\r\n3065\r\n3066\r\n3067\r\n3068\r\n3069\r\n3070\r\n3071\r\n3072\r\n3073\r\n3074\r\n3075\r\n3076\r\n3077\r\n3078\r\n3079\r\n3080\r\n3081\r\n3082\r\n3083\r\n3084\r\n3085\r\n3086\r\n3087\r\n3088\r\n3089\r\n3090\r\n3091\r\n3092\r\n3093\r\n3094\r\n3095\r\n3096\r\n3097\r\n3098\r\n3099\r\n3100\r\n3101\r\n3102\r\n3103\r\n3104\r\n3105\r\n3106\r\n3107\r\n3108\r\n3109\r\n3110\r\n3111\r\n3112\r\n3113\r\n3114\r\n3115\r\n3116\r\n3117\r\n3118\r\n3119\r\n3120\r\n3121\r\n3122\r\n3123\r\n3124\r\n3125\r\n3126\r\n3127\r\n3128\r\n3129\r\n3130\r\n3131\r\n3132\r\n3133\r\n3134\r\n3135\r\n3136\r\n3137\r\n3138\r\n3139\r\n3140\r\n3141\r\n3142\r\n3143\r\n3144\r\n3145\r\n3146\r\n3147\r\n3148\r\n3149\r\n3150\r\n3151\r\n3152\r\n3153\r\n3154\r\n3155\r\n3156\r\n3157\r\n3158\r\n3159\r\n3160\r\n3161\r\n3162\r\n3163\r\n3164\r\n3165\r\n3166\r\n3167\r\n3168\r\n3169\r\n3170\r\n3171\r\n3172\r\n3173\r\n3174\r\n3175\r\n3176\r\n3177\r\n3178\r\n3179\r\n3180\r\n3181\r\n3182\r\n3183\r\n3184\r\n3185\r\n3186\r\n3187\r\n3188\r\n3189\r\n3190\r\n3191\r\n3192\r\n3193\r\n3194\r\n3195\r\n3196\r\n3197\r\n3198\r\n3199\r\n3200\r\n3201\r\n3202\r\n3203\r\n3204\r\n3205\r\n3206\r\n3207\r\n3208\r\n3209\r\n3210\r\n3211\r\n3212\r\n3213\r\n3214\r\n3215\r\n3216\r\n3217\r\n3218\r\n3219\r\n3220\r\n3221\r\n3222\r\n3223\r\n3224\r\n3225\r\n3226\r\n3227\r\n3228\r\n3229\r\n3230\r\n3231\r\n3232\r\n3233\r\n3234\r\n3235\r\n3236\r\n3237\r\n3238\r\n3239\r\n3240\r\n3241\r\n3242\r\n3243\r\n3244\r\n3245\r\n3246\r\n3247\r\n3248\r\n3249\r\n3250\r\n3251\r\n3252\r\n3253\r\n3254\r\n3255\r\n3256\r\n3257\r\n3258\r\n3259\r\n3260\r\n3261\r\n3262\r\n3263\r\n3264\r\n3265\r\n3266\r\n3267\r\n3268\r\n3269\r\n3270\r\n3271\r\n3272\r\n3273\r\n3274\r\n3275\r\n3276\r\n3277\r\n3278\r\n3279\r\n3280\r\n3281\r\n3282\r\n3283\r\n3284\r\n3285\r\n3286\r\n3287\r\n3288\r\n3289\r\n3290\r\n3291\r\n3292\r\n3293\r\n3294\r\n3295\r\n3296\r\n3297\r\n3298\r\n3299\r\n3300\r\n3301\r\n3302\r\n3303\r\n3304\r\n3305\r\n3306\r\n3307\r\n3308\r\n3309\r\n3310\r\n3311\r\n3312\r\n3313\r\n3314\r\n3315\r\n3316\r\n3317\r\n3318\r\n3319\r\n3320\r\n3321\r\n3322\r\n3323\r\n3324\r\n3325\r\n3326\r\n3327\r\n3328\r\n3329\r\n3330\r\n3331\r\n3332\r\n3333\r\n3334\r\n3335\r\n3336\r\n3337\r\n3338\r\n3339\r\n3340\r\n3341\r\n3342\r\n3343\r\n3344\r\n3345\r\n3346\r\n3347\r\n3348\r\n3349\r\n3350\r\n3351\r\n3352\r\n3353\r\n3354\r\n3355\r\n3356\r\n3357\r\n3358\r\n3359\r\n3360\r\n3361\r\n3362\r\n3363\r\n3364\r\n3365\r\n3366\r\n3367\r\n3368\r\n3369\r\n3370\r\n3371\r\n3372\r\n3373\r\n3374\r\n3375\r\n3376\r\n3377\r\n3378\r\n3379\r\n3380\r\n3381\r\n3382\r\n3383\r\n3384\r\n3385\r\n3386\r\n3387\r\n3388\r\n3389\r\n3390\r\n3391\r\n3392\r\n3393\r\n3394\r\n3395\r\n3396\r\n3397\r\n3398\r\n3399\r\n3400\r\n3401\r\n3402\r\n3403\r\n3404\r\n3405\r\n3406\r\n3407\r\n3408\r\n3409\r\n3410\r\n3411\r\n3412\r\n3413\r\n3414\r\n3415\r\n3416\r\n3417\r\n3418\r\n3419\r\n3420\r\n3421\r\n3422\r\n3423\r\n3424\r\n3425\r\n3426\r\n3427\r\n3428\r\n3429\r\n3430\r\n3431\r\n3432\r\n3433\r\n3434\r\n3435\r\n3436\r\n3437\r\n3438\r\n3439\r\n3440\r\n3441\r\n3442\r\n3443\r\n3444\r\n3445\r\n3446\r\n3447\r\n3448\r\n3449\r\n3450\r\n3451\r\n3452\r\n3453\r\n3454\r\n3455\r\n3456\r\n3457\r\n3458\r\n3459\r\n3460\r\n3461\r\n3462\r\n3463\r\n3464\r\n3465\r\n3466\r\n3467\r\n3468\r\n3469\r\n3470\r\n3471\r\n3472\r\n3473\r\n3474\r\n3475\r\n3476\r\n3477\r\n3478\r\n3479\r\n3480\r\n3481\r\n3482\r\n3483\r\n3484\r\n3485\r\n3486\r\n3487\r\n3488\r\n3489\r\n3490\r\n3491\r\n3492\r\n3493\r\n3494\r\n3495\r\n3496\r\n3497\r\n3498\r\n3499\r\n3500\r\n3501\r\n3502\r\n3503\r\n3504\r\n3505\r\n3506\r\n3507\r\n3508\r\n3509\r\n3510\r\n3511\r\n3512\r\n3513\r\n3514\r\n3515\r\n3516\r\n3517\r\n3518\r\n3519\r\n3520\r\n3521\r\n3522\r\n3523\r\n3524\r\n3525\r\n3526\r\n3527\r\n3528\r\n3529\r\n3530\r\n3531\r\n3532\r\n3533\r\n3534\r\n3535\r\n3536\r\n3537\r\n3538\r\n3539\r\n3540\r\n3541\r\n3542\r\n3543\r\n3544\r\n3545\r\n3546\r\n3547\r\n3548\r\n3549\r\n3550\r\n3551\r\n3552\r\n3553\r\n3554\r\n3555\r\n3556\r\n3557\r\n3558\r\n3559\r\n3560\r\n3561\r\n3562\r\n3563\r\n3564\r\n3565\r\n3566\r\n3567\r\n3568\r\n3569\r\n3570\r\n3571\r\n3572\r\n3573\r\n3574\r\n3575\r\n3576\r\n3577\r\n3578\r\n3579\r\n3580\r\n3581\r\n3582\r\n3583\r\n3584\r\n3585\r\n3586\r\n3587\r\n3588\r\n3589\r\n3590\r\n3591\r\n3592\r\n3593\r\n3594\r\n3595\r\n3596\r\n3597\r\n3598\r\n3599\r\n3600\r\n3601\r\n3602\r\n3603\r\n3604\r\n3605\r\n3606\r\n3607\r\n3608\r\n3609\r\n3610\r\n3611\r\n3612\r\n3613\r\n3614\r\n3615\r\n3616\r\n3617\r\n3618\r\n3619\r\n3620\r\n3621\r\n3622\r\n3623\r\n3624\r\n3625\r\n3626\r\n3627\r\n3628\r\n3629\r\n3630\r\n3631\r\n3632\r\n3633\r\n3634\r\n3635\r\n3636\r\n3637\r\n3638\r\n3639\r\n3640\r\n3641\r\n3642\r\n3643\r\n3644\r\n3645\r\n3646\r\n3647\r\n3648\r\n3649\r\n3650\r\n3651\r\n3652\r\n3653\r\n3654\r\n3655\r\n3656\r\n3657\r\n3658\r\n3659\r\n3660\r\n3661\r\n3662\r\n3663\r\n3664\r\n3665\r\n3666\r\n3667\r\n3668\r\n3669\r\n3670\r\n3671\r\n3672\r\n3673\r\n3674\r\n3675\r\n3676\r\n3677\r\n3678\r\n3679\r\n3680\r\n3681\r\n3682\r\n3683\r\n3684\r\n3685\r\n3686\r\n3687\r\n3688\r\n3689\r\n3690\r\n3691\r\n3692\r\n3693\r\n3694\r\n3695\r\n3696\r\n3697\r\n3698\r\n3699\r\n3700\r\n3701\r\n3702\r\n3703\r\n3704\r\n3705\r\n3706\r\n3707\r\n3708\r\n3709\r\n3710\r\n3711\r\n3712\r\n3713\r\n3714\r\n3715\r\n3716\r\n3717\r\n3718\r\n3719\r\n3720\r\n3721\r\n3722\r\n3723\r\n3724\r\n3725\r\n3726\r\n3727\r\n3728\r\n3729\r\n3730\r\n3731\r\n3732\r\n3733\r\n3734\r\n3735\r\n3736\r\n3737\r\n3738\r\n3739\r\n3740\r\n3741\r\n3742\r\n3743\r\n3744\r\n3745\r\n3746\r\n3747\r\n3748\r\n3749\r\n3750\r\n3751\r\n3752\r\n3753\r\n3754\r\n3755\r\n3756\r\n3757\r\n3758\r\n3759\r\n3760\r\n3761\r\n3762\r\n3763\r\n3764\r\n3765\r\n3766\r\n3767\r\n3768\r\n3769\r\n3770\r\n3771\r\n3772\r\n3773\r\n3774\r\n3775\r\n3776\r\n3777\r\n3778\r\n3779\r\n3780\r\n3781\r\n3782\r\n3783\r\n3784\r\n3785\r\n3786\r\n3787\r\n3788\r\n3789\r\n3790\r\n3791\r\n3792\r\n3793\r\n3794\r\n3795\r\n3796\r\n3797\r\n3798\r\n3799\r\n3800\r\n3801\r\n3802\r\n3803\r\n3804\r\n3805\r\n3806\r\n3807\r\n3808\r\n3809\r\n3810\r\n3811\r\n3812\r\n3813\r\n3814\r\n3815\r\n3816\r\n3817\r\n3818\r\n3819\r\n3820\r\n3821\r\n3822\r\n3823\r\n3824\r\n3825\r\n3826\r\n3827\r\n3828\r\n3829\r\n3830\r\n3831\r\n3832\r\n3833\r\n3834\r\n3835\r\n3836\r\n3837\r\n3838\r\n3839\r\n3840\r\n3841\r\n3842\r\n3843\r\n3844\r\n3845\r\n3846\r\n3847\r\n3848\r\n3849\r\n3850\r\n3851\r\n3852\r\n3853\r\n3854\r\n3855\r\n3856\r\n3857\r\n3858\r\n3859\r\n3860\r\n3861\r\n3862\r\n3863\r\n3864\r\n3865\r\n3866\r\n3867\r\n3868\r\n3869\r\n3870\r\n3871\r\n3872\r\n3873\r\n3874\r\n3875\r\n3876\r\n3877\r\n3878\r\n3879\r\n3880\r\n3881\r\n3882\r\n3883\r\n3884\r\n3885\r\n3886\r\n3887\r\n3888\r\n3889\r\n3890\r\n3891\r\n3892\r\n3893\r\n3894\r\n3895\r\n3896\r\n3897\r\n3898\r\n3899\r\n3900\r\n3901\r\n3902\r\n3903\r\n3904\r\n3905\r\n3906\r\n3907\r\n3908\r\n3909\r\n3910\r\n3911\r\n3912\r\n3913\r\n3914\r\n3915\r\n3916\r\n3917\r\n3918\r\n3919\r\n3920\r\n3921\r\n3922\r\n3923\r\n3924\r\n3925\r\n3926\r\n3927\r\n3928\r\n3929\r\n3930\r\n3931\r\n3932\r\n3933\r\n3934\r\n3935\r\n3936\r\n3937\r\n3938\r\n3939\r\n3940\r\n3941\r\n3942\r\n3943\r\n3944\r\n3945\r\n3946\r\n3947\r\n3948\r\n3949\r\n3950\r\n3951\r\n3952\r\n3953\r\n3954\r\n3955\r\n3956\r\n3957\r\n3958\r\n3959\r\n3960\r\n3961\r\n3962\r\n3963\r\n3964\r\n3965\r\n3966\r\n3967\r\n3968\r\n3969\r\n3970\r\n3971\r\n3972\r\n3973\r\n3974\r\n3975\r\n3976\r\n3977\r\n3978\r\n3979\r\n3980\r\n3981\r\n3982\r\n3983\r\n3984\r\n3985\r\n3986\r\n3987\r\n3988\r\n3989\r\n3990\r\n3991\r\n3992\r\n3993\r\n3994\r\n3995\r\n3996\r\n3997\r\n3998\r\n3999\r\n4000\r\n4001\r\n4002\r\n4003\r\n4004\r\n4005\r\n4006\r\n4007\r\n4008\r\n4009\r\n4010\r\n4011\r\n4012\r\n4013\r\n4014\r\n4015\r\n4016\r\n4017\r\n4018\r\n4019\r\n4020\r\n4021\r\n4022\r\n4023\r\n4024\r\n4025\r\n4026\r\n4027\r\n4028\r\n4029\r\n4030\r\n4031\r\n4032\r\n4033\r\n4034\r\n4035\r\n4036\r\n4037\r\n4038\r\n4039\r\n4040\r\n4041\r\n4042\r\n4043\r\n4044\r\n4045\r\n4046\r\n4047\r\n4048\r\n4049\r\n4050\r\n4051\r\n4052\r\n4053\r\n4054\r\n4055\r\n4056\r\n4057\r\n4058\r\n4059\r\n4060\r\n4061\r\n4062\r\n4063\r\n4064\r\n4065\r\n4066\r\n4067\r\n4068\r\n4069\r\n4070\r\n4071\r\n4072\r\n4073\r\n4074\r\n4075\r\n4076\r\n4077\r\n4078\r\n4079\r\n4080\r\n4081\r\n4082\r\n4083\r\n4084\r\n4085\r\n4086\r\n4087\r\n4088\r\n4089\r\n4090\r\n4091\r\n4092\r\n4093\r\n4094\r\n4095\r\n4096\r\n4097\r\n4098\r\n4099\r\n4100\r\n4101\r\n4102\r\n4103\r\n4104\r\n4105\r\n4106\r\n4107\r\n4108\r\n4109\r\n4110\r\n4111\r\n4112\r\n4113\r\n4114\r\n4115\r\n4116\r\n4117\r\n4118\r\n4119\r\n4120\r\n4121\r\n4122\r\n4123\r\n4124\r\n4125\r\n4126\r\n4127\r\n4128\r\n4129\r\n4130\r\n4131\r\n4132\r\n4133\r\n4134\r\n4135\r\n4136\r\n4137\r\n4138\r\n4139\r\n4140\r\n4141\r\n4142\r\n4143\r\n4144\r\n4145\r\n4146\r\n4147\r\n4148\r\n4149\r\n4150\r\n4151\r\n4152\r\n4153\r\n4154\r\n4155\r\n4156\r\n4157\r\n4158\r\n4159\r\n4160\r\n4161\r\n4162\r\n4163\r\n4164\r\n4165\r\n4166\r\n4167\r\n4168\r\n4169\r\n4170\r\n4171\r\n4172\r\n4173\r\n4174\r\n4175\r\n4176\r\n4177\r\n4178\r\n4179\r\n4180\r\n4181\r\n4182\r\n4183\r\n4184\r\n4185\r\n4186\r\n4187\r\n4188\r\n4189\r\n4190\r\n4191\r\n4192\r\n4193\r\n4194\r\n4195\r\n4196\r\n4197\r\n4198\r\n4199\r\n4200\r\n4201\r\n4202\r\n4203\r\n4204\r\n4205\r\n4206\r\n4207\r\n4208\r\n4209\r\n4210\r\n4211\r\n4212\r\n4213\r\n4214\r\n4215\r\n4216\r\n4217\r\n4218\r\n4219\r\n4220\r\n4221\r\n4222\r\n4223\r\n4224\r\n4225\r\n4226\r\n4227\r\n4228\r\n4229\r\n4230\r\n4231\r\n4232\r\n4233\r\n4234\r\n4235\r\n4236\r\n4237\r\n4238\r\n4239\r\n4240\r\n4241\r\n4242\r\n4243\r\n4244\r\n4245\r\n4246\r\n4247\r\n4248\r\n4249\r\n4250\r\n4251\r\n4252\r\n4253\r\n4254\r\n4255\r\n4256\r\n4257\r\n4258\r\n4259\r\n4260\r\n4261\r\n4262\r\n4263\r\n4264\r\n4265\r\n4266\r\n4267\r\n4268\r\n4269\r\n4270\r\n4271\r\n4272\r\n4273\r\n4274\r\n4275\r\n4276\r\n4277\r\n4278\r\n4279\r\n4280\r\n4281\r\n4282\r\n4283\r\n4284\r\n4285\r\n4286\r\n4287\r\n4288\r\n4289\r\n4290\r\n4291\r\n4292\r\n4293\r\n4294\r\n4295\r\n4296\r\n4297\r\n4298\r\n4299\r\n4300\r\n4301\r\n4302\r\n4303\r\n4304\r\n4305\r\n4306\r\n4307\r\n4308\r\n4309\r\n4310\r\n4311\r\n4312\r\n4313\r\n4314\r\n4315\r\n4316\r\n4317\r\n4318\r\n4319\r\n4320\r\n4321\r\n4322\r\n4323\r\n4324\r\n4325\r\n4326\r\n4327\r\n4328\r\n4329\r\n4330\r\n4331\r\n4332\r\n4333\r\n4334\r\n4335\r\n4336\r\n4337\r\n4338\r\n4339\r\n4340\r\n4341\r\n4342\r\n4343\r\n4344\r\n4345\r\n4346\r\n4347\r\n4348\r\n4349\r\n4350\r\n4351\r\n4352\r\n4353\r\n4354\r\n4355\r\n4356\r\n4357\r\n4358\r\n4359\r\n4360\r\n4361\r\n4362\r\n4363\r\n4364\r\n4365\r\n4366\r\n4367\r\n4368\r\n4369\r\n4370\r\n4371\r\n4372\r\n4373\r\n4374\r\n4375\r\n4376\r\n4377\r\n4378\r\n4379\r\n4380\r\n4381\r\n4382\r\n4383\r\n4384\r\n4385\r\n4386\r\n4387\r\n4388\r\n4389\r\n4390\r\n4391\r\n4392\r\n4393\r\n4394\r\n4395\r\n4396\r\n4397\r\n4398\r\n4399\r\n4400\r\n4401\r\n4402\r\n4403\r\n4404\r\n4405\r\n4406\r\n4407\r\n4408\r\n4409\r\n4410\r\n4411\r\n4412\r\n4413\r\n4414\r\n4415\r\n4416\r\n4417\r\n4418\r\n4419\r\n4420\r\n4421\r\n4422\r\n4423\r\n4424\r\n4425\r\n4426\r\n4427\r\n4428\r\n4429\r\n4430\r\n4431\r\n4432\r\n4433\r\n4434\r\n4435\r\n4436\r\n4437\r\n4438\r\n4439\r\n4440\r\n4441\r\n4442\r\n4443\r\n4444\r\n4445\r\n4446\r\n4447\r\n4448\r\n4449\r\n4450\r\n4451\r\n4452\r\n4453\r\n4454\r\n4455\r\n4456\r\n4457\r\n4458\r\n4459\r\n4460\r\n4461\r\n4462\r\n4463\r\n4464\r\n4465\r\n4466\r\n4467\r\n4468\r\n4469\r\n4470\r\n4471\r\n4472\r\n4473\r\n4474\r\n4475\r\n4476\r\n4477\r\n4478\r\n4479\r\n4480\r\n4481\r\n4482\r\n4483\r\n4484\r\n4485\r\n4486\r\n4487\r\n4488\r\n4489\r\n4490\r\n4491\r\n4492\r\n4493\r\n4494\r\n4495\r\n4496\r\n4497\r\n4498\r\n4499\r\n4500\r\n4501\r\n4502\r\n4503\r\n4504\r\n4505\r\n4506\r\n4507\r\n4508\r\n4509\r\n4510\r\n4511\r\n4512\r\n4513\r\n4514\r\n4515\r\n4516\r\n4517\r\n4518\r\n4519\r\n4520\r\n4521\r\n4522\r\n4523\r\n4524\r\n4525\r\n4526\r\n4527\r\n4528\r\n4529\r\n4530\r\n4531\r\n4532\r\n4533\r\n4534\r\n4535\r\n4536\r\n4537\r\n4538\r\n4539\r\n4540\r\n4541\r\n4542\r\n4543\r\n4544\r\n4545\r\n4546\r\n4547\r\n4548\r\n4549\r\n4550\r\n4551\r\n4552\r\n4553\r\n4554\r\n4555\r\n4556\r\n4557\r\n4558\r\n4559\r\n4560\r\n4561\r\n4562\r\n4563\r\n4564\r\n4565\r\n4566\r\n4567\r\n4568\r\n4569\r\n4570\r\n4571\r\n4572\r\n4573\r\n4574\r\n4575\r\n4576\r\n4577\r\n4578\r\n4579\r\n4580\r\n4581\r\n4582\r\n4583\r\n4584\r\n4585\r\n4586\r\n4587\r\n4588\r\n4589\r\n4590\r\n4591\r\n4592\r\n4593\r\n4594\r\n4595\r\n4596\r\n4597\r\n4598\r\n4599\r\n4600\r\n4601\r\n4602\r\n4603\r\n4604\r\n4605\r\n4606\r\n4607\r\n4608\r\n4609\r\n4610\r\n4611\r\n4612\r\n4613\r\n4614\r\n4615\r\n4616\r\n4617\r\n4618\r\n4619\r\n4620\r\n4621\r\n4622\r\n4623\r\n4624\r\n4625\r\n4626\r\n4627\r\n4628\r\n4629\r\n4630\r\n4631\r\n4632\r\n4633\r\n4634\r\n4635\r\n4636\r\n4637\r\n4638\r\n4639\r\n4640\r\n4641\r\n4642\r\n4643\r\n4644\r\n4645\r\n4646\r\n4647\r\n4648\r\n4649\r\n4650\r\n4651\r\n4652\r\n4653\r\n4654\r\n4655\r\n4656\r\n4657\r\n4658\r\n4659\r\n4660\r\n4661\r\n4662\r\n4663\r\n4664\r\n4665\r\n4666\r\n4667\r\n4668\r\n4669\r\n4670\r\n4671\r\n4672\r\n4673\r\n4674\r\n4675\r\n4676\r\n4677\r\n4678\r\n4679\r\n4680\r\n4681\r\n4682\r\n4683\r\n4684\r\n4685\r\n4686\r\n4687\r\n4688\r\n4689\r\n4690\r\n4691\r\n4692\r\n4693\r\n4694\r\n4695\r\n4696\r\n4697\r\n4698\r\n4699\r\n4700\r\n4701\r\n4702\r\n4703\r\n4704\r\n4705\r\n4706\r\n4707\r\n4708\r\n4709\r\n4710\r\n4711\r\n4712\r\n4713\r\n4714\r\n4715\r\n4716\r\n4717\r\n4718\r\n4719\r\n4720\r\n4721\r\n4722\r\n4723\r\n4724\r\n4725\r\n4726\r\n4727\r\n4728\r\n4729\r\n4730\r\n4731\r\n4732\r\n4733\r\n4734\r\n4735\r\n4736\r\n4737\r\n4738\r\n4739\r\n4740\r\n4741\r\n4742\r\n4743\r\n4744\r\n4745\r\n4746\r\n4747\r\n4748\r\n4749\r\n4750\r\n4751\r\n4752\r\n4753\r\n4754\r\n4755\r\n4756\r\n4757\r\n4758\r\n4759\r\n4760\r\n4761\r\n4762\r\n4763\r\n4764\r\n4765\r\n4766\r\n4767\r\n4768\r\n4769\r\n4770\r\n4771\r\n4772\r\n4773\r\n4774\r\n4775\r\n4776\r\n4777\r\n4778\r\n4779\r\n4780\r\n4781\r\n4782\r\n4783\r\n4784\r\n4785\r\n4786\r\n4787\r\n4788\r\n4789\r\n4790\r\n4791\r\n4792\r\n4793\r\n4794\r\n4795\r\n4796\r\n4797\r\n4798\r\n4799\r\n4800\r\n4801\r\n4802\r\n4803\r\n4804\r\n4805\r\n4806\r\n4807\r\n4808\r\n4809\r\n4810\r\n4811\r\n4812\r\n4813\r\n4814\r\n4815\r\n4816\r\n4817\r\n4818\r\n4819\r\n4820\r\n4821\r\n4822\r\n4823\r\n4824\r\n4825\r\n4826\r\n4827\r\n4828\r\n4829\r\n4830\r\n4831\r\n4832\r\n4833\r\n4834\r\n4835\r\n4836\r\n4837\r\n4838\r\n4839\r\n4840\r\n4841\r\n4842\r\n4843\r\n4844\r\n4845\r\n4846\r\n4847\r\n4848\r\n4849\r\n4850\r\n4851\r\n4852\r\n4853\r\n4854\r\n4855\r\n4856\r\n4857\r\n4858\r\n4859\r\n4860\r\n4861\r\n4862\r\n4863\r\n4864\r\n4865\r\n4866\r\n4867\r\n4868\r\n4869\r\n4870\r\n4871\r\n4872\r\n4873\r\n4874\r\n4875\r\n4876\r\n4877\r\n4878\r\n4879\r\n4880\r\n4881\r\n4882\r\n4883\r\n4884\r\n4885\r\n4886\r\n4887\r\n4888\r\n4889\r\n4890\r\n4891\r\n4892\r\n4893\r\n4894\r\n4895\r\n4896\r\n4897\r\n4898\r\n4899\r\n4900\r\n4901\r\n4902\r\n4903\r\n4904\r\n4905\r\n4906\r\n4907\r\n4908\r\n4909\r\n4910\r\n4911\r\n4912\r\n4913\r\n4914\r\n4915\r\n4916\r\n4917\r\n4918\r\n4919\r\n4920\r\n4921\r\n4922\r\n4923\r\n4924\r\n4925\r\n4926\r\n4927\r\n4928\r\n4929\r\n4930\r\n4931\r\n4932\r\n4933\r\n4934\r\n4935\r\n4936\r\n4937\r\n4938\r\n4939\r\n4940\r\n4941\r\n4942\r\n4943\r\n4944\r\n4945\r\n4946\r\n4947\r\n4948\r\n4949\r\n4950\r\n4951\r\n4952\r\n4953\r\n4954\r\n4955\r\n4956\r\n4957\r\n4958\r\n4959\r\n4960\r\n4961\r\n4962\r\n4963\r\n4964\r\n4965\r\n4966\r\n4967\r\n4968\r\n4969\r\n4970\r\n4971\r\n4972\r\n4973\r\n4974\r\n4975\r\n4976\r\n4977\r\n4978\r\n4979\r\n4980\r\n4981\r\n4982\r\n4983\r\n4984\r\n4985\r\n4986\r\n4987\r\n4988\r\n4989\r\n4990\r\n4991\r\n4992\r\n4993\r\n4994\r\n4995\r\n4996\r\n4997\r\n4998\r\n4999\r\n5000\r\n5001\r\n5002\r\n5003\r\n5004\r\n5005\r\n5006\r\n5007\r\n5008\r\n5009\r\n5010\r\n5011\r\n5012\r\n5013\r\n5014\r\n5015\r\n5016\r\n5017\r\n5018\r\n5019\r\n5020\r\n5021\r\n5022\r\n5023\r\n5024\r\n5025\r\n5026\r\n5027\r\n5028\r\n5029\r\n5030\r\n5031\r\n5032\r\n5033\r\n5034\r\n5035\r\n5036\r\n5037\r\n5038\r\n5039\r\n5040\r\n5041\r\n5042\r\n5043\r\n5044\r\n5045\r\n5046\r\n5047\r\n5048\r\n5049\r\n5050\r\n5051\r\n5052\r\n5053\r\n5054\r\n5055\r\n5056\r\n5057\r\n5058\r\n5059\r\n5060\r\n5061\r\n5062\r\n5063\r\n5064\r\n5065\r\n5066\r\n5067\r\n5068\r\n5069\r\n5070\r\n5071\r\n5072\r\n5073\r\n5074\r\n5075\r\n5076\r\n5077\r\n5078\r\n5079\r\n5080\r\n5081\r\n5082\r\n5083\r\n5084\r\n5085\r\n5086\r\n5087\r\n5088\r\n5089\r\n5090\r\n5091\r\n5092\r\n5093\r\n5094\r\n5095\r\n5096\r\n5097\r\n5098\r\n5099\r\n5100\r\n5101\r\n5102\r\n5103\r\n5104\r\n5105\r\n5106\r\n5107\r\n5108\r\n5109\r\n5110\r\n5111\r\n5112\r\n5113\r\n5114\r\n5115\r\n5116\r\n5117\r\n5118\r\n5119\r\n5120\r\n5121\r\n5122\r\n5123\r\n5124\r\n5125\r\n5126\r\n5127\r\n5128\r\n5129\r\n5130\r\n5131\r\n5132\r\n5133\r\n5134\r\n5135\r\n5136\r\n5137\r\n5138\r\n5139\r\n5140\r\n5141\r\n5142\r\n5143\r\n5144\r\n5145\r\n5146\r\n5147\r\n5148\r\n5149\r\n5150\r\n5151\r\n5152\r\n5153\r\n5154\r\n5155\r\n5156\r\n5157\r\n5158\r\n5159\r\n5160\r\n5161\r\n5162\r\n5163\r\n5164\r\n5165\r\n5166\r\n5167\r\n5168\r\n5169\r\n5170\r\n5171\r\n5172\r\n5173\r\n5174\r\n5175\r\n5176\r\n5177\r\n5178\r\n5179\r\n5180\r\n5181\r\n5182\r\n5183\r\n5184\r\n5185\r\n5186\r\n5187\r\n5188\r\n5189\r\n5190\r\n5191\r\n5192\r\n5193\r\n5194\r\n5195\r\n5196\r\n5197\r\n5198\r\n5199\r\n5200\r\n5201\r\n5202\r\n5203\r\n5204\r\n5205\r\n5206\r\n5207\r\n5208\r\n5209\r\n5210\r\n5211\r\n5212\r\n5213\r\n5214\r\n5215\r\n5216\r\n5217\r\n5218\r\n5219\r\n5220\r\n5221\r\n5222\r\n5223\r\n5224\r\n5225\r\n5226\r\n5227\r\n5228\r\n5229\r\n5230\r\n5231\r\n5232\r\n5233\r\n5234\r\n5235\r\n5236\r\n5237\r\n5238\r\n5239\r\n5240\r\n5241\r\n5242\r\n5243\r\n5244\r\n5245\r\n5246\r\n5247\r\n5248\r\n5249\r\n5250\r\n5251\r\n5252\r\n5253\r\n5254\r\n5255\r\n5256\r\n5257\r\n5258\r\n5259\r\n5260\r\n5261\r\n5262\r\n5263\r\n5264\r\n5265\r\n5266\r\n5267\r\n5268\r\n5269\r\n5270\r\n5271\r\n5272\r\n5273\r\n5274\r\n5275\r\n5276\r\n5277\r\n5278\r\n5279\r\n5280\r\n5281\r\n5282\r\n5283\r\n5284\r\n5285\r\n5286\r\n5287\r\n5288\r\n5289\r\n5290\r\n5291\r\n5292\r\n5293\r\n5294\r\n5295\r\n5296\r\n5297\r\n5298\r\n5299\r\n5300\r\n5301\r\n5302\r\n5303\r\n5304\r\n5305\r\n5306\r\n5307\r\n5308\r\n5309\r\n5310\r\n5311\r\n5312\r\n5313\r\n5314\r\n5315\r\n5316\r\n5317\r\n5318\r\n5319\r\n5320\r\n5321\r\n5322\r\n5323\r\n5324\r\n5325\r\n5326\r\n5327\r\n5328\r\n5329\r\n5330\r\n5331\r\n5332\r\n5333\r\n5334\r\n5335\r\n5336\r\n5337\r\n5338\r\n5339\r\n5340\r\n5341\r\n5342\r\n5343\r\n5344\r\n5345\r\n5346\r\n5347\r\n5348\r\n5349\r\n5350\r\n5351\r\n5352\r\n5353\r\n5354\r\n5355\r\n5356\r\n5357\r\n5358\r\n5359\r\n5360\r\n5361\r\n5362\r\n5363\r\n5364\r\n5365\r\n5366\r\n5367\r\n5368\r\n5369\r\n5370\r\n5371\r\n5372\r\n5373\r\n5374\r\n5375\r\n5376\r\n5377\r\n5378\r\n5379\r\n5380\r\n5381\r\n5382\r\n5383\r\n5384\r\n5385\r\n5386\r\n5387\r\n5388\r\n5389\r\n5390\r\n5391\r\n5392\r\n5393\r\n5394\r\n5395\r\n5396\r\n5397\r\n5398\r\n5399\r\n5400\r\n5401\r\n5402\r\n5403\r\n5404\r\n5405\r\n5406\r\n5407\r\n5408\r\n5409\r\n5410\r\n5411\r\n5412\r\n5413\r\n5414\r\n5415\r\n5416\r\n5417\r\n5418\r\n5419\r\n5420\r\n5421\r\n5422\r\n5423\r\n5424\r\n5425\r\n5426\r\n5427\r\n5428\r\n5429\r\n5430\r\n5431\r\n5432\r\n5433\r\n5434\r\n5435\r\n5436\r\n5437\r\n5438\r\n5439\r\n5440\r\n5441\r\n5442\r\n5443\r\n5444\r\n5445\r\n5446\r\n5447\r\n5448\r\n5449\r\n5450\r\n5451\r\n5452\r\n5453\r\n5454\r\n5455\r\n5456\r\n5457\r\n5458\r\n5459\r\n5460\r\n5461\r\n5462\r\n5463\r\n5464\r\n5465\r\n5466\r\n5467\r\n5468\r\n5469\r\n5470\r\n5471\r\n5472\r\n5473\r\n5474\r\n5475\r\n5476\r\n5477\r\n5478\r\n5479\r\n5480\r\n5481\r\n5482\r\n5483\r\n5484\r\n5485\r\n5486\r\n5487\r\n5488\r\n5489\r\n5490\r\n5491\r\n5492\r\n5493\r\n5494\r\n5495\r\n5496\r\n5497\r\n5498\r\n5499\r\n5500\r\n5501\r\n5502\r\n5503\r\n5504\r\n5505\r\n5506\r\n5507\r\n5508\r\n5509\r\n5510\r\n5511\r\n5512\r\n5513\r\n5514\r\n5515\r\n5516\r\n5517\r\n5518\r\n5519\r\n5520\r\n5521\r\n5522\r\n5523\r\n5524\r\n5525\r\n5526\r\n5527\r\n5528\r\n5529\r\n5530\r\n5531\r\n5532\r\n5533\r\n5534\r\n5535\r\n5536\r\n5537\r\n5538\r\n5539\r\n5540\r\n5541\r\n5542\r\n5543\r\n5544\r\n5545\r\n5546\r\n5547\r\n5548\r\n5549\r\n5550\r\n5551\r\n5552\r\n5553\r\n5554\r\n5555\r\n5556\r\n5557\r\n5558\r\n5559\r\n5560\r\n5561\r\n5562\r\n5563\r\n5564\r\n5565\r\n5566\r\n5567\r\n5568\r\n5569\r\n5570\r\n5571\r\n5572\r\n5573\r\n5574\r\n5575\r\n5576\r\n5577\r\n5578\r\n5579\r\n5580\r\n5581\r\n5582\r\n5583\r\n5584\r\n5585\r\n5586\r\n5587\r\n5588\r\n5589\r\n5590\r\n5591\r\n5592\r\n5593\r\n5594\r\n5595\r\n5596\r\n5597\r\n5598\r\n5599\r\n5600\r\n5601\r\n5602\r\n5603\r\n5604\r\n5605\r\n5606\r\n5607\r\n5608\r\n5609\r\n5610\r\n5611\r\n5612\r\n5613\r\n5614\r\n5615\r\n5616\r\n5617\r\n5618\r\n5619\r\n5620\r\n5621\r\n5622\r\n5623\r\n5624\r\n5625\r\n5626\r\n5627\r\n5628\r\n5629\r\n5630\r\n5631\r\n5632\r\n5633\r\n5634\r\n5635\r\n5636\r\n5637\r\n5638\r\n5639\r\n5640\r\n5641\r\n5642\r\n5643\r\n5644\r\n5645\r\n5646\r\n5647\r\n5648\r\n5649\r\n5650\r\n5651\r\n5652\r\n5653\r\n5654\r\n5655\r\n5656\r\n5657\r\n5658\r\n5659\r\n5660\r\n5661\r\n5662\r\n5663\r\n5664\r\n5665\r\n5666\r\n5667\r\n5668\r\n5669\r\n5670\r\n5671\r\n5672\r\n5673\r\n5674\r\n5675\r\n5676\r\n5677\r\n5678\r\n5679\r\n5680\r\n5681\r\n5682\r\n5683\r\n5684\r\n5685\r\n5686\r\n5687\r\n5688\r\n5689\r\n5690\r\n5691\r\n5692\r\n5693\r\n5694\r\n5695\r\n5696\r\n5697\r\n5698\r\n5699\r\n5700\r\n5701\r\n5702\r\n5703\r\n5704\r\n5705\r\n5706\r\n5707\r\n5708\r\n5709\r\n5710\r\n5711\r\n5712\r\n5713\r\n5714\r\n5715\r\n5716\r\n5717\r\n5718\r\n5719\r\n5720\r\n5721\r\n5722\r\n5723\r\n5724\r\n5725\r\n5726\r\n5727\r\n5728\r\n5729\r\n5730\r\n5731\r\n5732\r\n5733\r\n5734\r\n5735\r\n5736\r\n5737\r\n5738\r\n5739\r\n5740\r\n5741\r\n5742\r\n5743\r\n5744\r\n5745\r\n5746\r\n5747\r\n5748\r\n5749\r\n5750\r\n5751\r\n5752\r\n5753\r\n5754\r\n5755\r\n5756\r\n5757\r\n5758\r\n5759\r\n5760\r\n5761\r\n5762\r\n5763\r\n5764\r\n5765\r\n5766\r\n5767\r\n5768\r\n5769\r\n5770\r\n5771\r\n5772\r\n5773\r\n5774\r\n5775\r\n5776\r\n5777\r\n5778\r\n5779\r\n5780\r\n5781\r\n5782\r\n5783\r\n5784\r\n5785\r\n5786\r\n5787\r\n5788\r\n5789\r\n5790\r\n5791\r\n5792\r\n5793\r\n5794\r\n5795\r\n5796\r\n5797\r\n5798\r\n5799\r\n5800\r\n5801\r\n5802\r\n5803\r\n5804\r\n5805\r\n5806\r\n5807\r\n5808\r\n5809\r\n5810\r\n5811\r\n5812\r\n5813\r\n5814\r\n5815\r\n5816\r\n5817\r\n5818\r\n5819\r\n5820\r\n5821\r\n5822\r\n5823\r\n5824\r\n5825\r\n5826\r\n5827\r\n5828\r\n5829\r\n5830\r\n5831\r\n5832\r\n5833\r\n5834\r\n5835\r\n5836\r\n5837\r\n5838\r\n5839\r\n5840\r\n5841\r\n5842\r\n5843\r\n5844\r\n5845\r\n5846\r\n5847\r\n5848\r\n5849\r\n5850\r\n5851\r\n5852\r\n5853\r\n5854\r\n5855\r\n5856\r\n5857\r\n5858\r\n5859\r\n5860\r\n5861\r\n5862\r\n5863\r\n5864\r\n5865\r\n5866\r\n5867\r\n5868\r\n5869\r\n5870\r\n5871\r\n5872\r\n5873\r\n5874\r\n5875\r\n5876\r\n5877\r\n5878\r\n5879\r\n5880\r\n5881\r\n5882\r\n5883\r\n5884\r\n5885\r\n5886\r\n5887\r\n5888\r\n5889\r\n5890\r\n5891\r\n5892\r\n5893\r\n5894\r\n5895\r\n5896\r\n5897\r\n5898\r\n5899\r\n5900\r\n5901\r\n5902\r\n5903\r\n5904\r\n5905\r\n5906\r\n5907\r\n5908\r\n5909\r\n5910\r\n5911\r\n5912\r\n5913\r\n5914\r\n5915\r\n5916\r\n5917\r\n5918\r\n5919\r\n5920\r\n5921\r\n5922\r\n5923\r\n5924\r\n5925\r\n5926\r\n5927\r\n5928\r\n5929\r\n5930\r\n5931\r\n5932\r\n5933\r\n5934\r\n5935\r\n5936\r\n5937\r\n5938\r\n5939\r\n5940\r\n5941\r\n5942\r\n5943\r\n5944\r\n5945\r\n5946\r\n5947\r\n5948\r\n5949\r\n5950\r\n5951\r\n5952\r\n5953\r\n5954\r\n5955\r\n5956\r\n5957\r\n5958\r\n5959\r\n5960\r\n5961\r\n5962\r\n5963\r\n5964\r\n5965\r\n5966\r\n5967\r\n5968\r\n5969\r\n5970\r\n5971\r\n5972\r\n5973\r\n5974\r\n5975\r\n5976\r\n5977\r\n5978\r\n5979\r\n5980\r\n5981\r\n5982\r\n5983\r\n5984\r\n5985\r\n5986\r\n5987\r\n5988\r\n5989\r\n5990\r\n5991\r\n5992\r\n5993\r\n5994\r\n5995\r\n5996\r\n5997\r\n5998\r\n5999\r\n6000\r\n6001\r\n6002\r\n6003\r\n6004\r\n6005\r\n6006\r\n6007\r\n6008\r\n6009\r\n6010\r\n6011\r\n6012\r\n6013\r\n6014\r\n6015\r\n6016\r\n6017\r\n6018\r\n6019\r\n6020\r\n6021\r\n6022\r\n6023\r\n6024\r\n6025\r\n6026\r\n6027\r\n6028\r\n6029\r\n6030\r\n6031\r\n6032\r\n6033\r\n6034\r\n6035\r\n6036\r\n6037\r\n6038\r\n6039\r\n6040\r\n6041\r\n6042\r\n6043\r\n6044\r\n6045\r\n6046\r\n6047\r\n6048\r\n6049\r\n6050\r\n6051\r\n6052\r\n6053\r\n6054\r\n6055\r\n6056\r\n6057\r\n6058\r\n6059\r\n6060\r\n6061\r\n6062\r\n6063\r\n6064\r\n6065\r\n6066\r\n6067\r\n6068\r\n6069\r\n6070\r\n6071\r\n6072\r\n6073\r\n6074\r\n6075\r\n6076\r\n6077\r\n6078\r\n6079\r\n6080\r\n6081\r\n6082\r\n6083\r\n6084\r\n6085\r\n6086\r\n6087\r\n6088\r\n6089\r\n6090\r\n6091\r\n6092\r\n6093\r\n6094\r\n6095\r\n6096\r\n6097\r\n6098\r\n6099\r\n6100\r\n6101\r\n6102\r\n6103\r\n6104\r\n6105\r\n6106\r\n6107\r\n6108\r\n6109\r\n6110\r\n6111\r\n6112\r\n6113\r\n6114\r\n6115\r\n6116\r\n6117\r\n6118\r\n6119\r\n6120\r\n6121\r\n6122\r\n6123\r\n6124\r\n6125\r\n6126\r\n6127\r\n6128\r\n6129\r\n6130\r\n6131\r\n6132\r\n6133\r\n6134\r\n6135\r\n6136\r\n6137\r\n6138\r\n6139\r\n6140\r\n6141\r\n6142\r\n6143\r\n6144\r\n6145\r\n6146\r\n6147\r\n6148\r\n6149\r\n6150\r\n6151\r\n6152\r\n6153\r\n6154\r\n6155\r\n6156\r\n6157\r\n6158\r\n6159\r\n6160\r\n6161\r\n6162\r\n6163\r\n6164\r\n6165\r\n6166\r\n6167\r\n6168\r\n6169\r\n6170\r\n6171\r\n6172\r\n6173\r\n6174\r\n6175\r\n6176\r\n6177\r\n6178\r\n6179\r\n6180\r\n6181\r\n6182\r\n6183\r\n6184\r\n6185\r\n6186\r\n6187\r\n6188\r\n6189\r\n6190\r\n6191\r\n6192\r\n6193\r\n6194\r\n6195\r\n6196\r\n6197\r\n6198\r\n6199\r\n6200\r\n6201\r\n6202\r\n6203\r\n6204\r\n6205\r\n6206\r\n6207\r\n6208\r\n6209\r\n6210\r\n6211\r\n6212\r\n6213\r\n6214\r\n6215\r\n6216\r\n6217\r\n6218\r\n6219\r\n6220\r\n6221\r\n6222\r\n6223\r\n6224\r\n6225\r\n6226\r\n6227\r\n6228\r\n6229\r\n6230\r\n6231\r\n6232\r\n6233\r\n6234\r\n6235\r\n6236\r\n6237\r\n6238\r\n6239\r\n6240\r\n6241\r\n6242\r\n6243\r\n6244\r\n6245\r\n6246\r\n6247\r\n6248\r\n6249\r\n6250\r\n6251\r\n6252\r\n6253\r\n6254\r\n6255\r\n6256\r\n6257\r\n6258\r\n6259\r\n6260\r\n6261\r\n6262\r\n6263\r\n6264\r\n6265\r\n6266\r\n6267\r\n6268\r\n6269\r\n6270\r\n6271\r\n6272\r\n6273\r\n6274\r\n6275\r\n6276\r\n6277\r\n6278\r\n6279\r\n6280\r\n6281\r\n6282\r\n6283\r\n6284\r\n6285\r\n6286\r\n6287\r\n6288\r\n6289\r\n6290\r\n6291\r\n6292\r\n6293\r\n6294\r\n6295\r\n6296\r\n6297\r\n6298\r\n6299\r\n6300\r\n6301\r\n6302\r\n6303\r\n6304\r\n6305\r\n6306\r\n6307\r\n6308\r\n6309\r\n6310\r\n6311\r\n6312\r\n6313\r\n6314\r\n6315\r\n6316\r\n6317\r\n6318\r\n6319\r\n6320\r\n6321\r\n6322\r\n6323\r\n6324\r\n6325\r\n6326\r\n6327\r\n6328\r\n6329\r\n6330\r\n6331\r\n6332\r\n6333\r\n6334\r\n6335\r\n6336\r\n6337\r\n6338\r\n6339\r\n6340\r\n6341\r\n6342\r\n6343\r\n6344\r\n6345\r\n6346\r\n6347\r\n6348\r\n6349\r\n6350\r\n6351\r\n6352\r\n6353\r\n6354\r\n6355\r\n6356\r\n6357\r\n6358\r\n6359\r\n6360\r\n6361\r\n6362\r\n6363\r\n6364\r\n6365\r\n6366\r\n6367\r\n6368\r\n6369\r\n6370\r\n6371\r\n6372\r\n6373\r\n6374\r\n6375\r\n6376\r\n6377\r\n6378\r\n6379\r\n6380\r\n6381\r\n6382\r\n6383\r\n6384\r\n6385\r\n6386\r\n6387\r\n6388\r\n6389\r\n6390\r\n6391\r\n6392\r\n6393\r\n6394\r\n6395\r\n6396\r\n6397\r\n6398\r\n6399\r\n6400\r\n6401\r\n6402\r\n6403\r\n6404\r\n6405\r\n6406\r\n6407\r\n6408\r\n6409\r\n6410\r\n6411\r\n6412\r\n6413\r\n6414\r\n6415\r\n6416\r\n6417\r\n6418\r\n6419\r\n6420\r\n6421\r\n6422\r\n6423\r\n6424\r\n6425\r\n6426\r\n6427\r\n6428\r\n6429\r\n6430\r\n6431\r\n6432\r\n6433\r\n6434\r\n6435\r\n6436\r\n6437\r\n6438\r\n6439\r\n6440\r\n6441\r\n6442\r\n6443\r\n6444\r\n6445\r\n6446\r\n6447\r\n6448\r\n6449\r\n6450\r\n6451\r\n6452\r\n6453\r\n6454\r\n6455\r\n6456\r\n6457\r\n6458\r\n6459\r\n6460\r\n6461\r\n6462\r\n6463\r\n6464\r\n6465\r\n6466\r\n6467\r\n6468\r\n6469\r\n6470\r\n6471\r\n6472\r\n6473\r\n6474\r\n6475\r\n6476\r\n6477\r\n6478\r\n6479\r\n6480\r\n6481\r\n6482\r\n6483\r\n6484\r\n6485\r\n6486\r\n6487\r\n6488\r\n6489\r\n6490\r\n6491\r\n6492\r\n6493\r\n6494\r\n6495\r\n6496\r\n6497\r\n6498\r\n6499\r\n6500\r\n6501\r\n6502\r\n6503\r\n6504\r\n6505\r\n6506\r\n6507\r\n6508\r\n6509\r\n6510\r\n6511\r\n6512\r\n6513\r\n6514\r\n6515\r\n6516\r\n6517\r\n6518\r\n6519\r\n6520\r\n6521\r\n6522\r\n6523\r\n6524\r\n6525\r\n6526\r\n6527\r\n6528\r\n6529\r\n6530\r\n6531\r\n6532\r\n6533\r\n6534\r\n6535\r\n6536\r\n6537\r\n6538\r\n6539\r\n6540\r\n6541\r\n6542\r\n6543\r\n6544\r\n6545\r\n6546\r\n6547\r\n6548\r\n6549\r\n6550\r\n6551\r\n6552\r\n6553\r\n6554\r\n6555\r\n6556\r\n6557\r\n6558\r\n6559\r\n6560\r\n6561\r\n6562\r\n6563\r\n6564\r\n6565\r\n6566\r\n6567\r\n6568\r\n6569\r\n6570\r\n6571\r\n6572\r\n6573\r\n6574\r\n6575\r\n6576\r\n6577\r\n6578\r\n6579\r\n6580\r\n6581\r\n6582\r\n6583\r\n6584\r\n6585\r\n6586\r\n6587\r\n6588\r\n6589\r\n6590\r\n6591\r\n6592\r\n6593\r\n6594\r\n6595\r\n6596\r\n6597\r\n6598\r\n6599\r\n6600\r\n6601\r\n6602\r\n6603\r\n6604\r\n6605\r\n6606\r\n6607\r\n6608\r\n6609\r\n6610\r\n6611\r\n6612\r\n6613\r\n6614\r\n6615\r\n6616\r\n6617\r\n6618\r\n6619\r\n6620\r\n6621\r\n6622\r\n6623\r\n6624\r\n6625\r\n6626\r\n6627\r\n6628\r\n6629\r\n6630\r\n6631\r\n6632\r\n6633\r\n6634\r\n6635\r\n6636\r\n6637\r\n6638\r\n6639\r\n6640\r\n6641\r\n6642\r\n6643\r\n6644\r\n6645\r\n6646\r\n6647\r\n6648\r\n6649\r\n6650\r\n6651\r\n6652\r\n6653\r\n6654\r\n6655\r\n6656\r\n6657\r\n6658\r\n6659\r\n6660\r\n6661\r\n6662\r\n6663\r\n6664\r\n6665\r\n6666\r\n6667\r\n6668\r\n6669\r\n6670\r\n6671\r\n6672\r\n6673\r\n6674\r\n6675\r\n6676\r\n6677\r\n6678\r\n6679\r\n6680\r\n6681\r\n6682\r\n6683\r\n6684\r\n6685\r\n6686\r\n6687\r\n6688\r\n6689\r\n6690\r\n6691\r\n6692\r\n6693\r\n6694\r\n6695\r\n6696\r\n6697\r\n6698\r\n6699\r\n6700\r\n6701\r\n6702\r\n6703\r\n6704\r\n6705\r\n6706\r\n6707\r\n6708\r\n6709\r\n6710\r\n6711\r\n6712\r\n6713\r\n6714\r\n6715\r\n6716\r\n6717\r\n6718\r\n6719\r\n6720\r\n6721\r\n6722\r\n6723\r\n6724\r\n6725\r\n6726\r\n6727\r\n6728\r\n6729\r\n6730\r\n6731\r\n6732\r\n6733\r\n6734\r\n6735\r\n6736\r\n6737\r\n6738\r\n6739\r\n6740\r\n6741\r\n6742\r\n6743\r\n6744\r\n6745\r\n6746\r\n6747\r\n6748\r\n6749\r\n6750\r\n6751\r\n6752\r\n6753\r\n6754\r\n6755\r\n6756\r\n6757\r\n6758\r\n6759\r\n6760\r\n6761\r\n6762\r\n6763\r\n6764\r\n6765\r\n6766\r\n6767\r\n6768\r\n6769\r\n6770\r\n6771\r\n6772\r\n6773\r\n6774\r\n6775\r\n6776\r\n6777\r\n6778\r\n6779\r\n6780\r\n6781\r\n6782\r\n6783\r\n6784\r\n6785\r\n6786\r\n6787\r\n6788\r\n6789\r\n6790\r\n6791\r\n6792\r\n6793\r\n6794\r\n6795\r\n6796\r\n6797\r\n6798\r\n6799\r\n6800\r\n6801\r\n6802\r\n6803\r\n6804\r\n6805\r\n6806\r\n6807\r\n6808\r\n6809\r\n6810\r\n6811\r\n6812\r\n6813\r\n6814\r\n6815\r\n6816\r\n6817\r\n6818\r\n6819\r\n6820\r\n6821\r\n6822\r\n6823\r\n6824\r\n6825\r\n6826\r\n6827\r\n6828\r\n6829\r\n6830\r\n6831\r\n6832\r\n6833\r\n6834\r\n6835\r\n6836\r\n6837\r\n6838\r\n6839\r\n6840\r\n6841\r\n6842\r\n6843\r\n6844\r\n6845\r\n6846\r\n6847\r\n6848\r\n6849\r\n6850\r\n6851\r\n6852\r\n6853\r\n6854\r\n6855\r\n6856\r\n6857\r\n6858\r\n6859\r\n6860\r\n6861\r\n6862\r\n6863\r\n6864\r\n6865\r\n6866\r\n6867\r\n6868\r\n6869\r\n6870\r\n6871\r\n6872\r\n6873\r\n6874\r\n6875\r\n6876\r\n6877\r\n6878\r\n6879\r\n6880\r\n6881\r\n6882\r\n6883\r\n6884\r\n6885\r\n6886\r\n6887\r\n6888\r\n6889\r\n6890\r\n6891\r\n6892\r\n6893\r\n6894\r\n6895\r\n6896\r\n6897\r\n6898\r\n6899\r\n6900\r\n6901\r\n6902\r\n6903\r\n6904\r\n6905\r\n6906\r\n6907\r\n6908\r\n6909\r\n6910\r\n6911\r\n6912\r\n6913\r\n6914\r\n6915\r\n6916\r\n6917\r\n6918\r\n6919\r\n6920\r\n6921\r\n6922\r\n6923\r\n6924\r\n6925\r\n6926\r\n6927\r\n6928\r\n6929\r\n6930\r\n6931\r\n6932\r\n6933\r\n6934\r\n6935\r\n6936\r\n6937\r\n6938\r\n6939\r\n6940\r\n6941\r\n6942\r\n6943\r\n6944\r\n6945\r\n6946\r\n6947\r\n6948\r\n6949\r\n6950\r\n6951\r\n6952\r\n6953\r\n6954\r\n6955\r\n6956\r\n6957\r\n6958\r\n6959\r\n6960\r\n6961\r\n6962\r\n6963\r\n6964\r\n6965\r\n6966\r\n6967\r\n6968\r\n6969\r\n6970\r\n6971\r\n6972\r\n6973\r\n6974\r\n6975\r\n6976\r\n6977\r\n6978\r\n6979\r\n6980\r\n6981\r\n6982\r\n6983\r\n6984\r\n6985\r\n6986\r\n6987\r\n6988\r\n6989\r\n6990\r\n6991\r\n6992\r\n6993\r\n6994\r\n6995\r\n6996\r\n6997\r\n6998\r\n6999\r\n7000\r\n7001\r\n7002\r\n7003\r\n7004\r\n7005\r\n7006\r\n7007\r\n7008\r\n7009\r\n7010\r\n7011\r\n7012\r\n7013\r\n7014\r\n7015\r\n7016\r\n7017\r\n7018\r\n7019\r\n7020\r\n7021\r\n7022\r\n7023\r\n7024\r\n7025\r\n7026\r\n7027\r\n7028\r\n7029\r\n7030\r\n7031\r\n7032\r\n7033\r\n7034\r\n7035\r\n7036\r\n7037\r\n7038\r\n7039\r\n7040\r\n7041\r\n7042\r\n7043\r\n7044\r\n7045\r\n7046\r\n7047\r\n7048\r\n7049\r\n7050\r\n7051\r\n7052\r\n7053\r\n7054\r\n7055\r\n7056\r\n7057\r\n7058\r\n7059\r\n7060\r\n7061\r\n7062\r\n7063\r\n7064\r\n7065\r\n7066\r\n7067\r\n7068\r\n7069\r\n7070\r\n7071\r\n7072\r\n7073\r\n7074\r\n7075\r\n7076\r\n7077\r\n7078\r\n7079\r\n7080\r\n7081\r\n7082\r\n7083\r\n7084\r\n7085\r\n7086\r\n7087\r\n7088\r\n7089\r\n7090\r\n7091\r\n7092\r\n7093\r\n7094\r\n7095\r\n7096\r\n7097\r\n7098\r\n7099\r\n7100\r\n7101\r\n7102\r\n7103\r\n7104\r\n7105\r\n7106\r\n7107\r\n7108\r\n7109\r\n7110\r\n7111\r\n7112\r\n7113\r\n7114\r\n7115\r\n7116\r\n7117\r\n7118\r\n7119\r\n7120\r\n7121\r\n7122\r\n7123\r\n7124\r\n7125\r\n7126\r\n7127\r\n7128\r\n7129\r\n7130\r\n7131\r\n7132\r\n7133\r\n7134\r\n7135\r\n7136\r\n7137\r\n7138\r\n7139\r\n7140\r\n7141\r\n7142\r\n7143\r\n7144\r\n7145\r\n7146\r\n7147\r\n7148\r\n7149\r\n7150\r\n7151\r\n7152\r\n7153\r\n7154\r\n7155\r\n7156\r\n7157\r\n7158\r\n7159\r\n7160\r\n7161\r\n7162\r\n7163\r\n7164\r\n7165\r\n7166\r\n7167\r\n7168\r\n7169\r\n7170\r\n7171\r\n7172\r\n7173\r\n7174\r\n7175\r\n7176\r\n7177\r\n7178\r\n7179\r\n7180\r\n7181\r\n7182\r\n7183\r\n7184\r\n7185\r\n7186\r\n7187\r\n7188\r\n7189\r\n7190\r\n7191\r\n7192\r\n7193\r\n7194\r\n7195\r\n7196\r\n7197\r\n7198\r\n7199\r\n7200\r\n7201\r\n7202\r\n7203\r\n7204\r\n7205\r\n7206\r\n7207\r\n7208\r\n7209\r\n7210\r\n7211\r\n7212\r\n7213\r\n7214\r\n7215\r\n7216\r\n7217\r\n7218\r\n7219\r\n7220\r\n7221\r\n7222\r\n7223\r\n7224\r\n7225\r\n7226\r\n7227\r\n7228\r\n7229\r\n7230\r\n7231\r\n7232\r\n7233\r\n7234\r\n7235\r\n7236\r\n7237\r\n7238\r\n7239\r\n7240\r\n7241\r\n7242\r\n7243\r\n7244\r\n7245\r\n7246\r\n7247\r\n7248\r\n7249\r\n7250\r\n7251\r\n7252\r\n7253\r\n7254\r\n7255\r\n7256\r\n7257\r\n7258\r\n7259\r\n7260\r\n7261\r\n7262\r\n7263\r\n7264\r\n7265\r\n7266\r\n7267\r\n7268\r\n7269\r\n7270\r\n7271\r\n7272\r\n7273\r\n7274\r\n7275\r\n7276\r\n7277\r\n7278\r\n7279\r\n7280\r\n7281\r\n7282\r\n7283\r\n7284\r\n7285\r\n7286\r\n7287\r\n7288\r\n7289\r\n7290\r\n7291\r\n7292\r\n7293\r\n7294\r\n7295\r\n7296\r\n7297\r\n7298\r\n7299\r\n7300\r\n7301\r\n7302\r\n7303\r\n7304\r\n7305\r\n7306\r\n7307\r\n7308\r\n7309\r\n7310\r\n7311\r\n7312\r\n7313\r\n7314\r\n7315\r\n7316\r\n7317\r\n7318\r\n7319\r\n7320\r\n7321\r\n7322\r\n7323\r\n7324\r\n7325\r\n7326\r\n7327\r\n7328\r\n7329\r\n7330\r\n7331\r\n7332\r\n7333\r\n7334\r\n7335\r\n7336\r\n7337\r\n7338\r\n7339\r\n7340\r\n7341\r\n7342\r\n7343\r\n7344\r\n7345\r\n7346\r\n7347\r\n7348\r\n7349\r\n7350\r\n7351\r\n7352\r\n7353\r\n7354\r\n7355\r\n7356\r\n7357\r\n7358\r\n7359\r\n7360\r\n7361\r\n7362\r\n7363\r\n7364\r\n7365\r\n7366\r\n7367\r\n7368\r\n7369\r\n7370\r\n7371\r\n7372\r\n7373\r\n7374\r\n7375\r\n7376\r\n7377\r\n7378\r\n7379\r\n7380\r\n7381\r\n7382\r\n7383\r\n7384\r\n7385\r\n7386\r\n7387\r\n7388\r\n7389\r\n7390\r\n7391\r\n7392\r\n7393\r\n7394\r\n7395\r\n7396\r\n7397\r\n7398\r\n7399\r\n7400\r\n7401\r\n7402\r\n7403\r\n7404\r\n7405\r\n7406\r\n7407\r\n7408\r\n7409\r\n7410\r\n7411\r\n7412\r\n7413\r\n7414\r\n7415\r\n7416\r\n7417\r\n7418\r\n7419\r\n7420\r\n7421\r\n7422\r\n7423\r\n7424\r\n7425\r\n7426\r\n7427\r\n7428\r\n7429\r\n7430\r\n7431\r\n7432\r\n7433\r\n7434\r\n7435\r\n7436\r\n7437\r\n7438\r\n7439\r\n7440\r\n7441\r\n7442\r\n7443\r\n7444\r\n7445\r\n7446\r\n7447\r\n7448\r\n7449\r\n7450\r\n7451\r\n7452\r\n7453\r\n7454\r\n7455\r\n7456\r\n7457\r\n7458\r\n7459\r\n7460\r\n7461\r\n7462\r\n7463\r\n7464\r\n7465\r\n7466\r\n7467\r\n7468\r\n7469\r\n7470\r\n7471\r\n7472\r\n7473\r\n7474\r\n7475\r\n7476\r\n7477\r\n7478\r\n7479\r\n7480\r\n7481\r\n7482\r\n7483\r\n7484\r\n7485\r\n7486\r\n7487\r\n7488\r\n7489\r\n7490\r\n7491\r\n7492\r\n7493\r\n7494\r\n7495\r\n7496\r\n7497\r\n7498\r\n7499\r\n7500\r\n7501\r\n7502\r\n7503\r\n7504\r\n7505\r\n7506\r\n7507\r\n7508\r\n7509\r\n7510\r\n7511\r\n7512\r\n7513\r\n7514\r\n7515\r\n7516\r\n7517\r\n7518\r\n7519\r\n7520\r\n7521\r\n7522\r\n7523\r\n7524\r\n7525\r\n7526\r\n7527\r\n7528\r\n7529\r\n7530\r\n7531\r\n7532\r\n7533\r\n7534\r\n7535\r\n7536\r\n7537\r\n7538\r\n7539\r\n7540\r\n7541\r\n7542\r\n7543\r\n7544\r\n7545\r\n7546\r\n7547\r\n7548\r\n7549\r\n7550\r\n7551\r\n7552\r\n7553\r\n7554\r\n7555\r\n7556\r\n7557\r\n7558\r\n7559\r\n7560\r\n7561\r\n7562\r\n7563\r\n7564\r\n7565\r\n7566\r\n7567\r\n7568\r\n7569\r\n7570\r\n7571\r\n7572\r\n7573\r\n7574\r\n7575\r\n7576\r\n7577\r\n7578\r\n7579\r\n7580\r\n7581\r\n7582\r\n7583\r\n7584\r\n7585\r\n7586\r\n7587\r\n7588\r\n7589\r\n7590\r\n7591\r\n7592\r\n7593\r\n7594\r\n7595\r\n7596\r\n7597\r\n7598\r\n7599\r\n7600\r\n7601\r\n7602\r\n7603\r\n7604\r\n7605\r\n7606\r\n7607\r\n7608\r\n7609\r\n7610\r\n7611\r\n7612\r\n7613\r\n7614\r\n7615\r\n7616\r\n7617\r\n7618\r\n7619\r\n7620\r\n7621\r\n7622\r\n7623\r\n7624\r\n7625\r\n7626\r\n7627\r\n7628\r\n7629\r\n7630\r\n7631\r\n7632\r\n7633\r\n7634\r\n7635\r\n7636\r\n7637\r\n7638\r\n7639\r\n7640\r\n7641\r\n7642\r\n7643\r\n7644\r\n7645\r\n7646\r\n7647\r\n7648\r\n7649\r\n7650\r\n7651\r\n7652\r\n7653\r\n7654\r\n7655\r\n7656\r\n7657\r\n7658\r\n7659\r\n7660\r\n7661\r\n7662\r\n7663\r\n7664\r\n7665\r\n7666\r\n7667\r\n7668\r\n7669\r\n7670\r\n7671\r\n7672\r\n7673\r\n7674\r\n7675\r\n7676\r\n7677\r\n7678\r\n7679\r\n7680\r\n7681\r\n7682\r\n7683\r\n7684\r\n7685\r\n7686\r\n7687\r\n7688\r\n7689\r\n7690\r\n7691\r\n7692\r\n7693\r\n7694\r\n7695\r\n7696\r\n7697\r\n7698\r\n7699\r\n7700\r\n7701\r\n7702\r\n7703\r\n7704\r\n7705\r\n7706\r\n7707\r\n7708\r\n7709\r\n7710\r\n7711\r\n7712\r\n7713\r\n7714\r\n7715\r\n7716\r\n7717\r\n7718\r\n7719\r\n7720\r\n7721\r\n7722\r\n7723\r\n7724\r\n7725\r\n7726\r\n7727\r\n7728\r\n7729\r\n7730\r\n7731\r\n7732\r\n7733\r\n7734\r\n7735\r\n7736\r\n7737\r\n7738\r\n7739\r\n7740\r\n7741\r\n7742\r\n7743\r\n7744\r\n7745\r\n7746\r\n7747\r\n7748\r\n7749\r\n7750\r\n7751\r\n7752\r\n7753\r\n7754\r\n7755\r\n7756\r\n7757\r\n7758\r\n7759\r\n7760\r\n7761\r\n7762\r\n7763\r\n7764\r\n7765\r\n7766\r\n7767\r\n7768\r\n7769\r\n7770\r\n7771\r\n7772\r\n7773\r\n7774\r\n7775\r\n7776\r\n7777\r\n7778\r\n7779\r\n7780\r\n7781\r\n7782\r\n7783\r\n7784\r\n7785\r\n7786\r\n7787\r\n7788\r\n7789\r\n7790\r\n7791\r\n7792\r\n7793\r\n7794\r\n7795\r\n7796\r\n7797\r\n7798\r\n7799\r\n7800\r\n7801\r\n7802\r\n7803\r\n7804\r\n7805\r\n7806\r\n7807\r\n7808\r\n7809\r\n7810\r\n7811\r\n7812\r\n7813\r\n7814\r\n7815\r\n7816\r\n7817\r\n7818\r\n7819\r\n7820\r\n7821\r\n7822\r\n7823\r\n7824\r\n7825\r\n7826\r\n7827\r\n7828\r\n7829\r\n7830\r\n7831\r\n7832\r\n7833\r\n7834\r\n7835\r\n7836\r\n7837\r\n7838\r\n7839\r\n7840\r\n7841\r\n7842\r\n7843\r\n7844\r\n7845\r\n7846\r\n7847\r\n7848\r\n7849\r\n7850\r\n7851\r\n7852\r\n7853\r\n7854\r\n7855\r\n7856\r\n7857\r\n7858\r\n7859\r\n7860\r\n7861\r\n7862\r\n7863\r\n7864\r\n7865\r\n7866\r\n7867\r\n7868\r\n7869\r\n7870\r\n7871\r\n7872\r\n7873\r\n7874\r\n7875\r\n7876\r\n7877\r\n7878\r\n7879\r\n7880\r\n7881\r\n7882\r\n7883\r\n7884\r\n7885\r\n7886\r\n7887\r\n7888\r\n7889\r\n7890\r\n7891\r\n7892\r\n7893\r\n7894\r\n7895\r\n7896\r\n7897\r\n7898\r\n7899\r\n7900\r\n7901\r\n7902\r\n7903\r\n7904\r\n7905\r\n7906\r\n7907\r\n7908\r\n7909\r\n7910\r\n7911\r\n7912\r\n7913\r\n7914\r\n7915\r\n7916\r\n7917\r\n7918\r\n7919\r\n7920\r\n7921\r\n7922\r\n7923\r\n7924\r\n7925\r\n7926\r\n7927\r\n7928\r\n7929\r\n7930\r\n7931\r\n7932\r\n7933\r\n7934\r\n7935\r\n7936\r\n7937\r\n7938\r\n7939\r\n7940\r\n7941\r\n7942\r\n7943\r\n7944\r\n7945\r\n7946\r\n7947\r\n7948\r\n7949\r\n7950\r\n7951\r\n7952\r\n7953\r\n7954\r\n7955\r\n7956\r\n7957\r\n7958\r\n7959\r\n7960\r\n7961\r\n7962\r\n7963\r\n7964\r\n7965\r\n7966\r\n7967\r\n7968\r\n7969\r\n7970\r\n7971\r\n7972\r\n7973\r\n7974\r\n7975\r\n7976\r\n7977\r\n7978\r\n7979\r\n7980\r\n7981\r\n7982\r\n7983\r\n7984\r\n7985\r\n7986\r\n7987\r\n7988\r\n7989\r\n7990\r\n7991\r\n7992\r\n7993\r\n7994\r\n7995\r\n7996\r\n7997\r\n7998\r\n7999\r\n8000\r\n8001\r\n8002\r\n8003\r\n8004\r\n8005\r\n8006\r\n8007\r\n8008\r\n8009\r\n8010\r\n8011\r\n8012\r\n8013\r\n8014\r\n8015\r\n8016\r\n8017\r\n8018\r\n8019\r\n8020\r\n8021\r\n8022\r\n8023\r\n8024\r\n8025\r\n8026\r\n8027\r\n8028\r\n8029\r\n8030\r\n8031\r\n8032\r\n8033\r\n8034\r\n8035\r\n8036\r\n8037\r\n8038\r\n8039\r\n8040\r\n8041\r\n8042\r\n8043\r\n8044\r\n8045\r\n8046\r\n8047\r\n8048\r\n8049\r\n8050\r\n8051\r\n8052\r\n8053\r\n8054\r\n8055\r\n8056\r\n8057\r\n8058\r\n8059\r\n8060\r\n8061\r\n8062\r\n8063\r\n8064\r\n8065\r\n8066\r\n8067\r\n8068\r\n8069\r\n8070\r\n8071\r\n8072\r\n8073\r\n8074\r\n8075\r\n8076\r\n8077\r\n8078\r\n8079\r\n8080\r\n8081\r\n8082\r\n8083\r\n8084\r\n8085\r\n8086\r\n8087\r\n8088\r\n8089\r\n8090\r\n8091\r\n8092\r\n8093\r\n8094\r\n8095\r\n8096\r\n8097\r\n8098\r\n8099\r\n8100\r\n8101\r\n8102\r\n8103\r\n8104\r\n8105\r\n8106\r\n8107\r\n8108\r\n8109\r\n8110\r\n8111\r\n8112\r\n8113\r\n8114\r\n8115\r\n8116\r\n8117\r\n8118\r\n8119\r\n8120\r\n8121\r\n8122\r\n8123\r\n8124\r\n8125\r\n8126\r\n8127\r\n8128\r\n8129\r\n8130\r\n8131\r\n8132\r\n8133\r\n8134\r\n8135\r\n8136\r\n8137\r\n8138\r\n8139\r\n8140\r\n8141\r\n8142\r\n8143\r\n8144\r\n8145\r\n8146\r\n8147\r\n8148\r\n8149\r\n8150\r\n8151\r\n8152\r\n8153\r\n8154\r\n8155\r\n8156\r\n8157\r\n8158\r\n8159\r\n8160\r\n8161\r\n8162\r\n8163\r\n8164\r\n8165\r\n8166\r\n8167\r\n8168\r\n8169\r\n8170\r\n8171\r\n8172\r\n8173\r\n8174\r\n8175\r\n8176\r\n8177\r\n8178\r\n8179\r\n8180\r\n8181\r\n8182\r\n8183\r\n8184\r\n8185\r\n8186\r\n8187\r\n8188\r\n8189\r\n8190\r\n8191\r\n8192\r\n8193\r\n8194\r\n8195\r\n8196\r\n8197\r\n8198\r\n8199\r\n8200\r\n8201\r\n8202\r\n8203\r\n8204\r\n8205\r\n8206\r\n8207\r\n8208\r\n8209\r\n8210\r\n8211\r\n8212\r\n8213\r\n8214\r\n8215\r\n8216\r\n8217\r\n8218\r\n8219\r\n8220\r\n8221\r\n8222\r\n8223\r\n8224\r\n8225\r\n8226\r\n8227\r\n8228\r\n8229\r\n8230\r\n8231\r\n8232\r\n8233\r\n8234\r\n8235\r\n8236\r\n8237\r\n8238\r\n8239\r\n8240\r\n8241\r\n8242\r\n8243\r\n8244\r\n8245\r\n8246\r\n8247\r\n8248\r\n8249\r\n8250\r\n8251\r\n8252\r\n8253\r\n8254\r\n8255\r\n8256\r\n8257\r\n8258\r\n8259\r\n8260\r\n8261\r\n8262\r\n8263\r\n8264\r\n8265\r\n8266\r\n8267\r\n8268\r\n8269\r\n8270\r\n8271\r\n8272\r\n8273\r\n8274\r\n8275\r\n8276\r\n8277\r\n8278\r\n8279\r\n8280\r\n8281\r\n8282\r\n8283\r\n8284\r\n8285\r\n8286\r\n8287\r\n8288\r\n8289\r\n8290\r\n8291\r\n8292\r\n8293\r\n8294\r\n8295\r\n8296\r\n8297\r\n8298\r\n8299\r\n8300\r\n8301\r\n8302\r\n8303\r\n8304\r\n8305\r\n8306\r\n8307\r\n8308\r\n8309\r\n8310\r\n8311\r\n8312\r\n8313\r\n8314\r\n8315\r\n8316\r\n8317\r\n8318\r\n8319\r\n8320\r\n8321\r\n8322\r\n8323\r\n8324\r\n8325\r\n8326\r\n8327\r\n8328\r\n8329\r\n8330\r\n8331\r\n8332\r\n8333\r\n8334\r\n8335\r\n8336\r\n8337\r\n8338\r\n8339\r\n8340\r\n8341\r\n8342\r\n8343\r\n8344\r\n8345\r\n8346\r\n8347\r\n8348\r\n8349\r\n8350\r\n8351\r\n8352\r\n8353\r\n8354\r\n8355\r\n8356\r\n8357\r\n8358\r\n8359\r\n8360\r\n8361\r\n8362\r\n8363\r\n8364\r\n8365\r\n8366\r\n8367\r\n8368\r\n8369\r\n8370\r\n8371\r\n8372\r\n8373\r\n8374\r\n8375\r\n8376\r\n8377\r\n8378\r\n8379\r\n8380\r\n8381\r\n8382\r\n8383\r\n8384\r\n8385\r\n8386\r\n8387\r\n8388\r\n8389\r\n8390\r\n8391\r\n8392\r\n8393\r\n8394\r\n8395\r\n8396\r\n8397\r\n8398\r\n8399\r\n8400\r\n8401\r\n8402\r\n8403\r\n8404\r\n8405\r\n8406\r\n8407\r\n8408\r\n8409\r\n8410\r\n8411\r\n8412\r\n8413\r\n8414\r\n8415\r\n8416\r\n8417\r\n8418\r\n8419\r\n8420\r\n8421\r\n8422\r\n8423\r\n8424\r\n8425\r\n8426\r\n8427\r\n8428\r\n8429\r\n8430\r\n8431\r\n8432\r\n8433\r\n8434\r\n8435\r\n8436\r\n8437\r\n8438\r\n8439\r\n8440\r\n8441\r\n8442\r\n8443\r\n8444\r\n8445\r\n8446\r\n8447\r\n8448\r\n8449\r\n8450\r\n8451\r\n8452\r\n8453\r\n8454\r\n8455\r\n8456\r\n8457\r\n8458\r\n8459\r\n8460\r\n8461\r\n8462\r\n8463\r\n8464\r\n8465\r\n8466\r\n8467\r\n8468\r\n8469\r\n8470\r\n8471\r\n8472\r\n8473\r\n8474\r\n8475\r\n8476\r\n8477\r\n8478\r\n8479\r\n8480\r\n8481\r\n8482\r\n8483\r\n8484\r\n8485\r\n8486\r\n8487\r\n8488\r\n8489\r\n8490\r\n8491\r\n8492\r\n8493\r\n8494\r\n8495\r\n8496\r\n8497\r\n8498\r\n8499\r\n8500\r\n8501\r\n8502\r\n8503\r\n8504\r\n8505\r\n8506\r\n8507\r\n8508\r\n8509\r\n8510\r\n8511\r\n8512\r\n8513\r\n8514\r\n8515\r\n8516\r\n8517\r\n8518\r\n8519\r\n8520\r\n8521\r\n8522\r\n8523\r\n8524\r\n8525\r\n8526\r\n8527\r\n8528\r\n8529\r\n8530\r\n8531\r\n8532\r\n8533\r\n8534\r\n8535\r\n8536\r\n8537\r\n8538\r\n8539\r\n8540\r\n8541\r\n8542\r\n8543\r\n8544\r\n8545\r\n8546\r\n8547\r\n8548\r\n8549\r\n8550\r\n8551\r\n8552\r\n8553\r\n8554\r\n8555\r\n8556\r\n8557\r\n8558\r\n8559\r\n8560\r\n8561\r\n8562\r\n8563\r\n8564\r\n8565\r\n8566\r\n8567\r\n8568\r\n8569\r\n8570\r\n8571\r\n8572\r\n8573\r\n8574\r\n8575\r\n8576\r\n8577\r\n8578\r\n8579\r\n8580\r\n8581\r\n8582\r\n8583\r\n8584\r\n8585\r\n8586\r\n8587\r\n8588\r\n8589\r\n8590\r\n8591\r\n8592\r\n8593\r\n8594\r\n8595\r\n8596\r\n8597\r\n8598\r\n8599\r\n8600\r\n8601\r\n8602\r\n8603\r\n8604\r\n8605\r\n8606\r\n8607\r\n8608\r\n8609\r\n8610\r\n8611\r\n8612\r\n8613\r\n8614\r\n8615\r\n8616\r\n8617\r\n8618\r\n8619\r\n8620\r\n8621\r\n8622\r\n8623\r\n8624\r\n8625\r\n8626\r\n8627\r\n8628\r\n8629\r\n8630\r\n8631\r\n8632\r\n8633\r\n8634\r\n8635\r\n8636\r\n8637\r\n8638\r\n8639\r\n8640\r\n8641\r\n8642\r\n8643\r\n8644\r\n8645\r\n8646\r\n8647\r\n8648\r\n8649\r\n8650\r\n8651\r\n8652\r\n8653\r\n8654\r\n8655\r\n8656\r\n8657\r\n8658\r\n8659\r\n8660\r\n8661\r\n8662\r\n8663\r\n8664\r\n8665\r\n8666\r\n8667\r\n8668\r\n8669\r\n8670\r\n8671\r\n8672\r\n8673\r\n8674\r\n8675\r\n8676\r\n8677\r\n8678\r\n8679\r\n8680\r\n8681\r\n8682\r\n8683\r\n8684\r\n8685\r\n8686\r\n8687\r\n8688\r\n8689\r\n8690\r\n8691\r\n8692\r\n8693\r\n8694\r\n8695\r\n8696\r\n8697\r\n8698\r\n8699\r\n8700\r\n8701\r\n8702\r\n8703\r\n8704\r\n8705\r\n8706\r\n8707\r\n8708\r\n8709\r\n8710\r\n8711\r\n8712\r\n8713\r\n8714\r\n8715\r\n8716\r\n8717\r\n8718\r\n8719\r\n8720\r\n8721\r\n8722\r\n8723\r\n8724\r\n8725\r\n8726\r\n8727\r\n8728\r\n8729\r\n8730\r\n8731\r\n8732\r\n8733\r\n8734\r\n8735\r\n8736\r\n8737\r\n8738\r\n8739\r\n8740\r\n8741\r\n8742\r\n8743\r\n8744\r\n8745\r\n8746\r\n8747\r\n8748\r\n8749\r\n8750\r\n8751\r\n8752\r\n8753\r\n8754\r\n8755\r\n8756\r\n8757\r\n8758\r\n8759\r\n8760\r\n8761\r\n8762\r\n8763\r\n8764\r\n8765\r\n8766\r\n8767\r\n8768\r\n8769\r\n8770\r\n8771\r\n8772\r\n8773\r\n8774\r\n8775\r\n8776\r\n8777\r\n8778\r\n8779\r\n8780\r\n8781\r\n8782\r\n8783\r\n8784\r\n8785\r\n8786\r\n8787\r\n8788\r\n8789\r\n8790\r\n8791\r\n8792\r\n8793\r\n8794\r\n8795\r\n8796\r\n8797\r\n8798\r\n8799\r\n8800\r\n8801\r\n8802\r\n8803\r\n8804\r\n8805\r\n8806\r\n8807\r\n8808\r\n8809\r\n8810\r\n8811\r\n8812\r\n8813\r\n8814\r\n8815\r\n8816\r\n8817\r\n8818\r\n8819\r\n8820\r\n8821\r\n8822\r\n8823\r\n8824\r\n8825\r\n8826\r\n8827\r\n8828\r\n8829\r\n8830\r\n8831\r\n8832\r\n8833\r\n8834\r\n8835\r\n8836\r\n8837\r\n8838\r\n8839\r\n8840\r\n8841\r\n8842\r\n8843\r\n8844\r\n8845\r\n8846\r\n8847\r\n8848\r\n8849\r\n8850\r\n8851\r\n8852\r\n8853\r\n8854\r\n8855\r\n8856\r\n8857\r\n8858\r\n8859\r\n8860\r\n8861\r\n8862\r\n8863\r\n8864\r\n8865\r\n8866\r\n8867\r\n8868\r\n8869\r\n8870\r\n8871\r\n8872\r\n8873\r\n8874\r\n8875\r\n8876\r\n8877\r\n8878\r\n8879\r\n8880\r\n8881\r\n8882\r\n8883\r\n8884\r\n8885\r\n8886\r\n8887\r\n8888\r\n8889\r\n8890\r\n8891\r\n8892\r\n8893\r\n8894\r\n8895\r\n8896\r\n8897\r\n8898\r\n8899\r\n8900\r\n8901\r\n8902\r\n8903\r\n8904\r\n8905\r\n8906\r\n8907\r\n8908\r\n8909\r\n8910\r\n8911\r\n8912\r\n8913\r\n8914\r\n8915\r\n8916\r\n8917\r\n8918\r\n8919\r\n8920\r\n8921\r\n8922\r\n8923\r\n8924\r\n8925\r\n8926\r\n8927\r\n8928\r\n8929\r\n8930\r\n8931\r\n8932\r\n8933\r\n8934\r\n8935\r\n8936\r\n8937\r\n8938\r\n8939\r\n8940\r\n8941\r\n8942\r\n8943\r\n8944\r\n8945\r\n8946\r\n8947\r\n8948\r\n8949\r\n8950\r\n8951\r\n8952\r\n8953\r\n8954\r\n8955\r\n8956\r\n8957\r\n8958\r\n8959\r\n8960\r\n8961\r\n8962\r\n8963\r\n8964\r\n8965\r\n8966\r\n8967\r\n8968\r\n8969\r\n8970\r\n8971\r\n8972\r\n8973\r\n8974\r\n8975\r\n8976\r\n8977\r\n8978\r\n8979\r\n8980\r\n8981\r\n8982\r\n8983\r\n8984\r\n8985\r\n8986\r\n8987\r\n8988\r\n8989\r\n8990\r\n8991\r\n8992\r\n8993\r\n8994\r\n8995\r\n8996\r\n8997\r\n8998\r\n8999\r\n9000\r\n9001\r\n9002\r\n9003\r\n9004\r\n9005\r\n9006\r\n9007\r\n9008\r\n9009\r\n9010\r\n9011\r\n9012\r\n9013\r\n9014\r\n9015\r\n9016\r\n9017\r\n9018\r\n9019\r\n9020\r\n9021\r\n9022\r\n9023\r\n9024\r\n9025\r\n9026\r\n9027\r\n9028\r\n9029\r\n9030\r\n9031\r\n9032\r\n9033\r\n9034\r\n9035\r\n9036\r\n9037\r\n9038\r\n9039\r\n9040\r\n9041\r\n9042\r\n9043\r\n9044\r\n9045\r\n9046\r\n9047\r\n9048\r\n9049\r\n9050\r\n9051\r\n9052\r\n9053\r\n9054\r\n9055\r\n9056\r\n9057\r\n9058\r\n9059\r\n9060\r\n9061\r\n9062\r\n9063\r\n9064\r\n9065\r\n9066\r\n9067\r\n9068\r\n9069\r\n9070\r\n9071\r\n9072\r\n9073\r\n9074\r\n9075\r\n9076\r\n9077\r\n9078\r\n9079\r\n9080\r\n9081\r\n9082\r\n9083\r\n9084\r\n9085\r\n9086\r\n9087\r\n9088\r\n9089\r\n9090\r\n9091\r\n9092\r\n9093\r\n9094\r\n9095\r\n9096\r\n9097\r\n9098\r\n9099\r\n9100\r\n9101\r\n9102\r\n9103\r\n9104\r\n9105\r\n9106\r\n9107\r\n9108\r\n9109\r\n9110\r\n9111\r\n9112\r\n9113\r\n9114\r\n9115\r\n9116\r\n9117\r\n9118\r\n9119\r\n9120\r\n9121\r\n9122\r\n9123\r\n9124\r\n9125\r\n9126\r\n9127\r\n9128\r\n9129\r\n9130\r\n9131\r\n9132\r\n9133\r\n9134\r\n9135\r\n9136\r\n9137\r\n9138\r\n9139\r\n9140\r\n9141\r\n9142\r\n9143\r\n9144\r\n9145\r\n9146\r\n9147\r\n9148\r\n9149\r\n9150\r\n9151\r\n9152\r\n9153\r\n9154\r\n9155\r\n9156\r\n9157\r\n9158\r\n9159\r\n9160\r\n9161\r\n9162\r\n9163\r\n9164\r\n9165\r\n9166\r\n9167\r\n9168\r\n9169\r\n9170\r\n9171\r\n9172\r\n9173\r\n9174\r\n9175\r\n9176\r\n9177\r\n9178\r\n9179\r\n9180\r\n9181\r\n9182\r\n9183\r\n9184\r\n9185\r\n9186\r\n9187\r\n9188\r\n9189\r\n9190\r\n9191\r\n9192\r\n9193\r\n9194\r\n9195\r\n9196\r\n9197\r\n9198\r\n9199\r\n9200\r\n9201\r\n9202\r\n9203\r\n9204\r\n9205\r\n9206\r\n9207\r\n9208\r\n9209\r\n9210\r\n9211\r\n9212\r\n9213\r\n9214\r\n9215\r\n9216\r\n9217\r\n9218\r\n9219\r\n9220\r\n9221\r\n9222\r\n9223\r\n9224\r\n9225\r\n9226\r\n9227\r\n9228\r\n9229\r\n9230\r\n9231\r\n9232\r\n9233\r\n9234\r\n9235\r\n9236\r\n9237\r\n9238\r\n9239\r\n9240\r\n9241\r\n9242\r\n9243\r\n9244\r\n9245\r\n9246\r\n9247\r\n9248\r\n9249\r\n9250\r\n9251\r\n9252\r\n9253\r\n9254\r\n9255\r\n9256\r\n9257\r\n9258\r\n9259\r\n9260\r\n9261\r\n9262\r\n9263\r\n9264\r\n9265\r\n9266\r\n9267\r\n9268\r\n9269\r\n9270\r\n9271\r\n9272\r\n9273\r\n9274\r\n9275\r\n9276\r\n9277\r\n9278\r\n9279\r\n9280\r\n9281\r\n9282\r\n9283\r\n9284\r\n9285\r\n9286\r\n9287\r\n9288\r\n9289\r\n9290\r\n9291\r\n9292\r\n9293\r\n9294\r\n9295\r\n9296\r\n9297\r\n9298\r\n9299\r\n9300\r\n9301\r\n9302\r\n9303\r\n9304\r\n9305\r\n9306\r\n9307\r\n9308\r\n9309\r\n9310\r\n9311\r\n9312\r\n9313\r\n9314\r\n9315\r\n9316\r\n9317\r\n9318\r\n9319\r\n9320\r\n9321\r\n9322\r\n9323\r\n9324\r\n9325\r\n9326\r\n9327\r\n9328\r\n9329\r\n9330\r\n9331\r\n9332\r\n9333\r\n9334\r\n9335\r\n9336\r\n9337\r\n9338\r\n9339\r\n9340\r\n9341\r\n9342\r\n9343\r\n9344\r\n9345\r\n9346\r\n9347\r\n9348\r\n9349\r\n9350\r\n9351\r\n9352\r\n9353\r\n9354\r\n9355\r\n9356\r\n9357\r\n9358\r\n9359\r\n9360\r\n9361\r\n9362\r\n9363\r\n9364\r\n9365\r\n9366\r\n9367\r\n9368\r\n9369\r\n9370\r\n9371\r\n9372\r\n9373\r\n9374\r\n9375\r\n9376\r\n9377\r\n9378\r\n9379\r\n9380\r\n9381\r\n9382\r\n9383\r\n9384\r\n9385\r\n9386\r\n9387\r\n9388\r\n9389\r\n9390\r\n9391\r\n9392\r\n9393\r\n9394\r\n9395\r\n9396\r\n9397\r\n9398\r\n9399\r\n9400\r\n9401\r\n9402\r\n9403\r\n9404\r\n9405\r\n9406\r\n9407\r\n9408\r\n9409\r\n9410\r\n9411\r\n9412\r\n9413\r\n9414\r\n9415\r\n9416\r\n9417\r\n9418\r\n9419\r\n9420\r\n9421\r\n9422\r\n9423\r\n9424\r\n9425\r\n9426\r\n9427\r\n9428\r\n9429\r\n9430\r\n9431\r\n9432\r\n9433\r\n9434\r\n9435\r\n9436\r\n9437\r\n9438\r\n9439\r\n9440\r\n9441\r\n9442\r\n9443\r\n9444\r\n9445\r\n9446\r\n9447\r\n9448\r\n9449\r\n9450\r\n9451\r\n9452\r\n9453\r\n9454\r\n9455\r\n9456\r\n9457\r\n9458\r\n9459\r\n9460\r\n9461\r\n9462\r\n9463\r\n9464\r\n9465\r\n9466\r\n9467\r\n9468\r\n9469\r\n9470\r\n9471\r\n9472\r\n9473\r\n9474\r\n9475\r\n9476\r\n9477\r\n9478\r\n9479\r\n9480\r\n9481\r\n9482\r\n9483\r\n9484\r\n9485\r\n9486\r\n9487\r\n9488\r\n9489\r\n9490\r\n9491\r\n9492\r\n9493\r\n9494\r\n9495\r\n9496\r\n9497\r\n9498\r\n9499\r\n9500\r\n9501\r\n9502\r\n9503\r\n9504\r\n9505\r\n9506\r\n9507\r\n9508\r\n9509\r\n9510\r\n9511\r\n9512\r\n9513\r\n9514\r\n9515\r\n9516\r\n9517\r\n9518\r\n9519\r\n9520\r\n9521\r\n9522\r\n9523\r\n9524\r\n9525\r\n9526\r\n9527\r\n9528\r\n9529\r\n9530\r\n9531\r\n9532\r\n9533\r\n9534\r\n9535\r\n9536\r\n9537\r\n9538\r\n9539\r\n9540\r\n9541\r\n9542\r\n9543\r\n9544\r\n9545\r\n9546\r\n9547\r\n9548\r\n9549\r\n9550\r\n9551\r\n9552\r\n9553\r\n9554\r\n9555\r\n9556\r\n9557\r\n9558\r\n9559\r\n9560\r\n9561\r\n9562\r\n9563\r\n9564\r\n9565\r\n9566\r\n9567\r\n9568\r\n9569\r\n9570\r\n9571\r\n9572\r\n9573\r\n9574\r\n9575\r\n9576\r\n9577\r\n9578\r\n9579\r\n9580\r\n9581\r\n9582\r\n9583\r\n9584\r\n9585\r\n9586\r\n9587\r\n9588\r\n9589\r\n9590\r\n9591\r\n9592\r\n9593\r\n9594\r\n9595\r\n9596\r\n9597\r\n9598\r\n9599\r\n9600\r\n9601\r\n9602\r\n9603\r\n9604\r\n9605\r\n9606\r\n9607\r\n9608\r\n9609\r\n9610\r\n9611\r\n9612\r\n9613\r\n9614\r\n9615\r\n9616\r\n9617\r\n9618\r\n9619\r\n9620\r\n9621\r\n9622\r\n9623\r\n9624\r\n9625\r\n9626\r\n9627\r\n9628\r\n9629\r\n9630\r\n9631\r\n9632\r\n9633\r\n9634\r\n9635\r\n9636\r\n9637\r\n9638\r\n9639\r\n9640\r\n9641\r\n9642\r\n9643\r\n9644\r\n9645\r\n9646\r\n9647\r\n9648\r\n9649\r\n9650\r\n9651\r\n9652\r\n9653\r\n9654\r\n9655\r\n9656\r\n9657\r\n9658\r\n9659\r\n9660\r\n9661\r\n9662\r\n9663\r\n9664\r\n9665\r\n9666\r\n9667\r\n9668\r\n9669\r\n9670\r\n9671\r\n9672\r\n9673\r\n9674\r\n9675\r\n9676\r\n9677\r\n9678\r\n9679\r\n9680\r\n9681\r\n9682\r\n9683\r\n9684\r\n9685\r\n9686\r\n9687\r\n9688\r\n9689\r\n9690\r\n9691\r\n9692\r\n9693\r\n9694\r\n9695\r\n9696\r\n9697\r\n9698\r\n9699\r\n9700\r\n9701\r\n9702\r\n9703\r\n9704\r\n9705\r\n9706\r\n9707\r\n9708\r\n9709\r\n9710\r\n9711\r\n9712\r\n9713\r\n9714\r\n9715\r\n9716\r\n9717\r\n9718\r\n9719\r\n9720\r\n9721\r\n9722\r\n9723\r\n9724\r\n9725\r\n9726\r\n9727\r\n9728\r\n9729\r\n9730\r\n9731\r\n9732\r\n9733\r\n9734\r\n9735\r\n9736\r\n9737\r\n9738\r\n9739\r\n9740\r\n9741\r\n9742\r\n9743\r\n9744\r\n9745\r\n9746\r\n9747\r\n9748\r\n9749\r\n9750\r\n9751\r\n9752\r\n9753\r\n9754\r\n9755\r\n9756\r\n9757\r\n9758\r\n9759\r\n9760\r\n9761\r\n9762\r\n9763\r\n9764\r\n9765\r\n9766\r\n9767\r\n9768\r\n9769\r\n9770\r\n9771\r\n9772\r\n9773\r\n9774\r\n9775\r\n9776\r\n9777\r\n9778\r\n9779\r\n9780\r\n9781\r\n9782\r\n9783\r\n9784\r\n9785\r\n9786\r\n9787\r\n9788\r\n9789\r\n9790\r\n9791\r\n9792\r\n9793\r\n9794\r\n9795\r\n9796\r\n9797\r\n9798\r\n9799\r\n9800\r\n9801\r\n9802\r\n9803\r\n9804\r\n9805\r\n9806\r\n9807\r\n9808\r\n9809\r\n9810\r\n9811\r\n9812\r\n9813\r\n9814\r\n9815\r\n9816\r\n9817\r\n9818\r\n9819\r\n9820\r\n9821\r\n9822\r\n9823\r\n9824\r\n9825\r\n9826\r\n9827\r\n9828\r\n9829\r\n9830\r\n9831\r\n9832\r\n9833\r\n9834\r\n9835\r\n9836\r\n9837\r\n9838\r\n9839\r\n9840\r\n9841\r\n9842\r\n9843\r\n9844\r\n9845\r\n9846\r\n9847\r\n9848\r\n9849\r\n9850\r\n9851\r\n9852\r\n9853\r\n9854\r\n9855\r\n9856\r\n9857\r\n9858\r\n9859\r\n9860\r\n9861\r\n9862\r\n9863\r\n9864\r\n9865\r\n9866\r\n9867\r\n9868\r\n9869\r\n9870\r\n9871\r\n9872\r\n9873\r\n9874\r\n9875\r\n9876\r\n9877\r\n9878\r\n9879\r\n9880\r\n9881\r\n9882\r\n9883\r\n9884\r\n9885\r\n9886\r\n9887\r\n9888\r\n9889\r\n9890\r\n9891\r\n9892\r\n9893\r\n9894\r\n9895\r\n9896\r\n9897\r\n9898\r\n9899\r\n9900\r\n9901\r\n9902\r\n9903\r\n9904\r\n9905\r\n9906\r\n9907\r\n9908\r\n9909\r\n9910\r\n9911\r\n9912\r\n9913\r\n9914\r\n9915\r\n9916\r\n9917\r\n9918\r\n9919\r\n9920\r\n9921\r\n9922\r\n9923\r\n9924\r\n9925\r\n9926\r\n9927\r\n9928\r\n9929\r\n9930\r\n9931\r\n9932\r\n9933\r\n9934\r\n9935\r\n9936\r\n9937\r\n9938\r\n9939\r\n9940\r\n9941\r\n9942\r\n9943\r\n9944\r\n9945\r\n9946\r\n9947\r\n9948\r\n9949\r\n9950\r\n9951\r\n9952\r\n9953\r\n9954\r\n9955\r\n9956\r\n9957\r\n9958\r\n9959\r\n9960\r\n9961\r\n9962\r\n9963\r\n9964\r\n9965\r\n9966\r\n9967\r\n9968\r\n9969\r\n9970\r\n9971\r\n9972\r\n9973\r\n9974\r\n9975\r\n9976\r\n9977\r\n9978\r\n9979\r\n9980\r\n9981\r\n9982\r\n9983\r\n9984\r\n9985\r\n9986\r\n9987\r\n9988\r\n9989\r\n9990\r\n9991\r\n9992\r\n9993\r\n9994\r\n9995\r\n9996\r\n9997\r\n9998\r\n9999\r\n10000\r\n10001\r\n10002\r\n10003\r\n10004\r\n10005\r\n10006\r\n10007\r\n10008\r\n10009\r\n10010\r\n10011\r\n10012\r\n10013\r\n10014\r\n10015\r\n10016\r\n10017\r\n10018\r\n10019\r\n10020\r\n10021\r\n10022\r\n10023\r\n10024\r\n10025\r\n10026\r\n10027\r\n10028\r\n10029\r\n10030\r\n10031\r\n10032\r\n10033\r\n10034\r\n10035\r\n10036\r\n10037\r\n10038\r\n10039\r\n10040\r\n10041\r\n10042\r\n10043\r\n10044\r\n10045\r\n10046\r\n10047\r\n10048\r\n10049\r\n10050\r\n10051\r\n10052\r\n10053\r\n10054\r\n10055\r\n10056\r\n10057\r\n10058\r\n10059\r\n10060\r\n10061\r\n10062\r\n10063\r\n10064\r\n10065\r\n10066\r\n10067\r\n10068\r\n10069\r\n10070\r\n10071\r\n10072\r\n10073\r\n10074\r\n10075\r\n10076\r\n10077\r\n10078\r\n10079\r\n10080\r\n10081\r\n10082\r\n10083\r\n10084\r\n10085\r\n10086\r\n10087\r\n10088\r\n10089\r\n10090\r\n10091\r\n10092\r\n10093\r\n10094\r\n10095\r\n10096\r\n10097\r\n10098\r\n10099\r\n10100\r\n10101\r\n10102\r\n10103\r\n10104\r\n10105\r\n10106\r\n10107\r\n10108\r\n10109\r\n10110\r\n10111\r\n10112\r\n10113\r\n10114\r\n10115\r\n10116\r\n10117\r\n10118\r\n10119\r\n10120\r\n10121\r\n10122\r\n10123\r\n10124\r\n10125\r\n10126\r\n10127\r\n10128\r\n10129\r\n10130\r\n10131\r\n10132\r\n10133\r\n10134\r\n10135\r\n10136\r\n10137\r\n10138\r\n10139\r\n10140\r\n10141\r\n10142\r\n10143\r\n10144\r\n10145\r\n10146\r\n10147\r\n10148\r\n10149\r\n10150\r\n10151\r\n10152\r\n10153\r\n10154\r\n10155\r\n10156\r\n10157\r\n10158\r\n10159\r\n10160\r\n10161\r\n10162\r\n10163\r\n10164\r\n10165\r\n10166\r\n10167\r\n10168\r\n10169\r\n10170\r\n10171\r\n10172\r\n10173\r\n10174\r\n10175\r\n10176\r\n10177\r\n10178\r\n10179\r\n10180\r\n10181\r\n10182\r\n10183\r\n10184\r\n10185\r\n10186\r\n10187\r\n10188\r\n10189\r\n10190\r\n10191\r\n10192\r\n10193\r\n10194\r\n10195\r\n10196\r\n10197\r\n10198\r\n10199\r\n10200\r\n10201\r\n10202\r\n10203\r\n10204\r\n10205\r\n10206\r\n10207\r\n10208\r\n10209\r\n10210\r\n10211\r\n10212\r\n10213\r\n10214\r\n10215\r\n10216\r\n10217\r\n10218\r\n10219\r\n10220\r\n10221\r\n10222\r\n10223\r\n10224\r\n10225\r\n10226\r\n10227\r\n10228\r\n10229\r\n10230\r\n10231\r\n10232\r\n10233\r\n10234\r\n10235\r\n10236\r\n10237\r\n10238\r\n10239\r\n10240\r\n10241\r\n10242\r\n10243\r\n10244\r\n10245\r\n10246\r\n10247\r\n10248\r\n10249\r\n10250\r\n10251\r\n10252\r\n10253\r\n10254\r\n10255\r\n10256\r\n10257\r\n10258\r\n10259\r\n10260\r\n10261\r\n10262\r\n10263\r\n10264\r\n10265\r\n10266\r\n10267\r\n10268\r\n10269\r\n10270\r\n10271\r\n10272\r\n10273\r\n10274\r\n10275\r\n10276\r\n10277\r\n10278\r\n10279\r\n10280\r\n10281\r\n10282\r\n10283\r\n10284\r\n10285\r\n10286\r\n10287\r\n10288\r\n10289\r\n10290\r\n10291\r\n10292\r\n10293\r\n10294\r\n10295\r\n10296\r\n10297\r\n10298\r\n10299\r\n10300\r\n10301\r\n10302\r\n10303\r\n10304\r\n10305\r\n10306\r\n10307\r\n10308\r\n10309\r\n10310\r\n10311\r\n10312\r\n10313\r\n10314\r\n10315\r\n10316\r\n10317\r\n10318\r\n10319\r\n10320\r\n10321\r\n10322\r\n10323\r\n10324\r\n10325\r\n10326\r\n10327\r\n10328\r\n10329\r\n10330\r\n10331\r\n10332\r\n10333\r\n10334\r\n10335\r\n10336\r\n10337\r\n10338\r\n10339\r\n10340\r\n10341\r\n10342\r\n10343\r\n10344\r\n10345\r\n10346\r\n10347\r\n10348\r\n10349\r\n10350\r\n10351\r\n10352\r\n10353\r\n10354\r\n10355\r\n10356\r\n10357\r\n10358\r\n10359\r\n10360\r\n10361\r\n10362\r\n10363\r\n10364\r\n10365\r\n10366\r\n10367\r\n10368\r\n10369\r\n10370\r\n10371\r\n10372\r\n10373\r\n10374\r\n10375\r\n10376\r\n10377\r\n10378\r\n10379\r\n10380\r\n10381\r\n10382\r\n10383\r\n10384\r\n10385\r\n10386\r\n10387\r\n10388\r\n10389\r\n10390\r\n10391\r\n10392\r\n10393\r\n10394\r\n10395\r\n10396\r\n10397\r\n10398\r\n10399\r\n10400\r\n10401\r\n10402\r\n10403\r\n10404\r\n10405\r\n10406\r\n10407\r\n10408\r\n10409\r\n10410\r\n10411\r\n10412\r\n10413\r\n10414\r\n10415\r\n10416\r\n10417\r\n10418\r\n10419\r\n10420\r\n10421\r\n10422\r\n10423\r\n10424\r\n10425\r\n10426\r\n10427\r\n10428\r\n10429\r\n10430\r\n10431\r\n10432\r\n10433\r\n10434\r\n10435\r\n10436\r\n10437\r\n10438\r\n10439\r\n10440\r\n10441\r\n10442\r\n10443\r\n10444\r\n10445\r\n10446\r\n10447\r\n10448\r\n10449\r\n10450\r\n10451\r\n10452\r\n10453\r\n10454\r\n10455\r\n10456\r\n10457\r\n10458\r\n10459\r\n10460\r\n10461\r\n10462\r\n10463\r\n10464\r\n10465\r\n10466\r\n10467\r\n10468\r\n10469\r\n10470\r\n10471\r\n10472\r\n10473\r\n10474\r\n10475\r\n10476\r\n10477\r\n10478\r\n10479\r\n10480\r\n10481\r\n10482\r\n10483\r\n10484\r\n10485\r\n10486\r\n10487\r\n10488\r\n10489\r\n10490\r\n10491\r\n10492\r\n10493\r\n10494\r\n10495\r\n10496\r\n10497\r\n10498\r\n10499\r\n10500\r\n10501\r\n10502\r\n10503\r\n10504\r\n10505\r\n10506\r\n10507\r\n10508\r\n10509\r\n10510\r\n10511\r\n10512\r\n10513\r\n10514\r\n10515\r\n10516\r\n10517\r\n10518\r\n10519\r\n10520\r\n10521\r\n10522\r\n10523\r\n10524\r\n10525\r\n10526\r\n10527\r\n10528\r\n10529\r\n10530\r\n10531\r\n10532\r\n10533\r\n10534\r\n10535\r\n10536\r\n10537\r\n10538\r\n10539\r\n10540\r\n10541\r\n10542\r\n10543\r\n10544\r\n10545\r\n10546\r\n10547\r\n10548\r\n10549\r\n10550\r\n10551\r\n10552\r\n10553\r\n10554\r\n10555\r\n10556\r\n10557\r\n10558\r\n10559\r\n10560\r\n10561\r\n10562\r\n10563\r\n10564\r\n10565\r\n10566\r\n10567\r\n10568\r\n10569\r\n10570\r\n10571\r\n10572\r\n10573\r\n10574\r\n10575\r\n10576\r\n10577\r\n10578\r\n10579\r\n10580\r\n10581\r\n10582\r\n10583\r\n10584\r\n10585\r\n10586\r\n10587\r\n10588\r\n10589\r\n10590\r\n10591\r\n10592\r\n10593\r\n10594\r\n10595\r\n10596\r\n10597\r\n10598\r\n10599\r\n10600\r\n10601\r\n10602\r\n10603\r\n10604\r\n10605\r\n10606\r\n10607\r\n10608\r\n10609\r\n10610\r\n10611\r\n10612\r\n10613\r\n10614\r\n10615\r\n10616\r\n10617\r\n10618\r\n10619\r\n10620\r\n10621\r\n10622\r\n10623\r\n10624\r\n10625\r\n10626\r\n10627\r\n10628\r\n10629\r\n10630\r\n10631\r\n10632\r\n10633\r\n10634\r\n10635\r\n10636\r\n10637\r\n10638\r\n10639\r\n10640\r\n10641\r\n10642\r\n10643\r\n10644\r\n10645\r\n10646\r\n10647\r\n10648\r\n10649\r\n10650\r\n10651\r\n10652\r\n10653\r\n10654\r\n10655\r\n10656\r\n10657\r\n10658\r\n10659\r\n10660\r\n10661\r\n10662\r\n10663\r\n10664\r\n10665\r\n10666\r\n10667\r\n10668\r\n10669\r\n10670\r\n10671\r\n10672\r\n10673\r\n10674\r\n10675\r\n10676\r\n10677\r\n10678\r\n10679\r\n10680\r\n10681\r\n10682\r\n10683\r\n10684\r\n10685\r\n10686\r\n10687\r\n10688\r\n10689\r\n10690\r\n10691\r\n10692\r\n10693\r\n10694\r\n10695\r\n10696\r\n10697\r\n10698\r\n10699\r\n10700\r\n10701\r\n10702\r\n10703\r\n10704\r\n10705\r\n10706\r\n10707\r\n10708\r\n10709\r\n10710\r\n10711\r\n10712\r\n10713\r\n10714\r\n10715\r\n10716\r\n10717\r\n10718\r\n10719\r\n10720\r\n10721\r\n10722\r\n10723\r\n10724\r\n10725\r\n10726\r\n10727\r\n10728\r\n10729\r\n10730\r\n10731\r\n10732\r\n10733\r\n10734\r\n10735\r\n10736\r\n10737\r\n10738\r\n10739\r\n10740\r\n10741\r\n10742\r\n10743\r\n10744\r\n10745\r\n10746\r\n10747\r\n10748\r\n10749\r\n10750\r\n10751\r\n10752\r\n10753\r\n10754\r\n10755\r\n10756\r\n10757\r\n10758\r\n10759\r\n10760\r\n10761\r\n10762\r\n10763\r\n10764\r\n10765\r\n10766\r\n10767\r\n10768\r\n10769\r\n10770\r\n10771\r\n10772\r\n10773\r\n10774\r\n10775\r\n10776\r\n10777\r\n10778\r\n10779\r\n10780\r\n10781\r\n10782\r\n10783\r\n10784\r\n10785\r\n10786\r\n10787\r\n10788\r\n10789\r\n10790\r\n10791\r\n10792\r\n10793\r\n10794\r\n10795\r\n10796\r\n10797\r\n10798\r\n10799\r\n10800\r\n10801\r\n10802\r\n10803\r\n10804\r\n10805\r\n10806\r\n10807\r\n10808\r\n10809\r\n10810\r\n10811\r\n10812\r\n10813\r\n10814\r\n10815\r\n10816\r\n10817\r\n10818\r\n10819\r\n10820\r\n10821\r\n10822\r\n10823\r\n10824\r\n10825\r\n10826\r\n10827\r\n10828\r\n10829\r\n10830\r\n10831\r\n10832\r\n10833\r\n10834\r\n10835\r\n10836\r\n10837\r\n10838\r\n10839\r\n10840\r\n10841\r\n10842\r\n10843\r\n10844\r\n10845\r\n10846\r\n10847\r\n10848\r\n10849\r\n10850\r\n10851\r\n10852\r\n10853\r\n10854\r\n10855\r\n10856\r\n10857\r\n10858\r\n10859\r\n10860\r\n10861\r\n10862\r\n10863\r\n10864\r\n10865\r\n10866\r\n10867\r\n10868\r\n10869\r\n10870\r\n10871\r\n10872\r\n10873\r\n10874\r\n10875\r\n10876\r\n10877\r\n10878\r\n10879\r\n10880\r\n10881\r\n10882\r\n10883\r\n10884\r\n10885\r\n10886\r\n10887\r\n10888\r\n10889\r\n10890\r\n10891\r\n10892\r\n10893\r\n10894\r\n10895\r\n10896\r\n10897\r\n10898\r\n10899\r\n10900\r\n10901\r\n10902\r\n10903\r\n10904\r\n10905\r\n10906\r\n10907\r\n10908\r\n10909\r\n10910\r\n10911\r\n10912\r\n10913\r\n10914\r\n10915\r\n10916\r\n10917\r\n10918\r\n10919\r\n10920\r\n10921\r\n10922\r\n10923\r\n10924\r\n10925\r\n10926\r\n10927\r\n10928\r\n10929\r\n10930\r\n10931\r\n10932\r\n10933\r\n10934\r\n10935\r\n10936\r\n10937\r\n10938\r\n10939\r\n10940\r\n10941\r\n10942\r\n10943\r\n10944\r\n10945\r\n10946\r\n10947\r\n10948\r\n10949\r\n10950\r\n10951\r\n10952\r\n10953\r\n10954\r\n10955\r\n10956\r\n10957\r\n10958\r\n10959\r\n10960\r\n10961\r\n10962\r\n10963\r\n10964\r\n10965\r\n10966\r\n10967\r\n10968\r\n10969\r\n10970\r\n10971\r\n10972\r\n10973\r\n10974\r\n10975\r\n10976\r\n10977\r\n10978\r\n10979\r\n10980\r\n10981\r\n10982\r\n10983\r\n10984\r\n10985\r\n10986\r\n10987\r\n10988\r\n10989\r\n10990\r\n10991\r\n10992\r\n10993\r\n10994\r\n10995\r\n10996\r\n10997\r\n10998\r\n10999\r\n11000\r\n11001\r\n11002\r\n11003\r\n11004\r\n11005\r\n11006\r\n11007\r\n11008\r\n11009\r\n11010\r\n11011\r\n11012\r\n11013\r\n11014\r\n11015\r\n11016\r\n11017\r\n11018\r\n11019\r\n11020\r\n11021\r\n11022\r\n11023\r\n11024\r\n11025\r\n11026\r\n11027\r\n11028\r\n11029\r\n11030\r\n11031\r\n11032\r\n11033\r\n11034\r\n11035\r\n11036\r\n11037\r\n11038\r\n11039\r\n11040\r\n11041\r\n11042\r\n11043\r\n11044\r\n11045\r\n11046\r\n11047\r\n11048\r\n11049\r\n11050\r\n11051\r\n11052\r\n11053\r\n11054\r\n11055\r\n11056\r\n11057\r\n11058\r\n11059\r\n11060\r\n11061\r\n11062\r\n11063\r\n11064\r\n11065\r\n11066\r\n11067\r\n11068\r\n11069\r\n11070\r\n11071\r\n11072\r\n11073\r\n11074\r\n11075\r\n11076\r\n11077\r\n11078\r\n11079\r\n11080\r\n11081\r\n11082\r\n11083\r\n11084\r\n11085\r\n11086\r\n11087\r\n11088\r\n11089\r\n11090\r\n11091\r\n11092\r\n11093\r\n11094\r\n11095\r\n11096\r\n11097\r\n11098\r\n11099\r\n11100\r\n11101\r\n11102\r\n11103\r\n11104\r\n11105\r\n11106\r\n11107\r\n11108\r\n11109\r\n11110\r\n11111\r\n11112\r\n11113\r\n11114\r\n11115\r\n11116\r\n11117\r\n11118\r\n11119\r\n11120\r\n11121\r\n11122\r\n11123\r\n11124\r\n11125\r\n11126\r\n11127\r\n11128\r\n11129\r\n11130\r\n11131\r\n11132\r\n11133\r\n11134\r\n11135\r\n11136\r\n11137\r\n11138\r\n11139\r\n11140\r\n11141\r\n11142\r\n11143\r\n11144\r\n11145\r\n11146\r\n11147\r\n11148\r\n11149\r\n11150\r\n11151\r\n11152\r\n11153\r\n11154\r\n11155\r\n11156\r\n11157\r\n11158\r\n11159\r\n11160\r\n11161\r\n11162\r\n11163\r\n11164\r\n11165\r\n11166\r\n11167\r\n11168\r\n11169\r\n11170\r\n11171\r\n11172\r\n11173\r\n11174\r\n11175\r\n11176\r\n11177\r\n11178\r\n11179\r\n11180\r\n11181\r\n11182\r\n11183\r\n11184\r\n11185\r\n11186\r\n11187\r\n11188\r\n11189\r\n11190\r\n11191\r\n11192\r\n11193\r\n11194\r\n11195\r\n11196\r\n11197\r\n11198\r\n11199\r\n11200\r\n11201\r\n11202\r\n11203\r\n11204\r\n11205\r\n11206\r\n11207\r\n11208\r\n11209\r\n11210\r\n11211\r\n11212\r\n11213\r\n11214\r\n11215\r\n11216\r\n11217\r\n11218\r\n11219\r\n11220\r\n11221\r\n11222\r\n11223\r\n11224\r\n11225\r\n11226\r\n11227\r\n11228\r\n11229\r\n11230\r\n11231\r\n11232\r\n11233\r\n11234\r\n11235\r\n11236\r\n11237\r\n11238\r\n11239\r\n11240\r\n11241\r\n11242\r\n11243\r\n11244\r\n11245\r\n11246\r\n11247\r\n11248\r\n11249\r\n11250\r\n11251\r\n11252\r\n11253\r\n11254\r\n11255\r\n11256\r\n11257\r\n11258\r\n11259\r\n11260\r\n11261\r\n11262\r\n11263\r\n11264\r\n11265\r\n11266\r\n11267\r\n11268\r\n11269\r\n11270\r\n11271\r\n11272\r\n11273\r\n11274\r\n11275\r\n11276\r\n11277\r\n11278\r\n11279\r\n11280\r\n11281\r\n11282\r\n11283\r\n11284\r\n11285\r\n11286\r\n11287\r\n11288\r\n11289\r\n11290\r\n11291\r\n11292\r\n11293\r\n11294\r\n11295\r\n11296\r\n11297\r\n11298\r\n11299\r\n11300\r\n11301\r\n11302\r\n11303\r\n11304\r\n11305\r\n11306\r\n11307\r\n11308\r\n11309\r\n11310\r\n11311\r\n11312\r\n11313\r\n11314\r\n11315\r\n11316\r\n11317\r\n11318\r\n11319\r\n11320\r\n11321\r\n11322\r\n11323\r\n11324\r\n11325\r\n11326\r\n11327\r\n11328\r\n11329\r\n11330\r\n11331\r\n11332\r\n11333\r\n11334\r\n11335\r\n11336\r\n11337\r\n11338\r\n11339\r\n11340\r\n11341\r\n11342\r\n11343\r\n11344\r\n11345\r\n11346\r\n11347\r\n11348\r\n11349\r\n11350\r\n11351\r\n11352\r\n11353\r\n11354\r\n11355\r\n11356\r\n11357\r\n11358\r\n11359\r\n11360\r\n11361\r\n11362\r\n11363\r\n11364\r\n11365\r\n11366\r\n11367\r\n11368\r\n11369\r\n11370\r\n11371\r\n11372\r\n11373\r\n11374\r\n11375\r\n11376\r\n11377\r\n11378\r\n11379\r\n11380\r\n11381\r\n11382\r\n11383\r\n11384\r\n11385\r\n11386\r\n11387\r\n11388\r\n11389\r\n11390\r\n11391\r\n11392\r\n11393\r\n11394\r\n11395\r\n11396\r\n11397\r\n11398\r\n11399\r\n11400\r\n11401\r\n11402\r\n11403\r\n11404\r\n11405\r\n11406\r\n11407\r\n11408\r\n11409\r\n11410\r\n11411\r\n11412\r\n11413\r\n11414\r\n11415\r\n11416\r\n11417\r\n11418\r\n11419\r\n11420\r\n11421\r\n11422\r\n11423\r\n11424\r\n11425\r\n11426\r\n11427\r\n11428\r\n11429\r\n11430\r\n11431\r\n11432\r\n11433\r\n11434\r\n11435\r\n11436\r\n11437\r\n11438\r\n11439\r\n11440\r\n11441\r\n11442\r\n11443\r\n11444\r\n11445\r\n11446\r\n11447\r\n11448\r\n11449\r\n11450\r\n11451\r\n11452\r\n11453\r\n11454\r\n11455\r\n11456\r\n11457\r\n11458\r\n11459\r\n11460\r\n11461\r\n11462\r\n11463\r\n11464\r\n11465\r\n11466\r\n11467\r\n11468\r\n11469\r\n11470\r\n11471\r\n11472\r\n11473\r\n11474\r\n11475\r\n11476\r\n11477\r\n11478\r\n11479\r\n11480\r\n11481\r\n11482\r\n11483\r\n11484\r\n11485\r\n11486\r\n11487\r\n11488\r\n11489\r\n11490\r\n11491\r\n11492\r\n11493\r\n11494\r\n11495\r\n11496\r\n11497\r\n11498\r\n11499\r\n11500\r\n11501\r\n11502\r\n11503\r\n11504\r\n11505\r\n11506\r\n11507\r\n11508\r\n11509\r\n11510\r\n11511\r\n11512\r\n11513\r\n11514\r\n11515\r\n11516\r\n11517\r\n11518\r\n11519\r\n11520\r\n11521\r\n11522\r\n11523\r\n11524\r\n11525\r\n11526\r\n11527\r\n11528\r\n11529\r\n11530\r\n11531\r\n11532\r\n11533\r\n11534\r\n11535\r\n11536\r\n11537\r\n11538\r\n11539\r\n11540\r\n11541\r\n11542\r\n11543\r\n11544\r\n11545\r\n11546\r\n11547\r\n11548\r\n11549\r\n11550\r\n11551\r\n11552\r\n11553\r\n11554\r\n11555\r\n11556\r\n11557\r\n11558\r\n11559\r\n11560\r\n11561\r\n11562\r\n11563\r\n11564\r\n11565\r\n11566\r\n11567\r\n11568\r\n11569\r\n11570\r\n11571\r\n11572\r\n11573\r\n11574\r\n11575\r\n11576\r\n11577\r\n11578\r\n11579\r\n11580\r\n11581\r\n11582\r\n11583\r\n11584\r\n11585\r\n11586\r\n11587\r\n11588\r\n11589\r\n11590\r\n11591\r\n11592\r\n11593\r\n11594\r\n11595\r\n11596\r\n11597\r\n11598\r\n11599\r\n11600\r\n11601\r\n11602\r\n11603\r\n11604\r\n11605\r\n11606\r\n11607\r\n11608\r\n11609\r\n11610\r\n11611\r\n11612\r\n11613\r\n11614\r\n11615\r\n11616\r\n11617\r\n11618\r\n11619\r\n11620\r\n11621\r\n11622\r\n11623\r\n11624\r\n11625\r\n11626\r\n11627\r\n11628\r\n11629\r\n11630\r\n11631\r\n11632\r\n11633\r\n11634\r\n11635\r\n11636\r\n11637\r\n11638\r\n11639\r\n11640\r\n11641\r\n11642\r\n11643\r\n11644\r\n11645\r\n11646\r\n11647\r\n11648\r\n11649\r\n11650\r\n11651\r\n11652\r\n11653\r\n11654\r\n11655\r\n11656\r\n11657\r\n11658\r\n11659\r\n11660\r\n11661\r\n11662\r\n11663\r\n11664\r\n11665\r\n11666\r\n11667\r\n11668\r\n11669\r\n11670\r\n11671\r\n11672\r\n11673\r\n11674\r\n11675\r\n11676\r\n11677\r\n11678\r\n11679\r\n11680\r\n11681\r\n11682\r\n11683\r\n11684\r\n11685\r\n11686\r\n11687\r\n11688\r\n11689\r\n11690\r\n11691\r\n11692\r\n11693\r\n11694\r\n11695\r\n11696\r\n11697\r\n11698\r\n11699\r\n11700\r\n11701\r\n11702\r\n11703\r\n11704\r\n11705\r\n11706\r\n11707\r\n11708\r\n11709\r\n11710\r\n11711\r\n11712\r\n11713\r\n11714\r\n11715\r\n11716\r\n11717\r\n11718\r\n11719\r\n11720\r\n11721\r\n11722\r\n11723\r\n11724\r\n11725\r\n11726\r\n11727\r\n11728\r\n11729\r\n11730\r\n11731\r\n11732\r\n11733\r\n11734\r\n11735\r\n11736\r\n11737\r\n11738\r\n11739\r\n11740\r\n11741\r\n11742\r\n11743\r\n11744\r\n11745\r\n11746\r\n11747\r\n11748\r\n11749\r\n11750\r\n11751\r\n11752\r\n11753\r\n11754\r\n11755\r\n11756\r\n11757\r\n11758\r\n11759\r\n11760\r\n11761\r\n11762\r\n11763\r\n11764\r\n11765\r\n11766\r\n11767\r\n11768\r\n11769\r\n11770\r\n11771\r\n11772\r\n11773\r\n11774\r\n11775\r\n11776\r\n11777\r\n11778\r\n11779\r\n11780\r\n11781\r\n11782\r\n11783\r\n11784\r\n11785\r\n11786\r\n11787\r\n11788\r\n11789\r\n11790\r\n11791\r\n11792\r\n11793\r\n11794\r\n11795\r\n11796\r\n11797\r\n11798\r\n11799\r\n11800\r\n11801\r\n11802\r\n11803\r\n11804\r\n11805\r\n11806\r\n11807\r\n11808\r\n11809\r\n11810\r\n11811\r\n11812\r\n11813\r\n11814\r\n11815\r\n11816\r\n11817\r\n11818\r\n11819\r\n11820\r\n11821\r\n11822\r\n11823\r\n11824\r\n11825\r\n11826\r\n11827\r\n11828\r\n11829\r\n11830\r\n11831\r\n11832\r\n11833\r\n11834\r\n11835\r\n11836\r\n11837\r\n11838\r\n11839\r\n11840\r\n11841\r\n11842\r\n11843\r\n11844\r\n11845\r\n11846\r\n11847\r\n11848\r\n11849\r\n11850\r\n11851\r\n11852\r\n11853\r\n11854\r\n11855\r\n11856\r\n11857\r\n11858\r\n11859\r\n11860\r\n11861\r\n11862\r\n11863\r\n11864\r\n11865\r\n11866\r\n11867\r\n11868\r\n11869\r\n11870\r\n11871\r\n11872\r\n11873\r\n11874\r\n11875\r\n11876\r\n11877\r\n11878\r\n11879\r\n11880\r\n11881\r\n11882\r\n11883\r\n11884\r\n11885\r\n11886\r\n11887\r\n11888\r\n11889\r\n11890\r\n11891\r\n11892\r\n11893\r\n11894\r\n11895\r\n11896\r\n11897\r\n11898\r\n11899\r\n11900\r\n11901\r\n11902\r\n11903\r\n11904\r\n11905\r\n11906\r\n11907\r\n11908\r\n11909\r\n11910\r\n11911\r\n11912\r\n11913\r\n11914\r\n11915\r\n11916\r\n11917\r\n11918\r\n11919\r\n11920\r\n11921\r\n11922\r\n11923\r\n11924\r\n11925\r\n11926\r\n11927\r\n11928\r\n11929\r\n11930\r\n11931\r\n11932\r\n11933\r\n11934\r\n11935\r\n11936\r\n11937\r\n11938\r\n11939\r\n11940\r\n11941\r\n11942\r\n11943\r\n11944\r\n11945\r\n11946\r\n11947\r\n11948\r\n11949\r\n11950\r\n11951\r\n11952\r\n11953\r\n11954\r\n11955\r\n11956\r\n11957\r\n11958\r\n11959\r\n11960\r\n11961\r\n11962\r\n11963\r\n11964\r\n11965\r\n11966\r\n11967\r\n11968\r\n11969\r\n11970\r\n11971\r\n11972\r\n11973\r\n11974\r\n11975\r\n11976\r\n11977\r\n11978\r\n11979\r\n11980\r\n11981\r\n11982\r\n11983\r\n11984\r\n11985\r\n11986\r\n11987\r\n11988\r\n11989\r\n11990\r\n11991\r\n11992\r\n11993\r\n11994\r\n11995\r\n11996\r\n11997\r\n11998\r\n11999\r\n12000\r\n12001\r\n12002\r\n12003\r\n12004\r\n12005\r\n12006\r\n12007\r\n12008\r\n12009\r\n12010\r\n12011\r\n12012\r\n12013\r\n12014\r\n12015\r\n12016\r\n12017\r\n12018\r\n12019\r\n12020\r\n12021\r\n12022\r\n12023\r\n12024\r\n12025\r\n12026\r\n12027\r\n12028\r\n12029\r\n12030\r\n12031\r\n12032\r\n12033\r\n12034\r\n12035\r\n12036\r\n12037\r\n12038\r\n12039\r\n12040\r\n12041\r\n12042\r\n12043\r\n12044\r\n12045\r\n12046\r\n12047\r\n12048\r\n12049\r\n12050\r\n12051\r\n12052\r\n12053\r\n12054\r\n12055\r\n12056\r\n12057\r\n12058\r\n12059\r\n12060\r\n12061\r\n12062\r\n12063\r\n12064\r\n12065\r\n12066\r\n12067\r\n12068\r\n12069\r\n12070\r\n12071\r\n12072\r\n12073\r\n12074\r\n12075\r\n12076\r\n12077\r\n12078\r\n12079\r\n12080\r\n12081\r\n12082\r\n12083\r\n12084\r\n12085\r\n12086\r\n12087\r\n12088\r\n12089\r\n12090\r\n12091\r\n12092\r\n12093\r\n12094\r\n12095\r\n12096\r\n12097\r\n12098\r\n12099\r\n12100\r\n12101\r\n12102\r\n12103\r\n12104\r\n12105\r\n12106\r\n12107\r\n12108\r\n12109\r\n12110\r\n12111\r\n12112\r\n12113\r\n12114\r\n12115\r\n12116\r\n12117\r\n12118\r\n12119\r\n12120\r\n12121\r\n12122\r\n12123\r\n12124\r\n12125\r\n12126\r\n12127\r\n12128\r\n12129\r\n12130\r\n12131\r\n12132\r\n12133\r\n12134\r\n12135\r\n12136\r\n12137\r\n12138\r\n12139\r\n12140\r\n12141\r\n12142\r\n12143\r\n12144\r\n12145\r\n12146\r\n12147\r\n12148\r\n12149\r\n12150\r\n12151\r\n12152\r\n12153\r\n12154\r\n12155\r\n12156\r\n12157\r\n12158\r\n12159\r\n12160\r\n12161\r\n12162\r\n12163\r\n12164\r\n12165\r\n12166\r\n12167\r\n12168\r\n12169\r\n12170\r\n12171\r\n12172\r\n12173\r\n12174\r\n12175\r\n12176\r\n12177\r\n12178\r\n12179\r\n12180\r\n12181\r\n12182\r\n12183\r\n12184\r\n12185\r\n12186\r\n12187\r\n12188\r\n12189\r\n12190\r\n12191\r\n12192\r\n12193\r\n12194\r\n12195\r\n12196\r\n12197\r\n12198\r\n12199\r\n12200\r\n12201\r\n12202\r\n12203\r\n12204\r\n12205\r\n12206\r\n12207\r\n12208\r\n12209\r\n12210\r\n12211\r\n12212\r\n12213\r\n12214\r\n12215\r\n12216\r\n12217\r\n12218\r\n12219\r\n12220\r\n12221\r\n12222\r\n12223\r\n12224\r\n12225\r\n12226\r\n12227\r\n12228\r\n12229\r\n12230\r\n12231\r\n12232\r\n12233\r\n12234\r\n12235\r\n12236\r\n12237\r\n12238\r\n12239\r\n12240\r\n12241\r\n12242\r\n12243\r\n12244\r\n12245\r\n12246\r\n12247\r\n12248\r\n12249\r\n12250\r\n12251\r\n12252\r\n12253\r\n12254\r\n12255\r\n12256\r\n12257\r\n12258\r\n12259\r\n12260\r\n12261\r\n12262\r\n12263\r\n12264\r\n12265\r\n12266\r\n12267\r\n12268\r\n12269\r\n12270\r\n12271\r\n12272\r\n12273\r\n12274\r\n12275\r\n12276\r\n12277\r\n12278\r\n12279\r\n12280\r\n12281\r\n12282\r\n12283\r\n12284\r\n12285\r\n12286\r\n12287\r\n12288\r\n12289\r\n12290\r\n12291\r\n12292\r\n12293\r\n12294\r\n12295\r\n12296\r\n12297\r\n12298\r\n12299\r\n12300\r\n12301\r\n12302\r\n12303\r\n12304\r\n12305\r\n12306\r\n12307\r\n12308\r\n12309\r\n12310\r\n12311\r\n12312\r\n12313\r\n12314\r\n12315\r\n12316\r\n12317\r\n12318\r\n12319\r\n12320\r\n12321\r\n12322\r\n12323\r\n12324\r\n12325\r\n12326\r\n12327\r\n12328\r\n12329\r\n12330\r\n12331\r\n12332\r\n12333\r\n12334\r\n12335\r\n12336\r\n12337\r\n12338\r\n12339\r\n12340\r\n12341\r\n12342\r\n12343\r\n12344\r\n12345\r\n12346\r\n12347\r\n12348\r\n12349\r\n12350\r\n12351\r\n12352\r\n12353\r\n12354\r\n12355\r\n12356\r\n12357\r\n12358\r\n12359\r\n12360\r\n12361\r\n12362\r\n12363\r\n12364\r\n12365\r\n12366\r\n12367\r\n12368\r\n12369\r\n12370\r\n12371\r\n12372\r\n12373\r\n12374\r\n12375\r\n12376\r\n12377\r\n12378\r\n12379\r\n12380\r\n12381\r\n12382\r\n12383\r\n12384\r\n12385\r\n12386\r\n12387\r\n12388\r\n12389\r\n12390\r\n12391\r\n12392\r\n12393\r\n12394\r\n12395\r\n12396\r\n12397\r\n12398\r\n12399\r\n12400\r\n12401\r\n12402\r\n12403\r\n12404\r\n12405\r\n12406\r\n12407\r\n12408\r\n12409\r\n12410\r\n12411\r\n12412\r\n12413\r\n12414\r\n12415\r\n12416\r\n12417\r\n12418\r\n12419\r\n12420\r\n12421\r\n12422\r\n12423\r\n12424\r\n12425\r\n12426\r\n12427\r\n12428\r\n12429\r\n12430\r\n12431\r\n12432\r\n12433\r\n12434\r\n12435\r\n12436\r\n12437\r\n12438\r\n12439\r\n12440\r\n12441\r\n12442\r\n12443\r\n12444\r\n12445\r\n12446\r\n12447\r\n12448\r\n12449\r\n12450\r\n12451\r\n12452\r\n12453\r\n12454\r\n12455\r\n12456\r\n12457\r\n12458\r\n12459\r\n12460\r\n12461\r\n12462\r\n12463\r\n12464\r\n12465\r\n12466\r\n12467\r\n12468\r\n12469\r\n12470\r\n12471\r\n12472\r\n12473\r\n12474\r\n12475\r\n12476\r\n12477\r\n12478\r\n12479\r\n12480\r\n12481\r\n12482\r\n12483\r\n12484\r\n12485\r\n12486\r\n12487\r\n12488\r\n12489\r\n12490\r\n12491\r\n12492\r\n12493\r\n12494\r\n12495\r\n12496\r\n12497\r\n12498\r\n12499\r\n12500\r\n12501\r\n12502\r\n12503\r\n12504\r\n12505\r\n12506\r\n12507\r\n12508\r\n12509\r\n12510\r\n12511\r\n12512\r\n12513\r\n12514\r\n12515\r\n12516\r\n12517\r\n12518\r\n12519\r\n12520\r\n12521\r\n12522\r\n12523\r\n12524\r\n12525\r\n12526\r\n12527\r\n12528\r\n12529\r\n12530\r\n12531\r\n12532\r\n12533\r\n12534\r\n12535\r\n12536\r\n12537\r\n12538\r\n12539\r\n12540\r\n12541\r\n12542\r\n12543\r\n12544\r\n12545\r\n12546\r\n12547\r\n12548\r\n12549\r\n12550\r\n12551\r\n12552\r\n12553\r\n12554\r\n12555\r\n12556\r\n12557\r\n12558\r\n12559\r\n12560\r\n12561\r\n12562\r\n12563\r\n12564\r\n12565\r\n12566\r\n12567\r\n12568\r\n12569\r\n12570\r\n12571\r\n12572\r\n12573\r\n12574\r\n12575\r\n12576\r\n12577\r\n12578\r\n12579\r\n12580\r\n12581\r\n12582\r\n12583\r\n12584\r\n12585\r\n12586\r\n12587\r\n12588\r\n12589\r\n12590\r\n12591\r\n12592\r\n12593\r\n12594\r\n12595\r\n12596\r\n12597\r\n12598\r\n12599\r\n12600\r\n12601\r\n12602\r\n12603\r\n12604\r\n12605\r\n12606\r\n12607\r\n12608\r\n12609\r\n12610\r\n12611\r\n12612\r\n12613\r\n12614\r\n12615\r\n12616\r\n12617\r\n12618\r\n12619\r\n12620\r\n12621\r\n12622\r\n12623\r\n12624\r\n12625\r\n12626\r\n12627\r\n12628\r\n12629\r\n12630\r\n12631\r\n12632\r\n12633\r\n12634\r\n12635\r\n12636\r\n12637\r\n12638\r\n12639\r\n12640\r\n12641\r\n12642\r\n12643\r\n12644\r\n12645\r\n12646\r\n12647\r\n12648\r\n12649\r\n12650\r\n12651\r\n12652\r\n12653\r\n12654\r\n12655\r\n12656\r\n12657\r\n12658\r\n12659\r\n12660\r\n12661\r\n12662\r\n12663\r\n12664\r\n12665\r\n12666\r\n12667\r\n12668\r\n12669\r\n12670\r\n12671\r\n12672\r\n12673\r\n12674\r\n12675\r\n12676\r\n12677\r\n12678\r\n12679\r\n12680\r\n12681\r\n12682\r\n12683\r\n12684\r\n12685\r\n12686\r\n12687\r\n12688\r\n12689\r\n12690\r\n12691\r\n12692\r\n12693\r\n12694\r\n12695\r\n12696\r\n12697\r\n12698\r\n12699\r\n12700\r\n12701\r\n12702\r\n12703\r\n12704\r\n12705\r\n12706\r\n12707\r\n12708\r\n12709\r\n12710\r\n12711\r\n12712\r\n12713\r\n12714\r\n12715\r\n12716\r\n12717\r\n12718\r\n12719\r\n12720\r\n12721\r\n12722\r\n12723\r\n12724\r\n12725\r\n12726\r\n12727\r\n12728\r\n12729\r\n12730\r\n12731\r\n12732\r\n12733\r\n12734\r\n12735\r\n12736\r\n12737\r\n12738\r\n12739\r\n12740\r\n12741\r\n12742\r\n12743\r\n12744\r\n12745\r\n12746\r\n12747\r\n12748\r\n12749\r\n12750\r\n12751\r\n12752\r\n12753\r\n12754\r\n12755\r\n12756\r\n12757\r\n12758\r\n12759\r\n12760\r\n12761\r\n12762\r\n12763\r\n12764\r\n12765\r\n12766\r\n12767\r\n12768\r\n12769\r\n12770\r\n12771\r\n12772\r\n12773\r\n12774\r\n12775\r\n12776\r\n12777\r\n12778\r\n12779\r\n12780\r\n12781\r\n12782\r\n12783\r\n12784\r\n12785\r\n12786\r\n12787\r\n12788\r\n12789\r\n12790\r\n12791\r\n12792\r\n12793\r\n12794\r\n12795\r\n12796\r\n12797\r\n12798\r\n12799\r\n12800\r\n12801\r\n12802\r\n12803\r\n12804\r\n12805\r\n12806\r\n12807\r\n12808\r\n12809\r\n12810\r\n12811\r\n12812\r\n12813\r\n12814\r\n12815\r\n12816\r\n12817\r\n12818\r\n12819\r\n12820\r\n12821\r\n12822\r\n12823\r\n12824\r\n12825\r\n12826\r\n12827\r\n12828\r\n12829\r\n12830\r\n12831\r\n12832\r\n12833\r\n12834\r\n12835\r\n12836\r\n12837\r\n12838\r\n12839\r\n12840\r\n12841\r\n12842\r\n12843\r\n12844\r\n12845\r\n12846\r\n12847\r\n12848\r\n12849\r\n12850\r\n12851\r\n12852\r\n12853\r\n12854\r\n12855\r\n12856\r\n12857\r\n12858\r\n12859\r\n12860\r\n12861\r\n12862\r\n12863\r\n12864\r\n12865\r\n12866\r\n12867\r\n12868\r\n12869\r\n12870\r\n12871\r\n12872\r\n12873\r\n12874\r\n12875\r\n12876\r\n12877\r\n12878\r\n12879\r\n12880\r\n12881\r\n12882\r\n12883\r\n12884\r\n12885\r\n12886\r\n12887\r\n12888\r\n12889\r\n12890\r\n12891\r\n12892\r\n12893\r\n12894\r\n12895\r\n12896\r\n12897\r\n12898\r\n12899\r\n12900\r\n12901\r\n12902\r\n12903\r\n12904\r\n12905\r\n12906\r\n12907\r\n12908\r\n12909\r\n12910\r\n12911\r\n12912\r\n12913\r\n12914\r\n12915\r\n12916\r\n12917\r\n12918\r\n12919\r\n12920\r\n12921\r\n12922\r\n12923\r\n12924\r\n12925\r\n12926\r\n12927\r\n12928\r\n12929\r\n12930\r\n12931\r\n12932\r\n12933\r\n12934\r\n12935\r\n12936\r\n12937\r\n12938\r\n12939\r\n12940\r\n12941\r\n12942\r\n12943\r\n12944\r\n12945\r\n12946\r\n12947\r\n12948\r\n12949\r\n12950\r\n12951\r\n12952\r\n12953\r\n12954\r\n12955\r\n12956\r\n12957\r\n12958\r\n12959\r\n12960\r\n12961\r\n12962\r\n12963\r\n12964\r\n12965\r\n12966\r\n12967\r\n12968\r\n12969\r\n12970\r\n12971\r\n12972\r\n12973\r\n12974\r\n12975\r\n12976\r\n12977\r\n12978\r\n12979\r\n12980\r\n12981\r\n12982\r\n12983\r\n12984\r\n12985\r\n12986\r\n12987\r\n12988\r\n12989\r\n12990\r\n12991\r\n12992\r\n12993\r\n12994\r\n12995\r\n12996\r\n12997\r\n12998\r\n12999\r\n13000\r\n13001\r\n13002\r\n13003\r\n13004\r\n13005\r\n13006\r\n13007\r\n13008\r\n13009\r\n13010\r\n13011\r\n13012\r\n13013\r\n13014\r\n13015\r\n13016\r\n13017\r\n13018\r\n13019\r\n13020\r\n13021\r\n13022\r\n13023\r\n13024\r\n13025\r\n13026\r\n13027\r\n13028\r\n13029\r\n13030\r\n13031\r\n13032\r\n13033\r\n13034\r\n13035\r\n13036\r\n13037\r\n13038\r\n13039\r\n13040\r\n13041\r\n13042\r\n13043\r\n13044\r\n13045\r\n13046\r\n13047\r\n13048\r\n13049\r\n13050\r\n13051\r\n13052\r\n13053\r\n13054\r\n13055\r\n13056\r\n13057\r\n13058\r\n13059\r\n13060\r\n13061\r\n13062\r\n13063\r\n13064\r\n13065\r\n13066\r\n13067\r\n13068\r\n13069\r\n13070\r\n13071\r\n13072\r\n13073\r\n13074\r\n13075\r\n13076\r\n13077\r\n13078\r\n13079\r\n13080\r\n13081\r\n13082\r\n13083\r\n13084\r\n13085\r\n13086\r\n13087\r\n13088\r\n13089\r\n13090\r\n13091\r\n13092\r\n13093\r\n13094\r\n13095\r\n13096\r\n13097\r\n13098\r\n13099\r\n13100\r\n13101\r\n13102\r\n13103\r\n13104\r\n13105\r\n13106\r\n13107\r\n13108\r\n13109\r\n13110\r\n13111\r\n13112\r\n13113\r\n13114\r\n13115\r\n13116\r\n13117\r\n13118\r\n13119\r\n13120\r\n13121\r\n13122\r\n13123\r\n13124\r\n13125\r\n13126\r\n13127\r\n13128\r\n13129\r\n13130\r\n13131\r\n13132\r\n13133\r\n13134\r\n13135\r\n13136\r\n13137\r\n13138\r\n13139\r\n13140\r\n13141\r\n13142\r\n13143\r\n13144\r\n13145\r\n13146\r\n13147\r\n13148\r\n13149\r\n13150\r\n13151\r\n13152\r\n13153\r\n13154\r\n13155\r\n13156\r\n13157\r\n13158\r\n13159\r\n13160\r\n13161\r\n13162\r\n13163\r\n13164\r\n13165\r\n13166\r\n13167\r\n13168\r\n13169\r\n13170\r\n13171\r\n13172\r\n13173\r\n13174\r\n13175\r\n13176\r\n13177\r\n13178\r\n13179\r\n13180\r\n13181\r\n13182\r\n13183\r\n13184\r\n13185\r\n13186\r\n13187\r\n13188\r\n13189\r\n13190\r\n13191\r\n13192\r\n13193\r\n13194\r\n13195\r\n13196\r\n13197\r\n13198\r\n13199\r\n13200\r\n13201\r\n13202\r\n13203\r\n13204\r\n13205\r\n13206\r\n13207\r\n13208\r\n13209\r\n13210\r\n13211\r\n13212\r\n13213\r\n13214\r\n13215\r\n13216\r\n13217\r\n13218\r\n13219\r\n13220\r\n13221\r\n13222\r\n13223\r\n13224\r\n13225\r\n13226\r\n13227\r\n13228\r\n13229\r\n13230\r\n13231\r\n13232\r\n13233\r\n13234\r\n13235\r\n13236\r\n13237\r\n13238\r\n13239\r\n13240\r\n13241\r\n13242\r\n13243\r\n13244\r\n13245\r\n13246\r\n13247\r\n13248\r\n13249\r\n13250\r\n13251\r\n13252\r\n13253\r\n13254\r\n13255\r\n13256\r\n13257\r\n13258\r\n13259\r\n13260\r\n13261\r\n13262\r\n13263\r\n13264\r\n13265\r\n13266\r\n13267\r\n13268\r\n13269\r\n13270\r\n13271\r\n13272\r\n13273\r\n13274\r\n13275\r\n13276\r\n13277\r\n13278\r\n13279\r\n13280\r\n13281\r\n13282\r\n13283\r\n13284\r\n13285\r\n13286\r\n13287\r\n13288\r\n13289\r\n13290\r\n13291\r\n13292\r\n13293\r\n13294\r\n13295\r\n13296\r\n13297\r\n13298\r\n13299\r\n13300\r\n13301\r\n13302\r\n13303\r\n13304\r\n13305\r\n13306\r\n13307\r\n13308\r\n13309\r\n13310\r\n13311\r\n13312\r\n13313\r\n13314\r\n13315\r\n13316\r\n13317\r\n13318\r\n13319\r\n13320\r\n13321\r\n13322\r\n13323\r\n13324\r\n13325\r\n13326\r\n13327\r\n13328\r\n13329\r\n13330\r\n13331\r\n13332\r\n13333\r\n13334\r\n13335\r\n13336\r\n13337\r\n13338\r\n13339\r\n13340\r\n13341\r\n13342\r\n13343\r\n13344\r\n13345\r\n13346\r\n13347\r\n13348\r\n13349\r\n13350\r\n13351\r\n13352\r\n13353\r\n13354\r\n13355\r\n13356\r\n13357\r\n13358\r\n13359\r\n13360\r\n13361\r\n13362\r\n13363\r\n13364\r\n13365\r\n13366\r\n13367\r\n13368\r\n13369\r\n13370\r\n13371\r\n13372\r\n13373\r\n13374\r\n13375\r\n13376\r\n13377\r\n13378\r\n13379\r\n13380\r\n13381\r\n13382\r\n13383\r\n13384\r\n13385\r\n13386\r\n13387\r\n13388\r\n13389\r\n13390\r\n13391\r\n13392\r\n13393\r\n13394\r\n13395\r\n13396\r\n13397\r\n13398\r\n13399\r\n13400\r\n13401\r\n13402\r\n13403\r\n13404\r\n13405\r\n13406\r\n13407\r\n13408\r\n13409\r\n13410\r\n13411\r\n13412\r\n13413\r\n13414\r\n13415\r\n13416\r\n13417\r\n13418\r\n13419\r\n13420\r\n13421\r\n13422\r\n13423\r\n13424\r\n13425\r\n13426\r\n13427\r\n13428\r\n13429\r\n13430\r\n13431\r\n13432\r\n13433\r\n13434\r\n13435\r\n13436\r\n13437\r\n13438\r\n13439\r\n13440\r\n13441\r\n13442\r\n13443\r\n13444\r\n13445\r\n13446\r\n13447\r\n13448\r\n13449\r\n13450\r\n13451\r\n13452\r\n13453\r\n13454\r\n13455\r\n13456\r\n13457\r\n13458\r\n13459\r\n13460\r\n13461\r\n13462\r\n13463\r\n13464\r\n13465\r\n13466\r\n13467\r\n13468\r\n13469\r\n13470\r\n13471\r\n13472\r\n13473\r\n13474\r\n13475\r\n13476\r\n13477\r\n13478\r\n13479\r\n13480\r\n13481\r\n13482\r\n13483\r\n13484\r\n13485\r\n13486\r\n13487\r\n13488\r\n13489\r\n13490\r\n13491\r\n13492\r\n13493\r\n13494\r\n13495\r\n13496\r\n13497\r\n13498\r\n13499\r\n13500\r\n13501\r\n13502\r\n13503\r\n13504\r\n13505\r\n13506\r\n13507\r\n13508\r\n13509\r\n13510\r\n13511\r\n13512\r\n13513\r\n13514\r\n13515\r\n13516\r\n13517\r\n13518\r\n13519\r\n13520\r\n13521\r\n13522\r\n13523\r\n13524\r\n13525\r\n13526\r\n13527\r\n13528\r\n13529\r\n13530\r\n13531\r\n13532\r\n13533\r\n13534\r\n13535\r\n13536\r\n13537\r\n13538\r\n13539\r\n13540\r\n13541\r\n13542\r\n13543\r\n13544\r\n13545\r\n13546\r\n13547\r\n13548\r\n13549\r\n13550\r\n13551\r\n13552\r\n13553\r\n13554\r\n13555\r\n13556\r\n13557\r\n13558\r\n13559\r\n13560\r\n13561\r\n13562\r\n13563\r\n13564\r\n13565\r\n13566\r\n13567\r\n13568\r\n13569\r\n13570\r\n13571\r\n13572\r\n13573\r\n13574\r\n13575\r\n13576\r\n13577\r\n13578\r\n13579\r\n13580\r\n13581\r\n13582\r\n13583\r\n13584\r\n13585\r\n13586\r\n13587\r\n13588\r\n13589\r\n13590\r\n13591\r\n13592\r\n13593\r\n13594\r\n13595\r\n13596\r\n13597\r\n13598\r\n13599\r\n13600\r\n13601\r\n13602\r\n13603\r\n13604\r\n13605\r\n13606\r\n13607\r\n13608\r\n13609\r\n13610\r\n13611\r\n13612\r\n13613\r\n13614\r\n13615\r\n13616\r\n13617\r\n13618\r\n13619\r\n13620\r\n13621\r\n13622\r\n13623\r\n13624\r\n13625\r\n13626\r\n13627\r\n13628\r\n13629\r\n13630\r\n13631\r\n13632\r\n13633\r\n13634\r\n13635\r\n13636\r\n13637\r\n13638\r\n13639\r\n13640\r\n13641\r\n13642\r\n13643\r\n13644\r\n13645\r\n13646\r\n13647\r\n13648\r\n13649\r\n13650\r\n13651\r\n13652\r\n13653\r\n13654\r\n13655\r\n13656\r\n13657\r\n13658\r\n13659\r\n13660\r\n13661\r\n13662\r\n13663\r\n13664\r\n13665\r\n13666\r\n13667\r\n13668\r\n13669\r\n13670\r\n13671\r\n13672\r\n13673\r\n13674\r\n13675\r\n13676\r\n13677\r\n13678\r\n13679\r\n13680\r\n13681\r\n13682\r\n13683\r\n13684\r\n13685\r\n13686\r\n13687\r\n13688\r\n13689\r\n13690\r\n13691\r\n13692\r\n13693\r\n13694\r\n13695\r\n13696\r\n13697\r\n13698\r\n13699\r\n13700\r\n13701\r\n13702\r\n13703\r\n13704\r\n13705\r\n13706\r\n13707\r\n13708\r\n13709\r\n13710\r\n13711\r\n13712\r\n13713\r\n13714\r\n13715\r\n13716\r\n13717\r\n13718\r\n13719\r\n13720\r\n13721\r\n13722\r\n13723\r\n13724\r\n13725\r\n13726\r\n13727\r\n13728\r\n13729\r\n13730\r\n13731\r\n13732\r\n13733\r\n13734\r\n13735\r\n13736\r\n13737\r\n13738\r\n13739\r\n13740\r\n13741\r\n13742\r\n13743\r\n13744\r\n13745\r\n13746\r\n13747\r\n13748\r\n13749\r\n13750\r\n13751\r\n13752\r\n13753\r\n13754\r\n13755\r\n13756\r\n13757\r\n13758\r\n13759\r\n13760\r\n13761\r\n13762\r\n13763\r\n13764\r\n13765\r\n13766\r\n13767\r\n13768\r\n13769\r\n13770\r\n13771\r\n13772\r\n13773\r\n13774\r\n13775\r\n13776\r\n13777\r\n13778\r\n13779\r\n13780\r\n13781\r\n13782\r\n13783\r\n13784\r\n13785\r\n13786\r\n13787\r\n13788\r\n13789\r\n13790\r\n13791\r\n13792\r\n13793\r\n13794\r\n13795\r\n13796\r\n13797\r\n13798\r\n13799\r\n13800\r\n13801\r\n13802\r\n13803\r\n13804\r\n13805\r\n13806\r\n13807\r\n13808\r\n13809\r\n13810\r\n13811\r\n13812\r\n13813\r\n13814\r\n13815\r\n13816\r\n13817\r\n13818\r\n13819\r\n13820\r\n13821\r\n13822\r\n13823\r\n13824\r\n13825\r\n13826\r\n13827\r\n13828\r\n13829\r\n13830\r\n13831\r\n13832\r\n13833\r\n13834\r\n13835\r\n13836\r\n13837\r\n13838\r\n13839\r\n13840\r\n13841\r\n13842\r\n13843\r\n13844\r\n13845\r\n13846\r\n13847\r\n13848\r\n13849\r\n13850\r\n13851\r\n13852\r\n13853\r\n13854\r\n13855\r\n13856\r\n13857\r\n13858\r\n13859\r\n13860\r\n13861\r\n13862\r\n13863\r\n13864\r\n13865\r\n13866\r\n13867\r\n13868\r\n13869\r\n13870\r\n13871\r\n13872\r\n13873\r\n13874\r\n13875\r\n13876\r\n13877\r\n13878\r\n13879\r\n13880\r\n13881\r\n13882\r\n13883\r\n13884\r\n13885\r\n13886\r\n13887\r\n13888\r\n13889\r\n13890\r\n13891\r\n13892\r\n13893\r\n13894\r\n13895\r\n13896\r\n13897\r\n13898\r\n13899\r\n13900\r\n13901\r\n13902\r\n13903\r\n13904\r\n13905\r\n13906\r\n13907\r\n13908\r\n13909\r\n13910\r\n13911\r\n13912\r\n13913\r\n13914\r\n13915\r\n13916\r\n13917\r\n13918\r\n13919\r\n13920\r\n13921\r\n13922\r\n13923\r\n13924\r\n13925\r\n13926\r\n13927\r\n13928\r\n13929\r\n13930\r\n13931\r\n13932\r\n13933\r\n13934\r\n13935\r\n13936\r\n13937\r\n13938\r\n13939\r\n13940\r\n13941\r\n13942\r\n13943\r\n13944\r\n13945\r\n13946\r\n13947\r\n13948\r\n13949\r\n13950\r\n13951\r\n13952\r\n13953\r\n13954\r\n13955\r\n13956\r\n13957\r\n13958\r\n13959\r\n13960\r\n13961\r\n13962\r\n13963\r\n13964\r\n13965\r\n13966\r\n13967\r\n13968\r\n13969\r\n13970\r\n13971\r\n13972\r\n13973\r\n13974\r\n13975\r\n13976\r\n13977\r\n13978\r\n13979\r\n13980\r\n13981\r\n13982\r\n13983\r\n13984\r\n13985\r\n13986\r\n13987\r\n13988\r\n13989\r\n13990\r\n13991\r\n13992\r\n13993\r\n13994\r\n13995\r\n13996\r\n13997\r\n13998\r\n13999\r\n14000\r\n14001\r\n14002\r\n14003\r\n14004\r\n14005\r\n14006\r\n14007\r\n14008\r\n14009\r\n14010\r\n14011\r\n14012\r\n14013\r\n14014\r\n14015\r\n14016\r\n14017\r\n14018\r\n14019\r\n14020\r\n14021\r\n14022\r\n14023\r\n14024\r\n14025\r\n14026\r\n14027\r\n14028\r\n14029\r\n14030\r\n14031\r\n14032\r\n14033\r\n14034\r\n14035\r\n14036\r\n14037\r\n14038\r\n14039\r\n14040\r\n14041\r\n14042\r\n14043\r\n14044\r\n14045\r\n14046\r\n14047\r\n14048\r\n14049\r\n14050\r\n14051\r\n14052\r\n14053\r\n14054\r\n14055\r\n14056\r\n14057\r\n14058\r\n14059\r\n14060\r\n14061\r\n14062\r\n14063\r\n14064\r\n14065\r\n14066\r\n14067\r\n14068\r\n14069\r\n14070\r\n14071\r\n14072\r\n14073\r\n14074\r\n14075\r\n14076\r\n14077\r\n14078\r\n14079\r\n14080\r\n14081\r\n14082\r\n14083\r\n14084\r\n14085\r\n14086\r\n14087\r\n14088\r\n14089\r\n14090\r\n14091\r\n14092\r\n14093\r\n14094\r\n14095\r\n14096\r\n14097\r\n14098\r\n14099\r\n14100\r\n14101\r\n14102\r\n14103\r\n14104\r\n14105\r\n14106\r\n14107\r\n14108\r\n14109\r\n14110\r\n14111\r\n14112\r\n14113\r\n14114\r\n14115\r\n14116\r\n14117\r\n14118\r\n14119\r\n14120\r\n14121\r\n14122\r\n14123\r\n14124\r\n14125\r\n14126\r\n14127\r\n14128\r\n14129\r\n14130\r\n14131\r\n14132\r\n14133\r\n14134\r\n14135\r\n14136\r\n14137\r\n14138\r\n14139\r\n14140\r\n14141\r\n14142\r\n14143\r\n14144\r\n14145\r\n14146\r\n14147\r\n14148\r\n14149\r\n14150\r\n14151\r\n14152\r\n14153\r\n14154\r\n14155\r\n14156\r\n14157\r\n14158\r\n14159\r\n14160\r\n14161\r\n14162\r\n14163\r\n14164\r\n14165\r\n14166\r\n14167\r\n14168\r\n14169\r\n14170\r\n14171\r\n14172\r\n14173\r\n14174\r\n14175\r\n14176\r\n14177\r\n14178\r\n14179\r\n14180\r\n14181\r\n14182\r\n14183\r\n14184\r\n14185\r\n14186\r\n14187\r\n14188\r\n14189\r\n14190\r\n14191\r\n14192\r\n14193\r\n14194\r\n14195\r\n14196\r\n14197\r\n14198\r\n14199\r\n14200\r\n14201\r\n14202\r\n14203\r\n14204\r\n14205\r\n14206\r\n14207\r\n14208\r\n14209\r\n14210\r\n14211\r\n14212\r\n14213\r\n14214\r\n14215\r\n14216\r\n14217\r\n14218\r\n14219\r\n14220\r\n14221\r\n14222\r\n14223\r\n14224\r\n14225\r\n14226\r\n14227\r\n14228\r\n14229\r\n14230\r\n14231\r\n14232\r\n14233\r\n14234\r\n14235\r\n14236\r\n14237\r\n14238\r\n14239\r\n14240\r\n14241\r\n14242\r\n14243\r\n14244\r\n14245\r\n14246\r\n14247\r\n14248\r\n14249\r\n14250\r\n14251\r\n14252\r\n14253\r\n14254\r\n14255\r\n14256\r\n14257\r\n14258\r\n14259\r\n14260\r\n14261\r\n14262\r\n14263\r\n14264\r\n14265\r\n14266\r\n14267\r\n14268\r\n14269\r\n14270\r\n14271\r\n14272\r\n14273\r\n14274\r\n14275\r\n14276\r\n14277\r\n14278\r\n14279\r\n14280\r\n14281\r\n14282\r\n14283\r\n14284\r\n14285\r\n14286\r\n14287\r\n14288\r\n14289\r\n14290\r\n14291\r\n14292\r\n14293\r\n14294\r\n14295\r\n14296\r\n14297\r\n14298\r\n14299\r\n14300\r\n14301\r\n14302\r\n14303\r\n14304\r\n14305\r\n14306\r\n14307\r\n14308\r\n14309\r\n14310\r\n14311\r\n14312\r\n14313\r\n14314\r\n14315\r\n14316\r\n14317\r\n14318\r\n14319\r\n14320\r\n14321\r\n14322\r\n14323\r\n14324\r\n14325\r\n14326\r\n14327\r\n14328\r\n14329\r\n14330\r\n14331\r\n14332\r\n14333\r\n14334\r\n14335\r\n14336\r\n14337\r\n14338\r\n14339\r\n14340\r\n14341\r\n14342\r\n14343\r\n14344\r\n14345\r\n14346\r\n14347\r\n14348\r\n14349\r\n14350\r\n14351\r\n14352\r\n14353\r\n14354\r\n14355\r\n14356\r\n14357\r\n14358\r\n14359\r\n14360\r\n14361\r\n14362\r\n14363\r\n14364\r\n14365\r\n14366\r\n14367\r\n14368\r\n14369\r\n14370\r\n14371\r\n14372\r\n14373\r\n14374\r\n14375\r\n14376\r\n14377\r\n14378\r\n14379\r\n14380\r\n14381\r\n14382\r\n14383\r\n14384\r\n14385\r\n14386\r\n14387\r\n14388\r\n14389\r\n14390\r\n14391\r\n14392\r\n14393\r\n14394\r\n14395\r\n14396\r\n14397\r\n14398\r\n14399\r\n14400\r\n14401\r\n14402\r\n14403\r\n14404\r\n14405\r\n14406\r\n14407\r\n14408\r\n14409\r\n14410\r\n14411\r\n14412\r\n14413\r\n14414\r\n14415\r\n14416\r\n14417\r\n14418\r\n14419\r\n14420\r\n14421\r\n14422\r\n14423\r\n14424\r\n14425\r\n14426\r\n14427\r\n14428\r\n14429\r\n14430\r\n14431\r\n14432\r\n14433\r\n14434\r\n14435\r\n14436\r\n14437\r\n14438\r\n14439\r\n14440\r\n14441\r\n14442\r\n14443\r\n14444\r\n14445\r\n14446\r\n14447\r\n14448\r\n14449\r\n14450\r\n14451\r\n14452\r\n14453\r\n14454\r\n14455\r\n14456\r\n14457\r\n14458\r\n14459\r\n14460\r\n14461\r\n14462\r\n14463\r\n14464\r\n14465\r\n14466\r\n14467\r\n14468\r\n14469\r\n14470\r\n14471\r\n14472\r\n14473\r\n14474\r\n14475\r\n14476\r\n14477\r\n14478\r\n14479\r\n14480\r\n14481\r\n14482\r\n14483\r\n14484\r\n14485\r\n14486\r\n14487\r\n14488\r\n14489\r\n14490\r\n14491\r\n14492\r\n14493\r\n14494\r\n14495\r\n14496\r\n14497\r\n14498\r\n14499\r\n14500\r\n14501\r\n14502\r\n14503\r\n14504\r\n14505\r\n14506\r\n14507\r\n14508\r\n14509\r\n14510\r\n14511\r\n14512\r\n14513\r\n14514\r\n14515\r\n14516\r\n14517\r\n14518\r\n14519\r\n14520\r\n14521\r\n14522\r\n14523\r\n14524\r\n14525\r\n14526\r\n14527\r\n14528\r\n14529\r\n14530\r\n14531\r\n14532\r\n14533\r\n14534\r\n14535\r\n14536\r\n14537\r\n14538\r\n14539\r\n14540\r\n14541\r\n14542\r\n14543\r\n14544\r\n14545\r\n14546\r\n14547\r\n14548\r\n14549\r\n14550\r\n14551\r\n14552\r\n14553\r\n14554\r\n14555\r\n14556\r\n14557\r\n14558\r\n14559\r\n14560\r\n14561\r\n14562\r\n14563\r\n14564\r\n14565\r\n14566\r\n14567\r\n14568\r\n14569\r\n14570\r\n14571\r\n14572\r\n14573\r\n14574\r\n14575\r\n14576\r\n14577\r\n14578\r\n14579\r\n14580\r\n14581\r\n14582\r\n14583\r\n14584\r\n14585\r\n14586\r\n14587\r\n14588\r\n14589\r\n14590\r\n14591\r\n14592\r\n14593\r\n14594\r\n14595\r\n14596\r\n14597\r\n14598\r\n14599\r\n14600\r\n14601\r\n14602\r\n14603\r\n14604\r\n14605\r\n14606\r\n14607\r\n14608\r\n14609\r\n14610\r\n14611\r\n14612\r\n14613\r\n14614\r\n14615\r\n14616\r\n14617\r\n14618\r\n14619\r\n14620\r\n14621\r\n14622\r\n14623\r\n14624\r\n14625\r\n14626\r\n14627\r\n14628\r\n14629\r\n14630\r\n14631\r\n14632\r\n14633\r\n14634\r\n14635\r\n14636\r\n14637\r\n14638\r\n14639\r\n14640\r\n14641\r\n14642\r\n14643\r\n14644\r\n14645\r\n14646\r\n14647\r\n14648\r\n14649\r\n14650\r\n14651\r\n14652\r\n14653\r\n14654\r\n14655\r\n14656\r\n14657\r\n14658\r\n14659\r\n14660\r\n14661\r\n14662\r\n14663\r\n14664\r\n14665\r\n14666\r\n14667\r\n14668\r\n14669\r\n14670\r\n14671\r\n14672\r\n14673\r\n14674\r\n14675\r\n14676\r\n14677\r\n14678\r\n14679\r\n14680\r\n14681\r\n14682\r\n14683\r\n14684\r\n14685\r\n14686\r\n14687\r\n14688\r\n14689\r\n14690\r\n14691\r\n14692\r\n14693\r\n14694\r\n14695\r\n14696\r\n14697\r\n14698\r\n14699\r\n14700\r\n14701\r\n14702\r\n14703\r\n14704\r\n14705\r\n14706\r\n14707\r\n14708\r\n14709\r\n14710\r\n14711\r\n14712\r\n14713\r\n14714\r\n14715\r\n14716\r\n14717\r\n14718\r\n14719\r\n14720\r\n14721\r\n14722\r\n14723\r\n14724\r\n14725\r\n14726\r\n14727\r\n14728\r\n14729\r\n14730\r\n14731\r\n14732\r\n14733\r\n14734\r\n14735\r\n14736\r\n14737\r\n14738\r\n14739\r\n14740\r\n14741\r\n14742\r\n14743\r\n14744\r\n14745\r\n14746\r\n14747\r\n14748\r\n14749\r\n14750\r\n14751\r\n14752\r\n14753\r\n14754\r\n14755\r\n14756\r\n14757\r\n14758\r\n14759\r\n14760\r\n14761\r\n14762\r\n14763\r\n14764\r\n14765\r\n14766\r\n14767\r\n14768\r\n14769\r\n14770\r\n14771\r\n14772\r\n14773\r\n14774\r\n14775\r\n14776\r\n14777\r\n14778\r\n14779\r\n14780\r\n14781\r\n14782\r\n14783\r\n14784\r\n14785\r\n14786\r\n14787\r\n14788\r\n14789\r\n14790\r\n14791\r\n14792\r\n14793\r\n14794\r\n14795\r\n14796\r\n14797\r\n14798\r\n14799\r\n14800\r\n14801\r\n14802\r\n14803\r\n14804\r\n14805\r\n14806\r\n14807\r\n14808\r\n14809\r\n14810\r\n14811\r\n14812\r\n14813\r\n14814\r\n14815\r\n14816\r\n14817\r\n14818\r\n14819\r\n14820\r\n14821\r\n14822\r\n14823\r\n14824\r\n14825\r\n14826\r\n14827\r\n14828\r\n14829\r\n14830\r\n14831\r\n14832\r\n14833\r\n14834\r\n14835\r\n14836\r\n14837\r\n14838\r\n14839\r\n14840\r\n14841\r\n14842\r\n14843\r\n14844\r\n14845\r\n14846\r\n14847\r\n14848\r\n14849\r\n14850\r\n14851\r\n14852\r\n14853\r\n14854\r\n14855\r\n14856\r\n14857\r\n14858\r\n14859\r\n14860\r\n14861\r\n14862\r\n14863\r\n14864\r\n14865\r\n14866\r\n14867\r\n14868\r\n14869\r\n14870\r\n14871\r\n14872\r\n14873\r\n14874\r\n14875\r\n14876\r\n14877\r\n14878\r\n14879\r\n14880\r\n14881\r\n14882\r\n14883\r\n14884\r\n14885\r\n14886\r\n14887\r\n14888\r\n14889\r\n14890\r\n14891\r\n14892\r\n14893\r\n14894\r\n14895\r\n14896\r\n14897\r\n14898\r\n14899\r\n14900\r\n14901\r\n14902\r\n14903\r\n14904\r\n14905\r\n14906\r\n14907\r\n14908\r\n14909\r\n14910\r\n14911\r\n14912\r\n14913\r\n14914\r\n14915\r\n14916\r\n14917\r\n14918\r\n14919\r\n14920\r\n14921\r\n14922\r\n14923\r\n14924\r\n14925\r\n14926\r\n14927\r\n14928\r\n14929\r\n14930\r\n14931\r\n14932\r\n14933\r\n14934\r\n14935\r\n14936\r\n14937\r\n14938\r\n14939\r\n14940\r\n14941\r\n14942\r\n14943\r\n14944\r\n14945\r\n14946\r\n14947\r\n14948\r\n14949\r\n14950\r\n14951\r\n14952\r\n14953\r\n14954\r\n14955\r\n14956\r\n14957\r\n14958\r\n14959\r\n14960\r\n14961\r\n14962\r\n14963\r\n14964\r\n14965\r\n14966\r\n14967\r\n14968\r\n14969\r\n14970\r\n14971\r\n14972\r\n14973\r\n14974\r\n14975\r\n14976\r\n14977\r\n14978\r\n14979\r\n14980\r\n14981\r\n14982\r\n14983\r\n14984\r\n14985\r\n14986\r\n14987\r\n14988\r\n14989\r\n14990\r\n14991\r\n14992\r\n14993\r\n14994\r\n14995\r\n14996\r\n14997\r\n14998\r\n14999\r\n15000\r\n15001\r\n15002\r\n15003\r\n15004\r\n15005\r\n15006\r\n15007\r\n15008\r\n15009\r\n15010\r\n15011\r\n15012\r\n15013\r\n15014\r\n15015\r\n15016\r\n15017\r\n15018\r\n15019\r\n15020\r\n15021\r\n15022\r\n15023\r\n15024\r\n15025\r\n15026\r\n15027\r\n15028\r\n15029\r\n15030\r\n15031\r\n15032\r\n15033\r\n15034\r\n15035\r\n15036\r\n15037\r\n15038\r\n15039\r\n15040\r\n15041\r\n15042\r\n15043\r\n15044\r\n15045\r\n15046\r\n15047\r\n15048\r\n15049\r\n15050\r\n15051\r\n15052\r\n15053\r\n15054\r\n15055\r\n15056\r\n15057\r\n15058\r\n15059\r\n15060\r\n15061\r\n15062\r\n15063\r\n15064\r\n15065\r\n15066\r\n15067\r\n15068\r\n15069\r\n15070\r\n15071\r\n15072\r\n15073\r\n15074\r\n15075\r\n15076\r\n15077\r\n15078\r\n15079\r\n15080\r\n15081\r\n15082\r\n15083\r\n15084\r\n15085\r\n15086\r\n15087\r\n15088\r\n15089\r\n15090\r\n15091\r\n15092\r\n15093\r\n15094\r\n15095\r\n15096\r\n15097\r\n15098\r\n15099\r\n15100\r\n15101\r\n15102\r\n15103\r\n15104\r\n15105\r\n15106\r\n15107\r\n15108\r\n15109\r\n15110\r\n15111\r\n15112\r\n15113\r\n15114\r\n15115\r\n15116\r\n15117\r\n15118\r\n15119\r\n15120\r\n15121\r\n15122\r\n15123\r\n15124\r\n15125\r\n15126\r\n15127\r\n15128\r\n15129\r\n15130\r\n15131\r\n15132\r\n15133\r\n15134\r\n15135\r\n15136\r\n15137\r\n15138\r\n15139\r\n15140\r\n15141\r\n15142\r\n15143\r\n15144\r\n15145\r\n15146\r\n15147\r\n15148\r\n15149\r\n15150\r\n15151\r\n15152\r\n15153\r\n15154\r\n15155\r\n15156\r\n15157\r\n15158\r\n15159\r\n15160\r\n15161\r\n15162\r\n15163\r\n15164\r\n15165\r\n15166\r\n15167\r\n15168\r\n15169\r\n15170\r\n15171\r\n15172\r\n15173\r\n15174\r\n15175\r\n15176\r\n15177\r\n15178\r\n15179\r\n15180\r\n15181\r\n15182\r\n15183\r\n15184\r\n15185\r\n15186\r\n15187\r\n15188\r\n15189\r\n15190\r\n15191\r\n15192\r\n15193\r\n15194\r\n15195\r\n15196\r\n15197\r\n15198\r\n15199\r\n15200\r\n15201\r\n15202\r\n15203\r\n15204\r\n15205\r\n15206\r\n15207\r\n15208\r\n15209\r\n15210\r\n15211\r\n15212\r\n15213\r\n15214\r\n15215\r\n15216\r\n15217\r\n15218\r\n15219\r\n15220\r\n15221\r\n15222\r\n15223\r\n15224\r\n15225\r\n15226\r\n15227\r\n15228\r\n15229\r\n15230\r\n15231\r\n15232\r\n15233\r\n15234\r\n15235\r\n15236\r\n15237\r\n15238\r\n15239\r\n15240\r\n15241\r\n15242\r\n15243\r\n15244\r\n15245\r\n15246\r\n15247\r\n15248\r\n15249\r\n15250\r\n15251\r\n15252\r\n15253\r\n15254\r\n15255\r\n15256\r\n15257\r\n15258\r\n15259\r\n15260\r\n15261\r\n15262\r\n15263\r\n15264\r\n15265\r\n15266\r\n15267\r\n15268\r\n15269\r\n15270\r\n15271\r\n15272\r\n15273\r\n15274\r\n15275\r\n15276\r\n15277\r\n15278\r\n15279\r\n15280\r\n15281\r\n15282\r\n15283\r\n15284\r\n15285\r\n15286\r\n15287\r\n15288\r\n15289\r\n15290\r\n15291\r\n15292\r\n15293\r\n15294\r\n15295\r\n15296\r\n15297\r\n15298\r\n15299\r\n15300\r\n15301\r\n15302\r\n15303\r\n15304\r\n15305\r\n15306\r\n15307\r\n15308\r\n15309\r\n15310\r\n15311\r\n15312\r\n15313\r\n15314\r\n15315\r\n15316\r\n15317\r\n15318\r\n15319\r\n15320\r\n15321\r\n15322\r\n15323\r\n15324\r\n15325\r\n15326\r\n15327\r\n15328\r\n15329\r\n15330\r\n15331\r\n15332\r\n15333\r\n15334\r\n15335\r\n15336\r\n15337\r\n15338\r\n15339\r\n15340\r\n15341\r\n15342\r\n15343\r\n15344\r\n15345\r\n15346\r\n15347\r\n15348\r\n15349\r\n15350\r\n15351\r\n15352\r\n15353\r\n15354\r\n15355\r\n15356\r\n15357\r\n15358\r\n15359\r\n15360\r\n15361\r\n15362\r\n15363\r\n15364\r\n15365\r\n15366\r\n15367\r\n15368\r\n15369\r\n15370\r\n15371\r\n15372\r\n15373\r\n15374\r\n15375\r\n15376\r\n15377\r\n15378\r\n15379\r\n15380\r\n15381\r\n15382\r\n15383\r\n15384\r\n15385\r\n15386\r\n15387\r\n15388\r\n15389\r\n15390\r\n15391\r\n15392\r\n15393\r\n15394\r\n15395\r\n15396\r\n15397\r\n15398\r\n15399\r\n15400\r\n15401\r\n15402\r\n15403\r\n15404\r\n15405\r\n15406\r\n15407\r\n15408\r\n15409\r\n15410\r\n15411\r\n15412\r\n15413\r\n15414\r\n15415\r\n15416\r\n15417\r\n15418\r\n15419\r\n15420\r\n15421\r\n15422\r\n15423\r\n15424\r\n15425\r\n15426\r\n15427\r\n15428\r\n15429\r\n15430\r\n15431\r\n15432\r\n15433\r\n15434\r\n15435\r\n15436\r\n15437\r\n15438\r\n15439\r\n15440\r\n15441\r\n15442\r\n15443\r\n15444\r\n15445\r\n15446\r\n15447\r\n15448\r\n15449\r\n15450\r\n15451\r\n15452\r\n15453\r\n15454\r\n15455\r\n15456\r\n15457\r\n15458\r\n15459\r\n15460\r\n15461\r\n15462\r\n15463\r\n15464\r\n15465\r\n15466\r\n15467\r\n15468\r\n15469\r\n15470\r\n15471\r\n15472\r\n15473\r\n15474\r\n15475\r\n15476\r\n15477\r\n15478\r\n15479\r\n15480\r\n15481\r\n15482\r\n15483\r\n15484\r\n15485\r\n15486\r\n15487\r\n15488\r\n15489\r\n15490\r\n15491\r\n15492\r\n15493\r\n15494\r\n15495\r\n15496\r\n15497\r\n15498\r\n15499\r\n15500\r\n15501\r\n15502\r\n15503\r\n15504\r\n15505\r\n15506\r\n15507\r\n15508\r\n15509\r\n15510\r\n15511\r\n15512\r\n15513\r\n15514\r\n15515\r\n15516\r\n15517\r\n15518\r\n15519\r\n15520\r\n15521\r\n15522\r\n15523\r\n15524\r\n15525\r\n15526\r\n15527\r\n15528\r\n15529\r\n15530\r\n15531\r\n15532\r\n15533\r\n15534\r\n15535\r\n15536\r\n15537\r\n15538\r\n15539\r\n15540\r\n15541\r\n15542\r\n15543\r\n15544\r\n15545\r\n15546\r\n15547\r\n15548\r\n15549\r\n15550\r\n15551\r\n15552\r\n15553\r\n15554\r\n15555\r\n15556\r\n15557\r\n15558\r\n15559\r\n15560\r\n15561\r\n15562\r\n15563\r\n15564\r\n15565\r\n15566\r\n15567\r\n15568\r\n15569\r\n15570\r\n15571\r\n15572\r\n15573\r\n15574\r\n15575\r\n15576\r\n15577\r\n15578\r\n15579\r\n15580\r\n15581\r\n15582\r\n15583\r\n15584\r\n15585\r\n15586\r\n15587\r\n15588\r\n15589\r\n15590\r\n15591\r\n15592\r\n15593\r\n15594\r\n15595\r\n15596\r\n15597\r\n15598\r\n15599\r\n15600\r\n15601\r\n15602\r\n15603\r\n15604\r\n15605\r\n15606\r\n15607\r\n15608\r\n15609\r\n15610\r\n15611\r\n15612\r\n15613\r\n15614\r\n15615\r\n15616\r\n15617\r\n15618\r\n15619\r\n15620\r\n15621\r\n15622\r\n15623\r\n15624\r\n15625\r\n15626\r\n15627\r\n15628\r\n15629\r\n15630\r\n15631\r\n15632\r\n15633\r\n15634\r\n15635\r\n15636\r\n15637\r\n15638\r\n15639\r\n15640\r\n15641\r\n15642\r\n15643\r\n15644\r\n15645\r\n15646\r\n15647\r\n15648\r\n15649\r\n15650\r\n15651\r\n15652\r\n15653\r\n15654\r\n15655\r\n15656\r\n15657\r\n15658\r\n15659\r\n15660\r\n15661\r\n15662\r\n15663\r\n15664\r\n15665\r\n15666\r\n15667\r\n15668\r\n15669\r\n15670\r\n15671\r\n15672\r\n15673\r\n15674\r\n15675\r\n15676\r\n15677\r\n15678\r\n15679\r\n15680\r\n15681\r\n15682\r\n15683\r\n15684\r\n15685\r\n15686\r\n15687\r\n15688\r\n15689\r\n15690\r\n15691\r\n15692\r\n15693\r\n15694\r\n15695\r\n15696\r\n15697\r\n15698\r\n15699\r\n15700\r\n15701\r\n15702\r\n15703\r\n15704\r\n15705\r\n15706\r\n15707\r\n15708\r\n15709\r\n15710\r\n15711\r\n15712\r\n15713\r\n15714\r\n15715\r\n15716\r\n15717\r\n15718\r\n15719\r\n15720\r\n15721\r\n15722\r\n15723\r\n15724\r\n15725\r\n15726\r\n15727\r\n15728\r\n15729\r\n15730\r\n15731\r\n15732\r\n15733\r\n15734\r\n15735\r\n15736\r\n15737\r\n15738\r\n15739\r\n15740\r\n15741\r\n15742\r\n15743\r\n15744\r\n15745\r\n15746\r\n15747\r\n15748\r\n15749\r\n15750\r\n15751\r\n15752\r\n15753\r\n15754\r\n15755\r\n15756\r\n15757\r\n15758\r\n15759\r\n15760\r\n15761\r\n15762\r\n15763\r\n15764\r\n15765\r\n15766\r\n15767\r\n15768\r\n15769\r\n15770\r\n15771\r\n15772\r\n15773\r\n15774\r\n15775\r\n15776\r\n15777\r\n15778\r\n15779\r\n15780\r\n15781\r\n15782\r\n15783\r\n15784\r\n15785\r\n15786\r\n15787\r\n15788\r\n15789\r\n15790\r\n15791\r\n15792\r\n15793\r\n15794\r\n15795\r\n15796\r\n15797\r\n15798\r\n15799\r\n15800\r\n15801\r\n15802\r\n15803\r\n15804\r\n15805\r\n15806\r\n15807\r\n15808\r\n15809\r\n15810\r\n15811\r\n15812\r\n15813\r\n15814\r\n15815\r\n15816\r\n15817\r\n15818\r\n15819\r\n15820\r\n15821\r\n15822\r\n15823\r\n15824\r\n15825\r\n15826\r\n15827\r\n15828\r\n15829\r\n15830\r\n15831\r\n15832\r\n15833\r\n15834\r\n15835\r\n15836\r\n15837\r\n15838\r\n15839\r\n15840\r\n15841\r\n15842\r\n15843\r\n15844\r\n15845\r\n15846\r\n15847\r\n15848\r\n15849\r\n15850\r\n15851\r\n15852\r\n15853\r\n15854\r\n15855\r\n15856\r\n15857\r\n15858\r\n15859\r\n15860\r\n15861\r\n15862\r\n15863\r\n15864\r\n15865\r\n15866\r\n15867\r\n15868\r\n15869\r\n15870\r\n15871\r\n15872\r\n15873\r\n15874\r\n15875\r\n15876\r\n15877\r\n15878\r\n15879\r\n15880\r\n15881\r\n15882\r\n15883\r\n15884\r\n15885\r\n15886\r\n15887\r\n15888\r\n15889\r\n15890\r\n15891\r\n15892\r\n15893\r\n15894\r\n15895\r\n15896\r\n15897\r\n15898\r\n15899\r\n15900\r\n15901\r\n15902\r\n15903\r\n15904\r\n15905\r\n15906\r\n15907\r\n15908\r\n15909\r\n15910\r\n15911\r\n15912\r\n15913\r\n15914\r\n15915\r\n15916\r\n15917\r\n15918\r\n15919\r\n15920\r\n15921\r\n15922\r\n15923\r\n15924\r\n15925\r\n15926\r\n15927\r\n15928\r\n15929\r\n15930\r\n15931\r\n15932\r\n15933\r\n15934\r\n15935\r\n15936\r\n15937\r\n15938\r\n15939\r\n15940\r\n15941\r\n15942\r\n15943\r\n15944\r\n15945\r\n15946\r\n15947\r\n15948\r\n15949\r\n15950\r\n15951\r\n15952\r\n15953\r\n15954\r\n15955\r\n15956\r\n15957\r\n15958\r\n15959\r\n15960\r\n15961\r\n15962\r\n15963\r\n15964\r\n15965\r\n15966\r\n15967\r\n15968\r\n15969\r\n15970\r\n15971\r\n15972\r\n15973\r\n15974\r\n15975\r\n15976\r\n15977\r\n15978\r\n15979\r\n15980\r\n15981\r\n15982\r\n15983\r\n15984\r\n15985\r\n15986\r\n15987\r\n15988\r\n15989\r\n15990\r\n15991\r\n15992\r\n15993\r\n15994\r\n15995\r\n15996\r\n15997\r\n15998\r\n15999\r\n16000\r\n16001\r\n16002\r\n16003\r\n16004\r\n16005\r\n16006\r\n16007\r\n16008\r\n16009\r\n16010\r\n16011\r\n16012\r\n16013\r\n16014\r\n16015\r\n16016\r\n16017\r\n16018\r\n16019\r\n16020\r\n16021\r\n16022\r\n16023\r\n16024\r\n16025\r\n16026\r\n16027\r\n16028\r\n16029\r\n16030\r\n16031\r\n16032\r\n16033\r\n16034\r\n16035\r\n16036\r\n16037\r\n16038\r\n16039\r\n16040\r\n16041\r\n16042\r\n16043\r\n16044\r\n16045\r\n16046\r\n16047\r\n16048\r\n16049\r\n16050\r\n16051\r\n16052\r\n16053\r\n16054\r\n16055\r\n16056\r\n16057\r\n16058\r\n16059\r\n16060\r\n16061\r\n16062\r\n16063\r\n16064\r\n16065\r\n16066\r\n16067\r\n16068\r\n16069\r\n16070\r\n16071\r\n16072\r\n16073\r\n16074\r\n16075\r\n16076\r\n16077\r\n16078\r\n16079\r\n16080\r\n16081\r\n16082\r\n16083\r\n16084\r\n16085\r\n16086\r\n16087\r\n16088\r\n16089\r\n16090\r\n16091\r\n16092\r\n16093\r\n16094\r\n16095\r\n16096\r\n16097\r\n16098\r\n16099\r\n16100\r\n16101\r\n16102\r\n16103\r\n16104\r\n16105\r\n16106\r\n16107\r\n16108\r\n16109\r\n16110\r\n16111\r\n16112\r\n16113\r\n16114\r\n16115\r\n16116\r\n16117\r\n16118\r\n16119\r\n16120\r\n16121\r\n16122\r\n16123\r\n16124\r\n16125\r\n16126\r\n16127\r\n16128\r\n16129\r\n16130\r\n16131\r\n16132\r\n16133\r\n16134\r\n16135\r\n16136\r\n16137\r\n16138\r\n16139\r\n16140\r\n16141\r\n16142\r\n16143\r\n16144\r\n16145\r\n16146\r\n16147\r\n16148\r\n16149\r\n16150\r\n16151\r\n16152\r\n16153\r\n16154\r\n16155\r\n16156\r\n16157\r\n16158\r\n16159\r\n16160\r\n16161\r\n16162\r\n16163\r\n16164\r\n16165\r\n16166\r\n16167\r\n16168\r\n16169\r\n16170\r\n16171\r\n16172\r\n16173\r\n16174\r\n16175\r\n16176\r\n16177\r\n16178\r\n16179\r\n16180\r\n16181\r\n16182\r\n16183\r\n16184\r\n16185\r\n16186\r\n16187\r\n16188\r\n16189\r\n16190\r\n16191\r\n16192\r\n16193\r\n16194\r\n16195\r\n16196\r\n16197\r\n16198\r\n16199\r\n16200\r\n16201\r\n16202\r\n16203\r\n16204\r\n16205\r\n16206\r\n16207\r\n16208\r\n16209\r\n16210\r\n16211\r\n16212\r\n16213\r\n16214\r\n16215\r\n16216\r\n16217\r\n16218\r\n16219\r\n16220\r\n16221\r\n16222\r\n16223\r\n16224\r\n16225\r\n16226\r\n16227\r\n16228\r\n16229\r\n16230\r\n16231\r\n16232\r\n16233\r\n16234\r\n16235\r\n16236\r\n16237\r\n16238\r\n16239\r\n16240\r\n16241\r\n16242\r\n16243\r\n16244\r\n16245\r\n16246\r\n16247\r\n16248\r\n16249\r\n16250\r\n16251\r\n16252\r\n16253\r\n16254\r\n16255\r\n16256\r\n16257\r\n16258\r\n16259\r\n16260\r\n16261\r\n16262\r\n16263\r\n16264\r\n16265\r\n16266\r\n16267\r\n16268\r\n16269\r\n16270\r\n16271\r\n16272\r\n16273\r\n16274\r\n16275\r\n16276\r\n16277\r\n16278\r\n16279\r\n16280\r\n16281\r\n16282\r\n16283\r\n16284\r\n16285\r\n16286\r\n16287\r\n16288\r\n16289\r\n16290\r\n16291\r\n16292\r\n16293\r\n16294\r\n16295\r\n16296\r\n16297\r\n16298\r\n16299\r\n16300\r\n16301\r\n16302\r\n16303\r\n16304\r\n16305\r\n16306\r\n16307\r\n16308\r\n16309\r\n16310\r\n16311\r\n16312\r\n16313\r\n16314\r\n16315\r\n16316\r\n16317\r\n16318\r\n16319\r\n16320\r\n16321\r\n16322\r\n16323\r\n16324\r\n16325\r\n16326\r\n16327\r\n16328\r\n16329\r\n16330\r\n16331\r\n16332\r\n16333\r\n16334\r\n16335\r\n16336\r\n16337\r\n16338\r\n16339\r\n16340\r\n16341\r\n16342\r\n16343\r\n16344\r\n16345\r\n16346\r\n16347\r\n16348\r\n16349\r\n16350\r\n16351\r\n16352\r\n16353\r\n16354\r\n16355\r\n16356\r\n16357\r\n16358\r\n16359\r\n16360\r\n16361\r\n16362\r\n16363\r\n16364\r\n16365\r\n16366\r\n16367\r\n16368\r\n16369\r\n16370\r\n16371\r\n16372\r\n16373\r\n16374\r\n16375\r\n16376\r\n16377\r\n16378\r\n16379\r\n16380\r\n16381\r\n16382\r\n16383\r\n16384\r\n16385\r\n16386\r\n16387\r\n16388\r\n16389\r\n16390\r\n16391\r\n16392\r\n16393\r\n16394\r\n16395\r\n16396\r\n16397\r\n16398\r\n16399\r\n16400\r\n16401\r\n16402\r\n16403\r\n16404\r\n16405\r\n16406\r\n16407\r\n16408\r\n16409\r\n16410\r\n16411\r\n16412\r\n16413\r\n16414\r\n16415\r\n16416\r\n16417\r\n16418\r\n16419\r\n16420\r\n16421\r\n16422\r\n16423\r\n16424\r\n16425\r\n16426\r\n16427\r\n16428\r\n16429\r\n16430\r\n16431\r\n16432\r\n16433\r\n16434\r\n16435\r\n16436\r\n16437\r\n16438\r\n16439\r\n16440\r\n16441\r\n16442\r\n16443\r\n16444\r\n16445\r\n16446\r\n16447\r\n16448\r\n16449\r\n16450\r\n16451\r\n16452\r\n16453\r\n16454\r\n16455\r\n16456\r\n16457\r\n16458\r\n16459\r\n16460\r\n16461\r\n16462\r\n16463\r\n16464\r\n16465\r\n16466\r\n16467\r\n16468\r\n16469\r\n16470\r\n16471\r\n16472\r\n16473\r\n16474\r\n16475\r\n16476\r\n16477\r\n16478\r\n16479\r\n16480\r\n16481\r\n16482\r\n16483\r\n16484\r\n16485\r\n16486\r\n16487\r\n16488\r\n16489\r\n16490\r\n16491\r\n16492\r\n16493\r\n16494\r\n16495\r\n16496\r\n16497\r\n16498\r\n16499\r\n16500\r\n16501\r\n16502\r\n16503\r\n16504\r\n16505\r\n16506\r\n16507\r\n16508\r\n16509\r\n16510\r\n16511\r\n16512\r\n16513\r\n16514\r\n16515\r\n16516\r\n16517\r\n16518\r\n16519\r\n16520\r\n16521\r\n16522\r\n16523\r\n16524\r\n16525\r\n16526\r\n16527\r\n16528\r\n16529\r\n16530\r\n16531\r\n16532\r\n16533\r\n16534\r\n16535\r\n16536\r\n16537\r\n16538\r\n16539\r\n16540\r\n16541\r\n16542\r\n16543\r\n16544\r\n16545\r\n16546\r\n16547\r\n16548\r\n16549\r\n16550\r\n16551\r\n16552\r\n16553\r\n16554\r\n16555\r\n16556\r\n16557\r\n16558\r\n16559\r\n16560\r\n16561\r\n16562\r\n16563\r\n16564\r\n16565\r\n16566\r\n16567\r\n16568\r\n16569\r\n16570\r\n16571\r\n16572\r\n16573\r\n16574\r\n16575\r\n16576\r\n16577\r\n16578\r\n16579\r\n16580\r\n16581\r\n16582\r\n16583\r\n16584\r\n16585\r\n16586\r\n16587\r\n16588\r\n16589\r\n16590\r\n16591\r\n16592\r\n16593\r\n16594\r\n16595\r\n16596\r\n16597\r\n16598\r\n16599\r\n16600\r\n16601\r\n16602\r\n16603\r\n16604\r\n16605\r\n16606\r\n16607\r\n16608\r\n16609\r\n16610\r\n16611\r\n16612\r\n16613\r\n16614\r\n16615\r\n16616\r\n16617\r\n16618\r\n16619\r\n16620\r\n16621\r\n16622\r\n16623\r\n16624\r\n16625\r\n16626\r\n16627\r\n16628\r\n16629\r\n16630\r\n16631\r\n16632\r\n16633\r\n16634\r\n16635\r\n16636\r\n16637\r\n16638\r\n16639\r\n16640\r\n16641\r\n16642\r\n16643\r\n16644\r\n16645\r\n16646\r\n16647\r\n16648\r\n16649\r\n16650\r\n16651\r\n16652\r\n16653\r\n16654\r\n16655\r\n16656\r\n16657\r\n16658\r\n16659\r\n16660\r\n16661\r\n16662\r\n16663\r\n16664\r\n16665\r\n16666\r\n16667\r\n16668\r\n16669\r\n16670\r\n16671\r\n16672\r\n16673\r\n16674\r\n16675\r\n16676\r\n16677\r\n16678\r\n16679\r\n16680\r\n16681\r\n16682\r\n16683\r\n16684\r\n16685\r\n16686\r\n16687\r\n16688\r\n16689\r\n16690\r\n16691\r\n16692\r\n16693\r\n16694\r\n16695\r\n16696\r\n16697\r\n16698\r\n16699\r\n16700\r\n16701\r\n16702\r\n16703\r\n16704\r\n16705\r\n16706\r\n16707\r\n16708\r\n16709\r\n16710\r\n16711\r\n16712\r\n16713\r\n16714\r\n16715\r\n16716\r\n16717\r\n16718\r\n16719\r\n16720\r\n16721\r\n16722\r\n16723\r\n16724\r\n16725\r\n16726\r\n16727\r\n16728\r\n16729\r\n16730\r\n16731\r\n16732\r\n16733\r\n16734\r\n16735\r\n16736\r\n16737\r\n16738\r\n16739\r\n16740\r\n16741\r\n16742\r\n16743\r\n16744\r\n16745\r\n16746\r\n16747\r\n16748\r\n16749\r\n16750\r\n16751\r\n16752\r\n16753\r\n16754\r\n16755\r\n16756\r\n16757\r\n16758\r\n16759\r\n16760\r\n16761\r\n16762\r\n16763\r\n16764\r\n16765\r\n16766\r\n16767\r\n16768\r\n16769\r\n16770\r\n16771\r\n16772\r\n16773\r\n16774\r\n16775\r\n16776\r\n16777\r\n16778\r\n16779\r\n16780\r\n16781\r\n16782\r\n16783\r\n16784\r\n16785\r\n16786\r\n16787\r\n16788\r\n16789\r\n16790\r\n16791\r\n16792\r\n16793\r\n16794\r\n16795\r\n16796\r\n16797\r\n16798\r\n16799\r\n16800\r\n16801\r\n16802\r\n16803\r\n16804\r\n16805\r\n16806\r\n16807\r\n16808\r\n16809\r\n16810\r\n16811\r\n16812\r\n16813\r\n16814\r\n16815\r\n16816\r\n16817\r\n16818\r\n16819\r\n16820\r\n16821\r\n16822\r\n16823\r\n16824\r\n16825\r\n16826\r\n16827\r\n16828\r\n16829\r\n16830\r\n16831\r\n16832\r\n16833\r\n16834\r\n16835\r\n16836\r\n16837\r\n16838\r\n16839\r\n16840\r\n16841\r\n16842\r\n16843\r\n16844\r\n16845\r\n16846\r\n16847\r\n16848\r\n16849\r\n16850\r\n16851\r\n16852\r\n16853\r\n16854\r\n16855\r\n16856\r\n16857\r\n16858\r\n16859\r\n16860\r\n16861\r\n16862\r\n16863\r\n16864\r\n16865\r\n16866\r\n16867\r\n16868\r\n16869\r\n16870\r\n16871\r\n16872\r\n16873\r\n16874\r\n16875\r\n16876\r\n16877\r\n16878\r\n16879\r\n16880\r\n16881\r\n16882\r\n16883\r\n16884\r\n16885\r\n16886\r\n16887\r\n16888\r\n16889\r\n16890\r\n16891\r\n16892\r\n16893\r\n16894\r\n16895\r\n16896\r\n16897\r\n16898\r\n16899\r\n16900\r\n16901\r\n16902\r\n16903\r\n16904\r\n16905\r\n16906\r\n16907\r\n16908\r\n16909\r\n16910\r\n16911\r\n16912\r\n16913\r\n16914\r\n16915\r\n16916\r\n16917\r\n16918\r\n16919\r\n16920\r\n16921\r\n16922\r\n16923\r\n16924\r\n16925\r\n16926\r\n16927\r\n16928\r\n16929\r\n16930\r\n16931\r\n16932\r\n16933\r\n16934\r\n16935\r\n16936\r\n16937\r\n16938\r\n16939\r\n16940\r\n16941\r\n16942\r\n16943\r\n16944\r\n16945\r\n16946\r\n16947\r\n16948\r\n16949\r\n16950\r\n16951\r\n16952\r\n16953\r\n16954\r\n16955\r\n16956\r\n16957\r\n16958\r\n16959\r\n16960\r\n16961\r\n16962\r\n16963\r\n16964\r\n16965\r\n16966\r\n16967\r\n16968\r\n16969\r\n16970\r\n16971\r\n16972\r\n16973\r\n16974\r\n16975\r\n16976\r\n16977\r\n16978\r\n16979\r\n16980\r\n16981\r\n16982\r\n16983\r\n16984\r\n16985\r\n16986\r\n16987\r\n16988\r\n16989\r\n16990\r\n16991\r\n16992\r\n16993\r\n16994\r\n16995\r\n16996\r\n16997\r\n16998\r\n16999\r\n17000\r\n17001\r\n17002\r\n17003\r\n17004\r\n17005\r\n17006\r\n17007\r\n17008\r\n17009\r\n17010\r\n17011\r\n17012\r\n17013\r\n17014\r\n17015\r\n17016\r\n17017\r\n17018\r\n17019\r\n17020\r\n17021\r\n17022\r\n17023\r\n17024\r\n17025\r\n17026\r\n17027\r\n17028\r\n17029\r\n17030\r\n17031\r\n17032\r\n17033\r\n17034\r\n17035\r\n17036\r\n17037\r\n17038\r\n17039\r\n17040\r\n17041\r\n17042\r\n17043\r\n17044\r\n17045\r\n17046\r\n17047\r\n17048\r\n17049\r\n17050\r\n17051\r\n17052\r\n17053\r\n17054\r\n17055\r\n17056\r\n17057\r\n17058\r\n17059\r\n17060\r\n17061\r\n17062\r\n17063\r\n17064\r\n17065\r\n17066\r\n17067\r\n17068\r\n17069\r\n17070\r\n17071\r\n17072\r\n17073\r\n17074\r\n17075\r\n17076\r\n17077\r\n17078\r\n17079\r\n17080\r\n17081\r\n17082\r\n17083\r\n17084\r\n17085\r\n17086\r\n17087\r\n17088\r\n17089\r\n17090\r\n17091\r\n17092\r\n17093\r\n17094\r\n17095\r\n17096\r\n17097\r\n17098\r\n17099\r\n17100\r\n17101\r\n17102\r\n17103\r\n17104\r\n17105\r\n17106\r\n17107\r\n17108\r\n17109\r\n17110\r\n17111\r\n17112\r\n17113\r\n17114\r\n17115\r\n17116\r\n17117\r\n17118\r\n17119\r\n17120\r\n17121\r\n17122\r\n17123\r\n17124\r\n17125\r\n17126\r\n17127\r\n17128\r\n17129\r\n17130\r\n17131\r\n17132\r\n17133\r\n17134\r\n17135\r\n17136\r\n17137\r\n17138\r\n17139\r\n17140\r\n17141\r\n17142\r\n17143\r\n17144\r\n17145\r\n17146\r\n17147\r\n17148\r\n17149\r\n17150\r\n17151\r\n17152\r\n17153\r\n17154\r\n17155\r\n17156\r\n17157\r\n17158\r\n17159\r\n17160\r\n17161\r\n17162\r\n17163\r\n17164\r\n17165\r\n17166\r\n17167\r\n17168\r\n17169\r\n17170\r\n17171\r\n17172\r\n17173\r\n17174\r\n17175\r\n17176\r\n17177\r\n17178\r\n17179\r\n17180\r\n17181\r\n17182\r\n17183\r\n17184\r\n17185\r\n17186\r\n17187\r\n17188\r\n17189\r\n17190\r\n17191\r\n17192\r\n17193\r\n17194\r\n17195\r\n17196\r\n17197\r\n17198\r\n17199\r\n17200\r\n17201\r\n17202\r\n17203\r\n17204\r\n17205\r\n17206\r\n17207\r\n17208\r\n17209\r\n17210\r\n17211\r\n17212\r\n17213\r\n17214\r\n17215\r\n17216\r\n17217\r\n17218\r\n17219\r\n17220\r\n17221\r\n17222\r\n17223\r\n17224\r\n17225\r\n17226\r\n17227\r\n17228\r\n17229\r\n17230\r\n17231\r\n17232\r\n17233\r\n17234\r\n17235\r\n17236\r\n17237\r\n17238\r\n17239\r\n17240\r\n17241\r\n17242\r\n17243\r\n17244\r\n17245\r\n17246\r\n17247\r\n17248\r\n17249\r\n17250\r\n17251\r\n17252\r\n17253\r\n17254\r\n17255\r\n17256\r\n17257\r\n17258\r\n17259\r\n17260\r\n17261\r\n17262\r\n17263\r\n17264\r\n17265\r\n17266\r\n17267\r\n17268\r\n17269\r\n17270\r\n17271\r\n17272\r\n17273\r\n17274\r\n17275\r\n17276\r\n17277\r\n17278\r\n17279\r\n17280\r\n17281\r\n17282\r\n17283\r\n17284\r\n17285\r\n17286\r\n17287\r\n17288\r\n17289\r\n17290\r\n17291\r\n17292\r\n17293\r\n17294\r\n17295\r\n17296\r\n17297\r\n17298\r\n17299\r\n17300\r\n17301\r\n17302\r\n17303\r\n17304\r\n17305\r\n17306\r\n17307\r\n17308\r\n17309\r\n17310\r\n17311\r\n17312\r\n17313\r\n17314\r\n17315\r\n17316\r\n17317\r\n17318\r\n17319\r\n17320\r\n17321\r\n17322\r\n17323\r\n17324\r\n17325\r\n17326\r\n17327\r\n17328\r\n17329\r\n17330\r\n17331\r\n17332\r\n17333\r\n17334\r\n17335\r\n17336\r\n17337\r\n17338\r\n17339\r\n17340\r\n17341\r\n17342\r\n17343\r\n17344\r\n17345\r\n17346\r\n17347\r\n17348\r\n17349\r\n17350\r\n17351\r\n17352\r\n17353\r\n17354\r\n17355\r\n17356\r\n17357\r\n17358\r\n17359\r\n17360\r\n17361\r\n17362\r\n17363\r\n17364\r\n17365\r\n17366\r\n17367\r\n17368\r\n17369\r\n17370\r\n17371\r\n17372\r\n17373\r\n17374\r\n17375\r\n17376\r\n17377\r\n17378\r\n17379\r\n17380\r\n17381\r\n17382\r\n17383\r\n17384\r\n17385\r\n17386\r\n17387\r\n17388\r\n17389\r\n17390\r\n17391\r\n17392\r\n17393\r\n17394\r\n17395\r\n17396\r\n17397\r\n17398\r\n17399\r\n17400\r\n17401\r\n17402\r\n17403\r\n17404\r\n17405\r\n17406\r\n17407\r\n17408\r\n17409\r\n17410\r\n17411\r\n17412\r\n17413\r\n17414\r\n17415\r\n17416\r\n17417\r\n17418\r\n17419\r\n17420\r\n17421\r\n17422\r\n17423\r\n17424\r\n17425\r\n17426\r\n17427\r\n17428\r\n17429\r\n17430\r\n17431\r\n17432\r\n17433\r\n17434\r\n17435\r\n17436\r\n17437\r\n17438\r\n17439\r\n17440\r\n17441\r\n17442\r\n17443\r\n17444\r\n17445\r\n17446\r\n17447\r\n17448\r\n17449\r\n17450\r\n17451\r\n17452\r\n17453\r\n17454\r\n17455\r\n17456\r\n17457\r\n17458\r\n17459\r\n17460\r\n17461\r\n17462\r\n17463\r\n17464\r\n17465\r\n17466\r\n17467\r\n17468\r\n17469\r\n17470\r\n17471\r\n17472\r\n17473\r\n17474\r\n17475\r\n17476\r\n17477\r\n17478\r\n17479\r\n17480\r\n17481\r\n17482\r\n17483\r\n17484\r\n17485\r\n17486\r\n17487\r\n17488\r\n17489\r\n17490\r\n17491\r\n17492\r\n17493\r\n17494\r\n17495\r\n17496\r\n17497\r\n17498\r\n17499\r\n17500\r\n17501\r\n17502\r\n17503\r\n17504\r\n17505\r\n17506\r\n17507\r\n17508\r\n17509\r\n17510\r\n17511\r\n17512\r\n17513\r\n17514\r\n17515\r\n17516\r\n17517\r\n17518\r\n17519\r\n17520\r\n17521\r\n17522\r\n17523\r\n17524\r\n17525\r\n17526\r\n17527\r\n17528\r\n17529\r\n17530\r\n17531\r\n17532\r\n17533\r\n17534\r\n17535\r\n17536\r\n17537\r\n17538\r\n17539\r\n17540\r\n17541\r\n17542\r\n17543\r\n17544\r\n17545\r\n17546\r\n17547\r\n17548\r\n17549\r\n17550\r\n17551\r\n17552\r\n17553\r\n17554\r\n17555\r\n17556\r\n17557\r\n17558\r\n17559\r\n17560\r\n17561\r\n17562\r\n17563\r\n17564\r\n17565\r\n17566\r\n17567\r\n17568\r\n17569\r\n17570\r\n17571\r\n17572\r\n17573\r\n17574\r\n17575\r\n17576\r\n17577\r\n17578\r\n17579\r\n17580\r\n17581\r\n17582\r\n17583\r\n17584\r\n17585\r\n17586\r\n17587\r\n17588\r\n17589\r\n17590\r\n17591\r\n17592\r\n17593\r\n17594\r\n17595\r\n17596\r\n17597\r\n17598\r\n17599\r\n17600\r\n17601\r\n17602\r\n17603\r\n17604\r\n17605\r\n17606\r\n17607\r\n17608\r\n17609\r\n17610\r\n17611\r\n17612\r\n17613\r\n17614\r\n17615\r\n17616\r\n17617\r\n17618\r\n17619\r\n17620\r\n17621\r\n17622\r\n17623\r\n17624\r\n17625\r\n17626\r\n17627\r\n17628\r\n17629\r\n17630\r\n17631\r\n17632\r\n17633\r\n17634\r\n17635\r\n17636\r\n17637\r\n17638\r\n17639\r\n17640\r\n17641\r\n17642\r\n17643\r\n17644\r\n17645\r\n17646\r\n17647\r\n17648\r\n17649\r\n17650\r\n17651\r\n17652\r\n17653\r\n17654\r\n17655\r\n17656\r\n17657\r\n17658\r\n17659\r\n17660\r\n17661\r\n17662\r\n17663\r\n17664\r\n17665\r\n17666\r\n17667\r\n17668\r\n17669\r\n17670\r\n17671\r\n17672\r\n17673\r\n17674\r\n17675\r\n17676\r\n17677\r\n17678\r\n17679\r\n17680\r\n17681\r\n17682\r\n17683\r\n17684\r\n17685\r\n17686\r\n17687\r\n17688\r\n17689\r\n17690\r\n17691\r\n17692\r\n17693\r\n17694\r\n17695\r\n17696\r\n17697\r\n17698\r\n17699\r\n17700\r\n17701\r\n17702\r\n17703\r\n17704\r\n17705\r\n17706\r\n17707\r\n17708\r\n17709\r\n17710\r\n17711\r\n17712\r\n17713\r\n17714\r\n17715\r\n17716\r\n17717\r\n17718\r\n17719\r\n17720\r\n17721\r\n17722\r\n17723\r\n17724\r\n17725\r\n17726\r\n17727\r\n17728\r\n17729\r\n17730\r\n17731\r\n17732\r\n17733\r\n17734\r\n17735\r\n17736\r\n17737\r\n17738\r\n17739\r\n17740\r\n17741\r\n17742\r\n17743\r\n17744\r\n17745\r\n17746\r\n17747\r\n17748\r\n17749\r\n17750\r\n17751\r\n17752\r\n17753\r\n17754\r\n17755\r\n17756\r\n17757\r\n17758\r\n17759\r\n17760\r\n17761\r\n17762\r\n17763\r\n17764\r\n17765\r\n17766\r\n17767\r\n17768\r\n17769\r\n17770\r\n17771\r\n17772\r\n17773\r\n17774\r\n17775\r\n17776\r\n17777\r\n17778\r\n17779\r\n17780\r\n17781\r\n17782\r\n17783\r\n17784\r\n17785\r\n17786\r\n17787\r\n17788\r\n17789\r\n17790\r\n17791\r\n17792\r\n17793\r\n17794\r\n17795\r\n17796\r\n17797\r\n17798\r\n17799\r\n17800\r\n17801\r\n17802\r\n17803\r\n17804\r\n17805\r\n17806\r\n17807\r\n17808\r\n17809\r\n17810\r\n17811\r\n17812\r\n17813\r\n17814\r\n17815\r\n17816\r\n17817\r\n17818\r\n17819\r\n17820\r\n17821\r\n17822\r\n17823\r\n17824\r\n17825\r\n17826\r\n17827\r\n17828\r\n17829\r\n17830\r\n17831\r\n17832\r\n17833\r\n17834\r\n17835\r\n17836\r\n17837\r\n17838\r\n17839\r\n17840\r\n17841\r\n17842\r\n17843\r\n17844\r\n17845\r\n17846\r\n17847\r\n17848\r\n17849\r\n17850\r\n17851\r\n17852\r\n17853\r\n17854\r\n17855\r\n17856\r\n17857\r\n17858\r\n17859\r\n17860\r\n17861\r\n17862\r\n17863\r\n17864\r\n17865\r\n17866\r\n17867\r\n17868\r\n17869\r\n17870\r\n17871\r\n17872\r\n17873\r\n17874\r\n17875\r\n17876\r\n17877\r\n17878\r\n17879\r\n17880\r\n17881\r\n17882\r\n17883\r\n17884\r\n17885\r\n17886\r\n17887\r\n17888\r\n17889\r\n17890\r\n17891\r\n17892\r\n17893\r\n17894\r\n17895\r\n17896\r\n17897\r\n17898\r\n17899\r\n17900\r\n17901\r\n17902\r\n17903\r\n17904\r\n17905\r\n17906\r\n17907\r\n17908\r\n17909\r\n17910\r\n17911\r\n17912\r\n17913\r\n17914\r\n17915\r\n17916\r\n17917\r\n17918\r\n17919\r\n17920\r\n17921\r\n17922\r\n17923\r\n17924\r\n17925\r\n17926\r\n17927\r\n17928\r\n17929\r\n17930\r\n17931\r\n17932\r\n17933\r\n17934\r\n17935\r\n17936\r\n17937\r\n17938\r\n17939\r\n17940\r\n17941\r\n17942\r\n17943\r\n17944\r\n17945\r\n17946\r\n17947\r\n17948\r\n17949\r\n17950\r\n17951\r\n17952\r\n17953\r\n17954\r\n17955\r\n17956\r\n17957\r\n17958\r\n17959\r\n17960\r\n17961\r\n17962\r\n17963\r\n17964\r\n17965\r\n17966\r\n17967\r\n17968\r\n17969\r\n17970\r\n17971\r\n17972\r\n17973\r\n17974\r\n17975\r\n17976\r\n17977\r\n17978\r\n17979\r\n17980\r\n17981\r\n17982\r\n17983\r\n17984\r\n17985\r\n17986\r\n17987\r\n17988\r\n17989\r\n17990\r\n17991\r\n17992\r\n17993\r\n17994\r\n17995\r\n17996\r\n17997\r\n17998\r\n17999\r\n18000\r\n18001\r\n18002\r\n18003\r\n18004\r\n18005\r\n18006\r\n18007\r\n18008\r\n18009\r\n18010\r\n18011\r\n18012\r\n18013\r\n18014\r\n18015\r\n18016\r\n18017\r\n18018\r\n18019\r\n18020\r\n18021\r\n18022\r\n18023\r\n18024\r\n18025\r\n18026\r\n18027\r\n18028\r\n18029\r\n18030\r\n18031\r\n18032\r\n18033\r\n18034\r\n18035\r\n18036\r\n18037\r\n18038\r\n18039\r\n18040\r\n18041\r\n18042\r\n18043\r\n18044\r\n18045\r\n18046\r\n18047\r\n18048\r\n18049\r\n18050\r\n18051\r\n18052\r\n18053\r\n18054\r\n18055\r\n18056\r\n18057\r\n18058\r\n18059\r\n18060\r\n18061\r\n18062\r\n18063\r\n18064\r\n18065\r\n18066\r\n18067\r\n18068\r\n18069\r\n18070\r\n18071\r\n18072\r\n18073\r\n18074\r\n18075\r\n18076\r\n18077\r\n18078\r\n18079\r\n18080\r\n18081\r\n18082\r\n18083\r\n18084\r\n18085\r\n18086\r\n18087\r\n18088\r\n18089\r\n18090\r\n18091\r\n18092\r\n18093\r\n18094\r\n18095\r\n18096\r\n18097\r\n18098\r\n18099\r\n18100\r\n18101\r\n18102\r\n18103\r\n18104\r\n18105\r\n18106\r\n18107\r\n18108\r\n18109\r\n18110\r\n18111\r\n18112\r\n18113\r\n18114\r\n18115\r\n18116\r\n18117\r\n18118\r\n18119\r\n18120\r\n18121\r\n18122\r\n18123\r\n18124\r\n18125\r\n18126\r\n18127\r\n18128\r\n18129\r\n18130\r\n18131\r\n18132\r\n18133\r\n18134\r\n18135\r\n18136\r\n18137\r\n18138\r\n18139\r\n18140\r\n18141\r\n18142\r\n18143\r\n18144\r\n18145\r\n18146\r\n18147\r\n18148\r\n18149\r\n18150\r\n18151\r\n18152\r\n18153\r\n18154\r\n18155\r\n18156\r\n18157\r\n18158\r\n18159\r\n18160\r\n18161\r\n18162\r\n18163\r\n18164\r\n18165\r\n18166\r\n18167\r\n18168\r\n18169\r\n18170\r\n18171\r\n18172\r\n18173\r\n18174\r\n18175\r\n18176\r\n18177\r\n18178\r\n18179\r\n18180\r\n18181\r\n18182\r\n18183\r\n18184\r\n18185\r\n18186\r\n18187\r\n18188\r\n18189\r\n18190\r\n18191\r\n18192\r\n18193\r\n18194\r\n18195\r\n18196\r\n18197\r\n18198\r\n18199\r\n18200\r\n18201\r\n18202\r\n18203\r\n18204\r\n18205\r\n18206\r\n18207\r\n18208\r\n18209\r\n18210\r\n18211\r\n18212\r\n18213\r\n18214\r\n18215\r\n18216\r\n18217\r\n18218\r\n18219\r\n18220\r\n18221\r\n18222\r\n18223\r\n18224\r\n18225\r\n18226\r\n18227\r\n18228\r\n18229\r\n18230\r\n18231\r\n18232\r\n18233\r\n18234\r\n18235\r\n18236\r\n18237\r\n18238\r\n18239\r\n18240\r\n18241\r\n18242\r\n18243\r\n18244\r\n18245\r\n18246\r\n18247\r\n18248\r\n18249\r\n18250\r\n18251\r\n18252\r\n18253\r\n18254\r\n18255\r\n18256\r\n18257\r\n18258\r\n18259\r\n18260\r\n18261\r\n18262\r\n18263\r\n18264\r\n18265\r\n18266\r\n18267\r\n18268\r\n18269\r\n18270\r\n18271\r\n18272\r\n18273\r\n18274\r\n18275\r\n18276\r\n18277\r\n18278\r\n18279\r\n18280\r\n18281\r\n18282\r\n18283\r\n18284\r\n18285\r\n18286\r\n18287\r\n18288\r\n18289\r\n18290\r\n18291\r\n18292\r\n18293\r\n18294\r\n18295\r\n18296\r\n18297\r\n18298\r\n18299\r\n18300\r\n18301\r\n18302\r\n18303\r\n18304\r\n18305\r\n18306\r\n18307\r\n18308\r\n18309\r\n18310\r\n18311\r\n18312\r\n18313\r\n18314\r\n18315\r\n18316\r\n18317\r\n18318\r\n18319\r\n18320\r\n18321\r\n18322\r\n18323\r\n18324\r\n18325\r\n18326\r\n18327\r\n18328\r\n18329\r\n18330\r\n18331\r\n18332\r\n18333\r\n18334\r\n18335\r\n18336\r\n18337\r\n18338\r\n18339\r\n18340\r\n18341\r\n18342\r\n18343\r\n18344\r\n18345\r\n18346\r\n18347\r\n18348\r\n18349\r\n18350\r\n18351\r\n18352\r\n18353\r\n18354\r\n18355\r\n18356\r\n18357\r\n18358\r\n18359\r\n18360\r\n18361\r\n18362\r\n18363\r\n18364\r\n18365\r\n18366\r\n18367\r\n18368\r\n18369\r\n18370\r\n18371\r\n18372\r\n18373\r\n18374\r\n18375\r\n18376\r\n18377\r\n18378\r\n18379\r\n18380\r\n18381\r\n18382\r\n18383\r\n18384\r\n18385\r\n18386\r\n18387\r\n18388\r\n18389\r\n18390\r\n18391\r\n18392\r\n18393\r\n18394\r\n18395\r\n18396\r\n18397\r\n18398\r\n18399\r\n18400\r\n18401\r\n18402\r\n18403\r\n18404\r\n18405\r\n18406\r\n18407\r\n18408\r\n18409\r\n18410\r\n18411\r\n18412\r\n18413\r\n18414\r\n18415\r\n18416\r\n18417\r\n18418\r\n18419\r\n18420\r\n18421\r\n18422\r\n18423\r\n18424\r\n18425\r\n18426\r\n18427\r\n18428\r\n18429\r\n18430\r\n18431\r\n18432\r\n18433\r\n18434\r\n18435\r\n18436\r\n18437\r\n18438\r\n18439\r\n18440\r\n18441\r\n18442\r\n18443\r\n18444\r\n18445\r\n18446\r\n18447\r\n18448\r\n18449\r\n18450\r\n18451\r\n18452\r\n18453\r\n18454\r\n18455\r\n18456\r\n18457\r\n18458\r\n18459\r\n18460\r\n18461\r\n18462\r\n18463\r\n18464\r\n18465\r\n18466\r\n18467\r\n18468\r\n18469\r\n18470\r\n18471\r\n18472\r\n18473\r\n18474\r\n18475\r\n18476\r\n18477\r\n18478\r\n18479\r\n18480\r\n18481\r\n18482\r\n18483\r\n18484\r\n18485\r\n18486\r\n18487\r\n18488\r\n18489\r\n18490\r\n18491\r\n18492\r\n18493\r\n18494\r\n18495\r\n18496\r\n18497\r\n18498\r\n18499\r\n18500\r\n18501\r\n18502\r\n18503\r\n18504\r\n18505\r\n18506\r\n18507\r\n18508\r\n18509\r\n18510\r\n18511\r\n18512\r\n18513\r\n18514\r\n18515\r\n18516\r\n18517\r\n18518\r\n18519\r\n18520\r\n18521\r\n18522\r\n18523\r\n18524\r\n18525\r\n18526\r\n18527\r\n18528\r\n18529\r\n18530\r\n18531\r\n18532\r\n18533\r\n18534\r\n18535\r\n18536\r\n18537\r\n18538\r\n18539\r\n18540\r\n18541\r\n18542\r\n18543\r\n18544\r\n18545\r\n18546\r\n18547\r\n18548\r\n18549\r\n18550\r\n18551\r\n18552\r\n18553\r\n18554\r\n18555\r\n18556\r\n18557\r\n18558\r\n18559\r\n18560\r\n18561\r\n18562\r\n18563\r\n18564\r\n18565\r\n18566\r\n18567\r\n18568\r\n18569\r\n18570\r\n18571\r\n18572\r\n18573\r\n18574\r\n18575\r\n18576\r\n18577\r\n18578\r\n18579\r\n18580\r\n18581\r\n18582\r\n18583\r\n18584\r\n18585\r\n18586\r\n18587\r\n18588\r\n18589\r\n18590\r\n18591\r\n18592\r\n18593\r\n18594\r\n18595\r\n18596\r\n18597\r\n18598\r\n18599\r\n18600\r\n18601\r\n18602\r\n18603\r\n18604\r\n18605\r\n18606\r\n18607\r\n18608\r\n18609\r\n18610\r\n18611\r\n18612\r\n18613\r\n18614\r\n18615\r\n18616\r\n18617\r\n18618\r\n18619\r\n18620\r\n18621\r\n18622\r\n18623\r\n18624\r\n18625\r\n18626\r\n18627\r\n18628\r\n18629\r\n18630\r\n18631\r\n18632\r\n18633\r\n18634\r\n18635\r\n18636\r\n18637\r\n18638\r\n18639\r\n18640\r\n18641\r\n18642\r\n18643\r\n18644\r\n18645\r\n18646\r\n18647\r\n18648\r\n18649\r\n18650\r\n18651\r\n18652\r\n18653\r\n18654\r\n18655\r\n18656\r\n18657\r\n18658\r\n18659\r\n18660\r\n18661\r\n18662\r\n18663\r\n18664\r\n18665\r\n18666\r\n18667\r\n18668\r\n18669\r\n18670\r\n18671\r\n18672\r\n18673\r\n18674\r\n18675\r\n18676\r\n18677\r\n18678\r\n18679\r\n18680\r\n18681\r\n18682\r\n18683\r\n18684\r\n18685\r\n18686\r\n18687\r\n18688\r\n18689\r\n18690\r\n18691\r\n18692\r\n18693\r\n18694\r\n18695\r\n18696\r\n18697\r\n18698\r\n18699\r\n18700\r\n18701\r\n18702\r\n18703\r\n18704\r\n18705\r\n18706\r\n18707\r\n18708\r\n18709\r\n18710\r\n18711\r\n18712\r\n18713\r\n18714\r\n18715\r\n18716\r\n18717\r\n18718\r\n18719\r\n18720\r\n18721\r\n18722\r\n18723\r\n18724\r\n18725\r\n18726\r\n18727\r\n18728\r\n18729\r\n18730\r\n18731\r\n18732\r\n18733\r\n18734\r\n18735\r\n18736\r\n18737\r\n18738\r\n18739\r\n18740\r\n18741\r\n18742\r\n18743\r\n18744\r\n18745\r\n18746\r\n18747\r\n18748\r\n18749\r\n18750\r\n18751\r\n18752\r\n18753\r\n18754\r\n18755\r\n18756\r\n18757\r\n18758\r\n18759\r\n18760\r\n18761\r\n18762\r\n18763\r\n18764\r\n18765\r\n18766\r\n18767\r\n18768\r\n18769\r\n18770\r\n18771\r\n18772\r\n18773\r\n18774\r\n18775\r\n18776\r\n18777\r\n18778\r\n18779\r\n18780\r\n18781\r\n18782\r\n18783\r\n18784\r\n18785\r\n18786\r\n18787\r\n18788\r\n18789\r\n18790\r\n18791\r\n18792\r\n18793\r\n18794\r\n18795\r\n18796\r\n18797\r\n18798\r\n18799\r\n18800\r\n18801\r\n18802\r\n18803\r\n18804\r\n18805\r\n18806\r\n18807\r\n18808\r\n18809\r\n18810\r\n18811\r\n18812\r\n18813\r\n18814\r\n18815\r\n18816\r\n18817\r\n18818\r\n18819\r\n18820\r\n18821\r\n18822\r\n18823\r\n18824\r\n18825\r\n18826\r\n18827\r\n18828\r\n18829\r\n18830\r\n18831\r\n18832\r\n18833\r\n18834\r\n18835\r\n18836\r\n18837\r\n18838\r\n18839\r\n18840\r\n18841\r\n18842\r\n18843\r\n18844\r\n18845\r\n18846\r\n18847\r\n18848\r\n18849\r\n18850\r\n18851\r\n18852\r\n18853\r\n18854\r\n18855\r\n18856\r\n18857\r\n18858\r\n18859\r\n18860\r\n18861\r\n18862\r\n18863\r\n18864\r\n18865\r\n18866\r\n18867\r\n18868\r\n18869\r\n18870\r\n18871\r\n18872\r\n18873\r\n18874\r\n18875\r\n18876\r\n18877\r\n18878\r\n18879\r\n18880\r\n18881\r\n18882\r\n18883\r\n18884\r\n18885\r\n18886\r\n18887\r\n18888\r\n18889\r\n18890\r\n18891\r\n18892\r\n18893\r\n18894\r\n18895\r\n18896\r\n18897\r\n18898\r\n18899\r\n18900\r\n18901\r\n18902\r\n18903\r\n18904\r\n18905\r\n18906\r\n18907\r\n18908\r\n18909\r\n18910\r\n18911\r\n18912\r\n18913\r\n18914\r\n18915\r\n18916\r\n18917\r\n18918\r\n18919\r\n18920\r\n18921\r\n18922\r\n18923\r\n18924\r\n18925\r\n18926\r\n18927\r\n18928\r\n18929\r\n18930\r\n18931\r\n18932\r\n18933\r\n18934\r\n18935\r\n18936\r\n18937\r\n18938\r\n18939\r\n18940\r\n18941\r\n18942\r\n18943\r\n18944\r\n18945\r\n18946\r\n18947\r\n18948\r\n18949\r\n18950\r\n18951\r\n18952\r\n18953\r\n18954\r\n18955\r\n18956\r\n18957\r\n18958\r\n18959\r\n18960\r\n18961\r\n18962\r\n18963\r\n18964\r\n18965\r\n18966\r\n18967\r\n18968\r\n18969\r\n18970\r\n18971\r\n18972\r\n18973\r\n18974\r\n18975\r\n18976\r\n18977\r\n18978\r\n18979\r\n18980\r\n18981\r\n18982\r\n18983\r\n18984\r\n18985\r\n18986\r\n18987\r\n18988\r\n18989\r\n18990\r\n18991\r\n18992\r\n18993\r\n18994\r\n18995\r\n18996\r\n18997\r\n18998\r\n18999\r\n19000\r\n19001\r\n19002\r\n19003\r\n19004\r\n19005\r\n19006\r\n19007\r\n19008\r\n19009\r\n19010\r\n19011\r\n19012\r\n19013\r\n19014\r\n19015\r\n19016\r\n19017\r\n19018\r\n19019\r\n19020\r\n19021\r\n19022\r\n19023\r\n19024\r\n19025\r\n19026\r\n19027\r\n19028\r\n19029\r\n19030\r\n19031\r\n19032\r\n19033\r\n19034\r\n19035\r\n19036\r\n19037\r\n19038\r\n19039\r\n19040\r\n19041\r\n19042\r\n19043\r\n19044\r\n19045\r\n19046\r\n19047\r\n19048\r\n19049\r\n19050\r\n19051\r\n19052\r\n19053\r\n19054\r\n19055\r\n19056\r\n19057\r\n19058\r\n19059\r\n19060\r\n19061\r\n19062\r\n19063\r\n19064\r\n19065\r\n19066\r\n19067\r\n19068\r\n19069\r\n19070\r\n19071\r\n19072\r\n19073\r\n19074\r\n19075\r\n19076\r\n19077\r\n19078\r\n19079\r\n19080\r\n19081\r\n19082\r\n19083\r\n19084\r\n19085\r\n19086\r\n19087\r\n19088\r\n19089\r\n19090\r\n19091\r\n19092\r\n19093\r\n19094\r\n19095\r\n19096\r\n19097\r\n19098\r\n19099\r\n19100\r\n19101\r\n19102\r\n19103\r\n19104\r\n19105\r\n19106\r\n19107\r\n19108\r\n19109\r\n19110\r\n19111\r\n19112\r\n19113\r\n19114\r\n19115\r\n19116\r\n19117\r\n19118\r\n19119\r\n19120\r\n19121\r\n19122\r\n19123\r\n19124\r\n19125\r\n19126\r\n19127\r\n19128\r\n19129\r\n19130\r\n19131\r\n19132\r\n19133\r\n19134\r\n19135\r\n19136\r\n19137\r\n19138\r\n19139\r\n19140\r\n19141\r\n19142\r\n19143\r\n19144\r\n19145\r\n19146\r\n19147\r\n19148\r\n19149\r\n19150\r\n19151\r\n19152\r\n19153\r\n19154\r\n19155\r\n19156\r\n19157\r\n19158\r\n19159\r\n19160\r\n19161\r\n19162\r\n19163\r\n19164\r\n19165\r\n19166\r\n19167\r\n19168\r\n19169\r\n19170\r\n19171\r\n19172\r\n19173\r\n19174\r\n19175\r\n19176\r\n19177\r\n19178\r\n19179\r\n19180\r\n19181\r\n19182\r\n19183\r\n19184\r\n19185\r\n19186\r\n19187\r\n19188\r\n19189\r\n19190\r\n19191\r\n19192\r\n19193\r\n19194\r\n19195\r\n19196\r\n19197\r\n19198\r\n19199\r\n19200\r\n19201\r\n19202\r\n19203\r\n19204\r\n19205\r\n19206\r\n19207\r\n19208\r\n19209\r\n19210\r\n19211\r\n19212\r\n19213\r\n19214\r\n19215\r\n19216\r\n19217\r\n19218\r\n19219\r\n19220\r\n19221\r\n19222\r\n19223\r\n19224\r\n19225\r\n19226\r\n19227\r\n19228\r\n19229\r\n19230\r\n19231\r\n19232\r\n19233\r\n19234\r\n19235\r\n19236\r\n19237\r\n19238\r\n19239\r\n19240\r\n19241\r\n19242\r\n19243\r\n19244\r\n19245\r\n19246\r\n19247\r\n19248\r\n19249\r\n19250\r\n19251\r\n19252\r\n19253\r\n19254\r\n19255\r\n19256\r\n19257\r\n19258\r\n19259\r\n19260\r\n19261\r\n19262\r\n19263\r\n19264\r\n19265\r\n19266\r\n19267\r\n19268\r\n19269\r\n19270\r\n19271\r\n19272\r\n19273\r\n19274\r\n19275\r\n19276\r\n19277\r\n19278\r\n19279\r\n19280\r\n19281\r\n19282\r\n19283\r\n19284\r\n19285\r\n19286\r\n19287\r\n19288\r\n19289\r\n19290\r\n19291\r\n19292\r\n19293\r\n19294\r\n19295\r\n19296\r\n19297\r\n19298\r\n19299\r\n19300\r\n19301\r\n19302\r\n19303\r\n19304\r\n19305\r\n19306\r\n19307\r\n19308\r\n19309\r\n19310\r\n19311\r\n19312\r\n19313\r\n19314\r\n19315\r\n19316\r\n19317\r\n19318\r\n19319\r\n19320\r\n19321\r\n19322\r\n19323\r\n19324\r\n19325\r\n19326\r\n19327\r\n19328\r\n19329\r\n19330\r\n19331\r\n19332\r\n19333\r\n19334\r\n19335\r\n19336\r\n19337\r\n19338\r\n19339\r\n19340\r\n19341\r\n19342\r\n19343\r\n19344\r\n19345\r\n19346\r\n19347\r\n19348\r\n19349\r\n19350\r\n19351\r\n19352\r\n19353\r\n19354\r\n19355\r\n19356\r\n19357\r\n19358\r\n19359\r\n19360\r\n19361\r\n19362\r\n19363\r\n19364\r\n19365\r\n19366\r\n19367\r\n19368\r\n19369\r\n19370\r\n19371\r\n19372\r\n19373\r\n19374\r\n19375\r\n19376\r\n19377\r\n19378\r\n19379\r\n19380\r\n19381\r\n19382\r\n19383\r\n19384\r\n19385\r\n19386\r\n19387\r\n19388\r\n19389\r\n19390\r\n19391\r\n19392\r\n19393\r\n19394\r\n19395\r\n19396\r\n19397\r\n19398\r\n19399\r\n19400\r\n19401\r\n19402\r\n19403\r\n19404\r\n19405\r\n19406\r\n19407\r\n19408\r\n19409\r\n19410\r\n19411\r\n19412\r\n19413\r\n19414\r\n19415\r\n19416\r\n19417\r\n19418\r\n19419\r\n19420\r\n19421\r\n19422\r\n19423\r\n19424\r\n19425\r\n19426\r\n19427\r\n19428\r\n19429\r\n19430\r\n19431\r\n19432\r\n19433\r\n19434\r\n19435\r\n19436\r\n19437\r\n19438\r\n19439\r\n19440\r\n19441\r\n19442\r\n19443\r\n19444\r\n19445\r\n19446\r\n19447\r\n19448\r\n19449\r\n19450\r\n19451\r\n19452\r\n19453\r\n19454\r\n19455\r\n19456\r\n19457\r\n19458\r\n19459\r\n19460\r\n19461\r\n19462\r\n19463\r\n19464\r\n19465\r\n19466\r\n19467\r\n19468\r\n19469\r\n19470\r\n19471\r\n19472\r\n19473\r\n19474\r\n19475\r\n19476\r\n19477\r\n19478\r\n19479\r\n19480\r\n19481\r\n19482\r\n19483\r\n19484\r\n19485\r\n19486\r\n19487\r\n19488\r\n19489\r\n19490\r\n19491\r\n19492\r\n19493\r\n19494\r\n19495\r\n19496\r\n19497\r\n19498\r\n19499\r\n19500\r\n19501\r\n19502\r\n19503\r\n19504\r\n19505\r\n19506\r\n19507\r\n19508\r\n19509\r\n19510\r\n19511\r\n19512\r\n19513\r\n19514\r\n19515\r\n19516\r\n19517\r\n19518\r\n19519\r\n19520\r\n19521\r\n19522\r\n19523\r\n19524\r\n19525\r\n19526\r\n19527\r\n19528\r\n19529\r\n19530\r\n19531\r\n19532\r\n19533\r\n19534\r\n19535\r\n19536\r\n19537\r\n19538\r\n19539\r\n19540\r\n19541\r\n19542\r\n19543\r\n19544\r\n19545\r\n19546\r\n19547\r\n19548\r\n19549\r\n19550\r\n19551\r\n19552\r\n19553\r\n19554\r\n19555\r\n19556\r\n19557\r\n19558\r\n19559\r\n19560\r\n19561\r\n19562\r\n19563\r\n19564\r\n19565\r\n19566\r\n19567\r\n19568\r\n19569\r\n19570\r\n19571\r\n19572\r\n19573\r\n19574\r\n19575\r\n19576\r\n19577\r\n19578\r\n19579\r\n19580\r\n19581\r\n19582\r\n19583\r\n19584\r\n19585\r\n19586\r\n19587\r\n19588\r\n19589\r\n19590\r\n19591\r\n19592\r\n19593\r\n19594\r\n19595\r\n19596\r\n19597\r\n19598\r\n19599\r\n19600\r\n19601\r\n19602\r\n19603\r\n19604\r\n19605\r\n19606\r\n19607\r\n19608\r\n19609\r\n19610\r\n19611\r\n19612\r\n19613\r\n19614\r\n19615\r\n19616\r\n19617\r\n19618\r\n19619\r\n19620\r\n19621\r\n19622\r\n19623\r\n19624\r\n19625\r\n19626\r\n19627\r\n19628\r\n19629\r\n19630\r\n19631\r\n19632\r\n19633\r\n19634\r\n19635\r\n19636\r\n19637\r\n19638\r\n19639\r\n19640\r\n19641\r\n19642\r\n19643\r\n19644\r\n19645\r\n19646\r\n19647\r\n19648\r\n19649\r\n19650\r\n19651\r\n19652\r\n19653\r\n19654\r\n19655\r\n19656\r\n19657\r\n19658\r\n19659\r\n19660\r\n19661\r\n19662\r\n19663\r\n19664\r\n19665\r\n19666\r\n19667\r\n19668\r\n19669\r\n19670\r\n19671\r\n19672\r\n19673\r\n19674\r\n19675\r\n19676\r\n19677\r\n19678\r\n19679\r\n19680\r\n19681\r\n19682\r\n19683\r\n19684\r\n19685\r\n19686\r\n19687\r\n19688\r\n19689\r\n19690\r\n19691\r\n19692\r\n19693\r\n19694\r\n19695\r\n19696\r\n19697\r\n19698\r\n19699\r\n19700\r\n19701\r\n19702\r\n19703\r\n19704\r\n19705\r\n19706\r\n19707\r\n19708\r\n19709\r\n19710\r\n19711\r\n19712\r\n19713\r\n19714\r\n19715\r\n19716\r\n19717\r\n19718\r\n19719\r\n19720\r\n19721\r\n19722\r\n19723\r\n19724\r\n19725\r\n19726\r\n19727\r\n19728\r\n19729\r\n19730\r\n19731\r\n19732\r\n19733\r\n19734\r\n19735\r\n19736\r\n19737\r\n19738\r\n19739\r\n19740\r\n19741\r\n19742\r\n19743\r\n19744\r\n19745\r\n19746\r\n19747\r\n19748\r\n19749\r\n19750\r\n19751\r\n19752\r\n19753\r\n19754\r\n19755\r\n19756\r\n19757\r\n19758\r\n19759\r\n19760\r\n19761\r\n19762\r\n19763\r\n19764\r\n19765\r\n19766\r\n19767\r\n19768\r\n19769\r\n19770\r\n19771\r\n19772\r\n19773\r\n19774\r\n19775\r\n19776\r\n19777\r\n19778\r\n19779\r\n19780\r\n19781\r\n19782\r\n19783\r\n19784\r\n19785\r\n19786\r\n19787\r\n19788\r\n19789\r\n19790\r\n19791\r\n19792\r\n19793\r\n19794\r\n19795\r\n19796\r\n19797\r\n19798\r\n19799\r\n19800\r\n19801\r\n19802\r\n19803\r\n19804\r\n19805\r\n19806\r\n19807\r\n19808\r\n19809\r\n19810\r\n19811\r\n19812\r\n19813\r\n19814\r\n19815\r\n19816\r\n19817\r\n19818\r\n19819\r\n19820\r\n19821\r\n19822\r\n19823\r\n19824\r\n19825\r\n19826\r\n19827\r\n19828\r\n19829\r\n19830\r\n19831\r\n19832\r\n19833\r\n19834\r\n19835\r\n19836\r\n19837\r\n19838\r\n19839\r\n19840\r\n19841\r\n19842\r\n19843\r\n19844\r\n19845\r\n19846\r\n19847\r\n19848\r\n19849\r\n19850\r\n19851\r\n19852\r\n19853\r\n19854\r\n19855\r\n19856\r\n19857\r\n19858\r\n19859\r\n19860\r\n19861\r\n19862\r\n19863\r\n19864\r\n19865\r\n19866\r\n19867\r\n19868\r\n19869\r\n19870\r\n19871\r\n19872\r\n19873\r\n19874\r\n19875\r\n19876\r\n19877\r\n19878\r\n19879\r\n19880\r\n19881\r\n19882\r\n19883\r\n19884\r\n19885\r\n19886\r\n19887\r\n19888\r\n19889\r\n19890\r\n19891\r\n19892\r\n19893\r\n19894\r\n19895\r\n19896\r\n19897\r\n19898\r\n19899\r\n19900\r\n19901\r\n19902\r\n19903\r\n19904\r\n19905\r\n19906\r\n19907\r\n19908\r\n19909\r\n19910\r\n19911\r\n19912\r\n19913\r\n19914\r\n19915\r\n19916\r\n19917\r\n19918\r\n19919\r\n19920\r\n19921\r\n19922\r\n19923\r\n19924\r\n19925\r\n19926\r\n19927\r\n19928\r\n19929\r\n19930\r\n19931\r\n19932\r\n19933\r\n19934\r\n19935\r\n19936\r\n19937\r\n19938\r\n19939\r\n19940\r\n19941\r\n19942\r\n19943\r\n19944\r\n19945\r\n19946\r\n19947\r\n19948\r\n19949\r\n19950\r\n19951\r\n19952\r\n19953\r\n19954\r\n19955\r\n19956\r\n19957\r\n19958\r\n19959\r\n19960\r\n19961\r\n19962\r\n19963\r\n19964\r\n19965\r\n19966\r\n19967\r\n19968\r\n19969\r\n19970\r\n19971\r\n19972\r\n19973\r\n19974\r\n19975\r\n19976\r\n19977\r\n19978\r\n19979\r\n19980\r\n19981\r\n19982\r\n19983\r\n19984\r\n19985\r\n19986\r\n19987\r\n19988\r\n19989\r\n19990\r\n19991\r\n19992\r\n19993\r\n19994\r\n19995\r\n19996\r\n19997\r\n19998\r\n19999\r\n20000\r\n\u001b[?2004hrouter1:~$ "} +{"t": 14.231, "dir": "send", "data": "ls /nonexistent\r", "phase": "command"} +{"t": 14.233, "dir": "recv", "data": "ls /nonexistent\r\n\u001b[?2004l\rls: /nonexistent: No such file or directory\r\n\u001b[?2004hrouter1:~$ "} +{"t": 16.234, "dir": "send", "data": "exit\r", "phase": "command"} +{"t": 16.238, "dir": "recv", "data": "exit\r\n\u001b[?2004l\rlogout\r\n"} +{"t": 16.238, "dir": "close", "data": ""} diff --git a/tests/recordings/linux/alpine-3.22-bash-ssh.golden.json b/tests/recordings/linux/alpine-3.22-bash-ssh.golden.json new file mode 100644 index 00000000..76348ceb --- /dev/null +++ b/tests/recordings/linux/alpine-3.22-bash-ssh.golden.json @@ -0,0 +1,71 @@ +{ + "login": "ok", + "driver": "generic", + "os": "unknown", + "commands": [ + { + "command": "uname -sr", + "error": null, + "lines": 2, + "sha256": "3cbc3fd8aee393a4d91615acfa6f59c14e10b97dd2158f66721ff4ecb75cefb3", + "first": "uname -sr", + "last": "Linux 6.18.54.1-microsoft-standard-WSL2" + }, + { + "command": "cat /etc/os-release", + "error": null, + "lines": 7, + "sha256": "a37b836ba274652b2ca6f2444721a998419ac4fcd1ce2ce69efc935476a39e6d", + "first": "cat /etc/os-release", + "last": "BUG_REPORT_URL=\"https://gitlab.alpinelinux.org/alpine/aports/-/issues\"" + }, + { + "command": "ls -l --color=always /", + "error": null, + "lines": 19, + "sha256": "81b90d20ed47e9049a2a11ab51ee58dc809e41c55e16f75d8a69c9d25dcba10d", + "first": "ls -l --color=always /", + "last": "drwxr-xr-x 11 root root 4096 Oct 21 19:59 \u001b[1;34mvar\u001b[m" + }, + { + "command": "printf 'umlauts: \\303\\244\\303\\266\\303\\274, euro sign: \\342\\202\\254\\n'", + "error": null, + "lines": 3, + "sha256": "7006fb43f9319310a5f7143a1dc4af8045d44b9e68a4c951d846be29c786baa5", + "first": "", + "last": "umlauts: \u00c3\u00a4\u00c3\u00b6\u00c3\u00bc, euro sign: \u00e2\u0082\u00ac" + }, + { + "command": "echo the quick brown fox jumps over the lazy dog, the quick brown fox jumps over the lazy dog, and again", + "error": null, + "lines": 3, + "sha256": "17fced575a130044d3ebc67b1c0836749d0ae0cf757816e5fce5d05324b74167", + "first": "", + "last": "the quick brown fox jumps over the lazy dog, the quick brown fox jumps over the lazy dog, and again" + }, + { + "command": "seq 1 20000", + "error": null, + "lines": 20001, + "sha256": "de64cc695dbacf19a096fef9a7b898241e0984d9a0e54018e1458b798b80cda0", + "first": "seq 1 20000", + "last": "20000" + }, + { + "command": "ls /nonexistent", + "error": null, + "lines": 2, + "sha256": "c44056b106b058656342e300441def2785e91ac86b8e7be94ba989ebe963466b", + "first": "ls /nonexistent", + "last": "ls: /nonexistent: No such file or directory" + }, + { + "command": "exit", + "error": "ProtocolException", + "lines": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "first": "", + "last": "" + } + ] +} diff --git a/tests/recordings/linux/alpine-3.22-bash-ssh.jsonl b/tests/recordings/linux/alpine-3.22-bash-ssh.jsonl new file mode 100644 index 00000000..1e703078 --- /dev/null +++ b/tests/recordings/linux/alpine-3.22-bash-ssh.jsonl @@ -0,0 +1,19 @@ +{"format": "exscript-recording", "version": 1, "protocol": "ssh", "date": "2026-10-31", "sanitized": true, "device": {"os": "shell", "version": "Alpine Linux 3.22", "platform": "docker alpine:3.22, bash 5.2"}, "ssh": {"server_version": "SSH-2.0-OpenSSH_10.0", "banner": null, "auth_methods": ["publickey", "password", "keyboard-interactive"], "interactive": null, "host_key_type": "ssh-ed25519"}, "terminal": {"type": "dumb", "width": 80, "height": 24}} +{"t": 0.087, "dir": "recv", "data": "Welcome to router1, an Exscript test host.\r\nrouter1:~$ "} +{"t": 2.089, "dir": "send", "data": "uname -sr\r", "phase": "command"} +{"t": 2.091, "dir": "recv", "data": "uname -sr\r\nLinux 6.18.54.1-microsoft-standard-WSL2\r\nrouter1:~$ "} +{"t": 4.092, "dir": "send", "data": "cat /etc/os-release\r", "phase": "command"} +{"t": 4.094, "dir": "recv", "data": "cat /etc/os-release\r\nNAME=\"Alpine Linux\"\r\nID=alpine\r\nVERSION_ID=3.22.7\r\nPRETTY_NAME=\"Alpine Linux v3.22\"\r\nHOME_URL=\"https://alpinelinux.org/\"\r\nBUG_REPORT_URL=\"https://gitlab.alpinelinux.org/alpine/aports/-/issues\"\r\nrouter1:~$ "} +{"t": 6.095, "dir": "send", "data": "ls -l --color=always /\r", "phase": "command"} +{"t": 6.097, "dir": "recv", "data": "ls -l --color=always /\r\ntotal 68\r\ndrwxr-xr-x 1 root root 4096 Oct 31 08:37 \u001b[1;34mbin\u001b[m\r\ndrwxr-xr-x 5 root root 340 Oct 31 08:37 \u001b[1;34mdev\u001b[m\r\ndrwxr-xr-x 1 root root 4096 Oct 31 08:38 \u001b[1;34metc\u001b[m\r\ndrwxr-xr-x 1 root root 4096 Oct 31 08:38 \u001b[1;34mhome\u001b[m\r\ndrwxr-xr-x 1 root root 4096 Oct 21 19:59 \u001b[1;34mlib\u001b[m\r\ndrwxr-xr-x 5 root root 4096 Oct 21 19:59 \u001b[1;34mmedia\u001b[m\r\ndrwxr-xr-x 2 root root 4096 Oct 21 19:59 \u001b[1;34mmnt\u001b[m\r\ndrwxr-xr-x 2 root root 4096 Oct 21 19:59 \u001b[1;34mopt\u001b[m\r\ndr-xr-xr-x 398 root root 0 Oct 31 08:37 \u001b[1;34mproc\u001b[m\r\ndrwx------ 2 root root 4096 Oct 21 19:59 \u001b[1;34mroot\u001b[m\r\ndrwxr-xr-x 1 root root 4096 Oct 31 08:38 \u001b[1;34mrun\u001b[m\r\ndrwxr-xr-x 2 root root 4096 Oct 21 19:59 \u001b[1;34msbin\u001b[m\r\ndrwxr-xr-x 2 root root 4096 Oct 21 19:59 \u001b[1;34msrv\u001b[m\r\ndr-xr-xr-x 12 root root 0 Oct 30 12:05 \u001b[1;34msys\u001b[m\r\ndrwxrwxrwt 2 root root 4096 Oct 21 19:59 \u001b[1;34mtmp\u001b[m\r\ndrwxr-xr-x 1 root root 4096 Oct 21 19:59 \u001b[1;34musr\u001b[m\r\ndrwxr-xr-x 11 root root 4096 Oct 21 19:59 \u001b[1;34mvar\u001b[m\r\nrouter1:~$ "} +{"t": 8.098, "dir": "send", "data": "printf 'umlauts: \\303\\244\\303\\266\\303\\274, euro sign: \\342\\202\\254\\n'\r", "phase": "command"} +{"t": 8.1, "dir": "recv", "data": "\r<303\\244\\303\\266\\303\\274, euro sign: \\342\\202\\254\\n'\r\numlauts: \u00c3\u00a4\u00c3\u00b6\u00c3\u00bc, euro sign: \u00e2\u0082\u00ac\r\nrouter1:~$ "} +{"t": 10.101, "dir": "send", "data": "echo the quick brown fox jumps over the lazy dog, the quick brown fox jumps over the lazy dog, and again\r", "phase": "command"} +{"t": 10.105, "dir": "recv", "data": "\r< quick brown fox jumps over the lazy dog, and again\r\nthe quick brown fox jumps over the lazy dog, the quick brown fox jumps over the lazy dog, and again\r\nrouter1:~$ "} +{"t": 12.105, "dir": "send", "data": "seq 1 20000\r", "phase": "command"} +{"t": 12.11, "dir": "recv", "data": "seq 1 20000\r\n1\r\n2\r\n3\r\n4\r\n5\r\n6\r\n7\r\n8\r\n9\r\n10\r\n11\r\n12\r\n13\r\n14\r\n15\r\n16\r\n17\r\n18\r\n19\r\n20\r\n21\r\n22\r\n23\r\n24\r\n25\r\n26\r\n27\r\n28\r\n29\r\n30\r\n31\r\n32\r\n33\r\n34\r\n35\r\n36\r\n37\r\n38\r\n39\r\n40\r\n41\r\n42\r\n43\r\n44\r\n45\r\n46\r\n47\r\n48\r\n49\r\n50\r\n51\r\n52\r\n53\r\n54\r\n55\r\n56\r\n57\r\n58\r\n59\r\n60\r\n61\r\n62\r\n63\r\n64\r\n65\r\n66\r\n67\r\n68\r\n69\r\n70\r\n71\r\n72\r\n73\r\n74\r\n75\r\n76\r\n77\r\n78\r\n79\r\n80\r\n81\r\n82\r\n83\r\n84\r\n85\r\n86\r\n87\r\n88\r\n89\r\n90\r\n91\r\n92\r\n93\r\n94\r\n95\r\n96\r\n97\r\n98\r\n99\r\n100\r\n101\r\n102\r\n103\r\n104\r\n105\r\n106\r\n107\r\n108\r\n109\r\n110\r\n111\r\n112\r\n113\r\n114\r\n115\r\n116\r\n117\r\n118\r\n119\r\n120\r\n121\r\n122\r\n123\r\n124\r\n125\r\n126\r\n127\r\n128\r\n129\r\n130\r\n131\r\n132\r\n133\r\n134\r\n135\r\n136\r\n137\r\n138\r\n139\r\n140\r\n141\r\n142\r\n143\r\n144\r\n145\r\n146\r\n147\r\n148\r\n149\r\n150\r\n151\r\n152\r\n153\r\n154\r\n155\r\n156\r\n157\r\n158\r\n159\r\n160\r\n161\r\n162\r\n163\r\n164\r\n165\r\n166\r\n167\r\n168\r\n169\r\n170\r\n171\r\n172\r\n173\r\n174\r\n175\r\n176\r\n177\r\n178\r\n179\r\n180\r\n181\r\n182\r\n183\r\n184\r\n185\r\n186\r\n187\r\n188\r\n189\r\n190\r\n191\r\n192\r\n193\r\n194\r\n195\r\n196\r\n197\r\n198\r\n199\r\n200\r\n201\r\n202\r\n203\r\n204\r\n205\r\n206\r\n207\r\n208\r\n209\r\n210\r\n211\r\n212\r\n213\r\n214\r\n215\r\n216\r\n217\r\n218\r\n219\r\n220\r\n221\r\n222\r\n223\r\n224\r\n225\r\n226\r\n227\r\n228\r\n229\r\n230\r\n231\r\n232\r\n233\r\n234\r\n235\r\n236\r\n237\r\n238\r\n239\r\n240\r\n241\r\n242\r\n243\r\n244\r\n245\r\n246\r\n247\r\n248\r\n249\r\n250\r\n251\r\n252\r\n253\r\n254\r\n255\r\n256\r\n257\r\n258\r\n259\r\n260\r\n261\r\n262\r\n263\r\n264\r\n265\r\n266\r\n267\r\n268\r\n269\r\n270\r\n271\r\n272\r\n273\r\n274\r\n275\r\n276\r\n277\r\n278\r\n279\r\n280\r\n281\r\n282\r\n283\r\n284\r\n285\r\n286\r\n287\r\n288\r\n289\r\n290\r\n291\r\n292\r\n293\r\n294\r\n295\r\n296\r\n297\r\n298\r\n299\r\n300\r\n301\r\n302\r\n303\r\n304\r\n305\r\n306\r\n307\r\n308\r\n309\r\n310\r\n311\r\n312\r\n313\r\n314\r\n315\r\n316\r\n317\r\n318\r\n319\r\n320\r\n321\r\n322\r\n323\r\n324\r\n325\r\n326\r\n327\r\n328\r\n329\r\n330\r\n331\r\n332\r\n333\r\n334\r\n335\r\n336\r\n337\r\n338\r\n339\r\n340\r\n341\r\n342\r\n343\r\n344\r\n345\r\n346\r\n347\r\n348\r\n349\r\n350\r\n351\r\n352\r\n353\r\n354\r\n355\r\n356\r\n357\r\n358\r\n359\r\n360\r\n361\r\n362\r\n363\r\n364\r\n365\r\n366\r\n367\r\n368\r\n369\r\n370\r\n371\r\n372\r\n373\r\n374\r\n375\r\n376\r\n377\r\n378\r\n379\r\n380\r\n381\r\n382\r\n383\r\n384\r\n385\r\n386\r\n387\r\n388\r\n389\r\n390\r\n391\r\n392\r\n393\r\n394\r\n395\r\n396\r\n397\r\n398\r\n399\r\n400\r\n401\r\n402\r\n403\r\n404\r\n405\r\n406\r\n407\r\n408\r\n409\r\n410\r\n411\r\n412\r\n413\r\n414\r\n415\r\n416\r\n417\r\n418\r\n419\r\n420\r\n421\r\n422\r\n423\r\n424\r\n425\r\n426\r\n427\r\n428\r\n429\r\n430\r\n431\r\n432\r\n433\r\n434\r\n435\r\n436\r\n437\r\n438\r\n439\r\n440\r\n441\r\n442\r\n443\r\n444\r\n445\r\n446\r\n447\r\n448\r\n449\r\n450\r\n451\r\n452\r\n453\r\n454\r\n455\r\n456\r\n457\r\n458\r\n459\r\n460\r\n461\r\n462\r\n463\r\n464\r\n465\r\n466\r\n467\r\n468\r\n469\r\n470\r\n471\r\n472\r\n473\r\n474\r\n475\r\n476\r\n477\r\n478\r\n479\r\n480\r\n481\r\n482\r\n483\r\n484\r\n485\r\n486\r\n487\r\n488\r\n489\r\n490\r\n491\r\n492\r\n493\r\n494\r\n495\r\n496\r\n497\r\n498\r\n499\r\n500\r\n501\r\n502\r\n503\r\n504\r\n505\r\n506\r\n507\r\n508\r\n509\r\n510\r\n511\r\n512\r\n513\r\n514\r\n515\r\n516\r\n517\r\n518\r\n519\r\n520\r\n521\r\n522\r\n523\r\n524\r\n525\r\n526\r\n527\r\n528\r\n529\r\n530\r\n531\r\n532\r\n533\r\n534\r\n535\r\n536\r\n537\r\n538\r\n539\r\n540\r\n541\r\n542\r\n543\r\n544\r\n545\r\n546\r\n547\r\n548\r\n549\r\n550\r\n551\r\n552\r\n553\r\n554\r\n555\r\n556\r\n557\r\n558\r\n559\r\n560\r\n561\r\n562\r\n563\r\n564\r\n565\r\n566\r\n567\r\n568\r\n569\r\n570\r\n571\r\n572\r\n573\r\n574\r\n575\r\n576\r\n577\r\n578\r\n579\r\n580\r\n581\r\n582\r\n583\r\n584\r\n585\r\n586\r\n587\r\n588\r\n589\r\n590\r\n591\r\n592\r\n593\r\n594\r\n595\r\n596\r\n597\r\n598\r\n599\r\n600\r\n601\r\n602\r\n603\r\n604\r\n605\r\n606\r\n607\r\n608\r\n609\r\n610\r\n611\r\n612\r\n613\r\n614\r\n615\r\n616\r\n617\r\n618\r\n619\r\n620\r\n621\r\n622\r\n623\r\n624\r\n625\r\n626\r\n627\r\n628\r\n629\r\n630\r\n631\r\n632\r\n633\r\n634\r\n635\r\n636\r\n637\r\n638\r\n639\r\n640\r\n641\r\n642\r\n643\r\n644\r\n645\r\n646\r\n647\r\n648\r\n649\r\n650\r\n651\r\n652\r\n653\r\n654\r\n655\r\n656\r\n657\r\n658\r\n659\r\n660\r\n661\r\n662\r\n663\r\n664\r\n665\r\n666\r\n667\r\n668\r\n669\r\n670\r\n671\r\n672\r\n673\r\n674\r\n675\r\n676\r\n677\r\n678\r\n679\r\n680\r\n681\r\n682\r\n683\r\n684\r\n685\r\n686\r\n687\r\n688\r\n689\r\n690\r\n691\r\n692\r\n693\r\n694\r\n695\r\n696\r\n697\r\n698\r\n699\r\n700\r\n701\r\n702\r\n703\r\n704\r\n705\r\n706\r\n707\r\n708\r\n709\r\n710\r\n711\r\n712\r\n713\r\n714\r\n715\r\n716\r\n717\r\n718\r\n719\r\n720\r\n721\r\n722\r\n723\r\n724\r\n725\r\n726\r\n727\r\n728\r\n729\r\n730\r\n731\r\n732\r\n733\r\n734\r\n735\r\n736\r\n737\r\n738\r\n739\r\n740\r\n741\r\n742\r\n743\r\n744\r\n745\r\n746\r\n747\r\n748\r\n749\r\n750\r\n751\r\n752\r\n753\r\n754\r\n755\r\n756\r\n757\r\n758\r\n759\r\n760\r\n761\r\n762\r\n763\r\n764\r\n765\r\n766\r\n767\r\n768\r\n769\r\n770\r\n771\r\n772\r\n773\r\n774\r\n775\r\n776\r\n777\r\n778\r\n779\r\n780\r\n781\r\n782\r\n783\r\n784\r\n785\r\n786\r\n787\r\n788\r\n789\r\n790\r\n791\r\n792\r\n793\r\n794\r\n795\r\n796\r\n797\r\n798\r\n799\r\n800\r\n801\r\n802\r\n803\r\n804\r\n805\r\n806\r\n807\r\n808\r\n809\r\n810\r\n811\r\n812\r\n813\r\n814\r\n815\r\n816\r\n817\r\n818\r\n819\r\n820\r\n821\r\n822\r\n823\r\n824\r\n825\r\n826\r\n827\r\n828\r\n829\r\n830\r\n831\r\n832\r\n833\r\n834\r\n835\r\n836\r\n837\r\n838\r\n839\r\n840\r\n841\r\n842\r\n843\r\n844\r\n845\r\n846\r\n847\r\n848\r\n849\r\n850\r\n851\r\n852\r\n853\r\n854\r\n855\r\n856\r\n857\r\n858\r\n859\r\n860\r\n861\r\n862\r\n863\r\n864\r\n865\r\n866\r\n867\r\n868\r\n869\r\n870\r\n871\r\n872\r\n873\r\n874\r\n875\r\n876\r\n877\r\n878\r\n879\r\n880\r\n881\r\n882\r\n883\r\n884\r\n885\r\n886\r\n887\r\n888\r\n889\r\n890\r\n891\r\n892\r\n893\r\n894\r\n895\r\n896\r\n897\r\n898\r\n899\r\n900\r\n901\r\n902\r\n903\r\n904\r\n905\r\n906\r\n907\r\n908\r\n909\r\n910\r\n911\r\n912\r\n913\r\n914\r\n915\r\n916\r\n917\r\n918\r\n919\r\n920\r\n921\r\n922\r\n923\r\n924\r\n925\r\n926\r\n927\r\n928\r\n929\r\n930\r\n931\r\n932\r\n933\r\n934\r\n935\r\n936\r\n937\r\n938\r\n939\r\n940\r\n941\r\n942\r\n943\r\n944\r\n945\r\n946\r\n947\r\n948\r\n949\r\n950\r\n951\r\n952\r\n953\r\n954\r\n955\r\n956\r\n957\r\n958\r\n959\r\n960\r\n961\r\n962\r\n963\r\n964\r\n965\r\n966\r\n967\r\n968\r\n969\r\n970\r\n971\r\n972\r\n973\r\n974\r\n975\r\n976\r\n977\r\n978\r\n979\r\n980\r\n981\r\n982\r\n983\r\n984\r\n985\r\n986\r\n987\r\n988\r\n989\r\n990\r\n991\r\n992\r\n993\r\n994\r\n995\r\n996\r\n997\r\n998\r\n999\r\n1000\r\n1001\r\n1002\r\n1003\r\n1004\r\n1005\r\n1006\r\n1007\r\n1008\r\n1009\r\n1010\r\n1011\r\n1012\r\n1013\r\n1014\r\n1015\r\n1016\r\n1017\r\n1018\r\n1019\r\n1020\r\n1021\r\n1022\r\n1023\r\n1024\r\n1025\r\n1026\r\n1027\r\n1028\r\n1029\r\n1030\r\n1031\r\n1032\r\n1033\r\n1034\r\n1035\r\n1036\r\n1037\r\n1038\r\n1039\r\n1040\r\n1041\r\n1042\r\n1043\r\n1044\r\n1045\r\n1046\r\n1047\r\n1048\r\n1049\r\n1050\r\n1051\r\n1052\r\n1053\r\n1054\r\n1055\r\n1056\r\n1057\r\n1058\r\n1059\r\n1060\r\n1061\r\n1062\r\n1063\r\n1064\r\n1065\r\n1066\r\n1067\r\n1068\r\n1069\r\n1070\r\n1071\r\n1072\r\n1073\r\n1074\r\n1075\r\n1076\r\n1077\r\n1078\r\n1079\r\n1080\r\n1081\r\n1082\r\n1083\r\n1084\r\n1085\r\n1086\r\n1087\r\n1088\r\n1089\r\n1090\r\n1091\r\n1092\r\n1093\r\n1094\r\n1095\r\n1096\r\n1097\r\n1098\r\n1099\r\n1100\r\n1101\r\n1102\r\n1103\r\n1104\r\n1105\r\n1106\r\n1107\r\n1108\r\n1109\r\n1110\r\n1111\r\n1112\r\n1113\r\n1114\r\n1115\r\n1116\r\n1117\r\n1118\r\n1119\r\n1120\r\n1121\r\n1122\r\n1123\r\n1124\r\n1125\r\n1126\r\n1127\r\n1128\r\n1129\r\n1130\r\n1131\r\n1132\r\n1133\r\n1134\r\n1135\r\n1136\r\n1137\r\n1138\r\n1139\r\n1140\r\n1141\r\n1142\r\n1143\r\n1144\r\n1145\r\n1146\r\n1147\r\n1148\r\n1149\r\n1150\r\n1151\r\n1152\r\n1153\r\n1154\r\n1155\r\n1156\r\n1157\r\n1158\r\n1159\r\n1160\r\n1161\r\n1162\r\n1163\r\n1164\r\n1165\r\n1166\r\n1167\r\n1168\r\n1169\r\n1170\r\n1171\r\n1172\r\n1173\r\n1174\r\n1175\r\n1176\r\n1177\r\n1178\r\n1179\r\n1180\r\n1181\r\n1182\r\n1183\r\n1184\r\n1185\r\n1186\r\n1187\r\n1188\r\n1189\r\n1190\r\n1191\r\n1192\r\n1193\r\n1194\r\n1195\r\n1196\r\n1197\r\n1198\r\n1199\r\n1200\r\n1201\r\n1202\r\n1203\r\n1204\r\n1205\r\n1206\r\n1207\r\n1208\r\n1209\r\n1210\r\n1211\r\n1212\r\n1213\r\n1214\r\n1215\r\n1216\r\n1217\r\n1218\r\n1219\r\n1220\r\n1221\r\n1222\r\n1223\r\n1224\r\n1225\r\n1226\r\n1227\r\n1228\r\n1229\r\n1230\r\n1231\r\n1232\r\n1233\r\n1234\r\n1235\r\n1236\r\n1237\r\n1238\r\n1239\r\n1240\r\n1241\r\n1242\r\n1243\r\n1244\r\n1245\r\n1246\r\n1247\r\n1248\r\n1249\r\n1250\r\n1251\r\n1252\r\n1253\r\n1254\r\n1255\r\n1256\r\n1257\r\n1258\r\n1259\r\n1260\r\n1261\r\n1262\r\n1263\r\n1264\r\n1265\r\n1266\r\n1267\r\n1268\r\n1269\r\n1270\r\n1271\r\n1272\r\n1273\r\n1274\r\n1275\r\n1276\r\n1277\r\n1278\r\n1279\r\n1280\r\n1281\r\n1282\r\n1283\r\n1284\r\n1285\r\n1286\r\n1287\r\n1288\r\n1289\r\n1290\r\n1291\r\n1292\r\n1293\r\n1294\r\n1295\r\n1296\r\n1297\r\n1298\r\n1299\r\n1300\r\n1301\r\n1302\r\n1303\r\n1304\r\n1305\r\n1306\r\n1307\r\n1308\r\n1309\r\n1310\r\n1311\r\n1312\r\n1313\r\n1314\r\n1315\r\n1316\r\n1317\r\n1318\r\n1319\r\n1320\r\n1321\r\n1322\r\n1323\r\n1324\r\n1325\r\n1326\r\n1327\r\n1328\r\n1329\r\n1330\r\n1331\r\n1332\r\n1333\r\n1334\r\n1335\r\n1336\r\n1337\r\n1338\r\n1339\r\n1340\r\n1341\r\n1342\r\n1343\r\n1344\r\n1345\r\n1346\r\n1347\r\n1348\r\n1349\r\n1350\r\n1351\r\n1352\r\n1353\r\n1354\r\n1355\r\n1356\r\n1357\r\n1358\r\n1359\r\n1360\r\n1361\r\n1362\r\n1363\r\n1364\r\n1365\r\n1366\r\n1367\r\n1368\r\n1369\r\n1370\r\n1371\r\n1372\r\n1373\r\n1374\r\n1375\r\n1376\r\n1377\r\n1378\r\n1379\r\n1380\r\n1381\r\n1382\r\n1383\r\n1384\r\n1385\r\n1386\r\n1387\r\n1388\r\n1389\r\n1390\r\n1391\r\n1392\r\n1393\r\n1394\r\n1395\r\n1396\r\n1397\r\n1398\r\n1399\r\n1400\r\n1401\r\n1402\r\n1403\r\n1404\r\n1405\r\n1406\r\n1407\r\n1408\r\n1409\r\n1410\r\n1411\r\n1412\r\n1413\r\n1414\r\n1415\r\n1416\r\n1417\r\n1418\r\n1419\r\n1420\r\n1421\r\n1422\r\n1423\r\n1424\r\n1425\r\n1426\r\n1427\r\n1428\r\n1429\r\n1430\r\n1431\r\n1432\r\n1433\r\n1434\r\n1435\r\n1436\r\n1437\r\n1438\r\n1439\r\n1440\r\n1441\r\n1442\r\n1443\r\n1444\r\n1445\r\n1446\r\n1447\r\n1448\r\n1449\r\n1450\r\n1451\r\n1452\r\n1453\r\n1454\r\n1455\r\n1456\r\n1457\r\n1458\r\n1459\r\n1460\r\n1461\r\n1462\r\n1463\r\n1464\r\n1465\r\n1466\r\n1467\r\n1468\r\n1469\r\n1470\r\n1471\r\n1472\r\n1473\r\n1474\r\n1475\r\n1476\r\n1477\r\n1478\r\n1479\r\n1480\r\n1481\r\n1482\r\n1483\r\n1484\r\n1485\r\n1486\r\n1487\r\n1488\r\n1489\r\n1490\r\n1491\r\n1492\r\n1493\r\n1494\r\n1495\r\n1496\r\n1497\r\n1498\r\n1499\r\n1500\r\n1501\r\n1502\r\n1503\r\n1504\r\n1505\r\n1506\r\n1507\r\n1508\r\n1509\r\n1510\r\n1511\r\n1512\r\n1513\r\n1514\r\n1515\r\n1516\r\n1517\r\n1518\r\n1519\r\n1520\r\n1521\r\n1522\r\n1523\r\n1524\r\n1525\r\n1526\r\n1527\r\n1528\r\n1529\r\n1530\r\n1531\r\n1532\r\n1533\r\n1534\r\n1535\r\n1536\r\n1537\r\n1538\r\n1539\r\n1540\r\n1541\r\n1542\r\n1543\r\n1544\r\n1545\r\n1546\r\n1547\r\n1548\r\n1549\r\n1550\r\n1551\r\n1552\r\n1553\r\n1554\r\n1555\r\n1556\r\n1557\r\n1558\r\n1559\r\n1560\r\n1561\r\n1562\r\n1563\r\n1564\r\n1565\r\n1566\r\n1567\r\n1568\r\n1569\r\n1570\r\n1571\r\n1572\r\n1573\r\n1574\r\n1575\r\n1576\r\n1577\r\n1578\r\n1579\r\n1580\r\n1581\r\n1582\r\n1583\r\n1584\r\n1585\r\n1586\r\n1587\r\n1588\r\n1589\r\n1590\r\n1591\r\n1592\r\n1593\r\n1594\r\n1595\r\n1596\r\n1597\r\n1598\r\n1599\r\n1600\r\n1601\r\n1602\r\n1603\r\n1604\r\n1605\r\n1606\r\n1607\r\n1608\r\n1609\r\n1610\r\n1611\r\n1612\r\n1613\r\n1614\r\n1615\r\n1616\r\n1617\r\n1618\r\n1619\r\n1620\r\n1621\r\n1622\r\n1623\r\n1624\r\n1625\r\n1626\r\n1627\r\n1628\r\n1629\r\n1630\r\n1631\r\n1632\r\n1633\r\n1634\r\n1635\r\n1636\r\n1637\r\n1638\r\n1639\r\n1640\r\n1641\r\n1642\r\n1643\r\n1644\r\n1645\r\n1646\r\n1647\r\n1648\r\n1649\r\n1650\r\n1651\r\n1652\r\n1653\r\n1654\r\n1655\r\n1656\r\n1657\r\n1658\r\n1659\r\n1660\r\n1661\r\n1662\r\n1663\r\n1664\r\n1665\r\n1666\r\n1667\r\n1668\r\n1669\r\n1670\r\n1671\r\n1672\r\n1673\r\n1674\r\n1675\r\n1676\r\n1677\r\n1678\r\n1679\r\n1680\r\n1681\r\n1682\r\n1683\r\n1684\r\n1685\r\n1686\r\n1687\r\n1688\r\n1689\r\n1690\r\n1691\r\n1692\r\n1693\r\n1694\r\n1695\r\n1696\r\n1697\r\n1698\r\n1699\r\n1700\r\n1701\r\n1702\r\n1703\r\n1704\r\n1705\r\n1706\r\n1707\r\n1708\r\n1709\r\n1710\r\n1711\r\n1712\r\n1713\r\n1714\r\n1715\r\n1716\r\n1717\r\n1718\r\n1719\r\n1720\r\n1721\r\n1722\r\n1723\r\n1724\r\n1725\r\n1726\r\n1727\r\n1728\r\n1729\r\n1730\r\n1731\r\n1732\r\n1733\r\n1734\r\n1735\r\n1736\r\n1737\r\n1738\r\n1739\r\n1740\r\n1741\r\n1742\r\n1743\r\n1744\r\n1745\r\n1746\r\n1747\r\n1748\r\n1749\r\n1750\r\n1751\r\n1752\r\n1753\r\n1754\r\n1755\r\n1756\r\n1757\r\n1758\r\n1759\r\n1760\r\n1761\r\n1762\r\n1763\r\n1764\r\n1765\r\n1766\r\n1767\r\n1768\r\n1769\r\n1770\r\n1771\r\n1772\r\n1773\r\n1774\r\n1775\r\n1776\r\n1777\r\n1778\r\n1779\r\n1780\r\n1781\r\n1782\r\n1783\r\n1784\r\n1785\r\n1786\r\n1787\r\n1788\r\n1789\r\n1790\r\n1791\r\n1792\r\n1793\r\n1794\r\n1795\r\n1796\r\n1797\r\n1798\r\n1799\r\n1800\r\n1801\r\n1802\r\n1803\r\n1804\r\n1805\r\n1806\r\n1807\r\n1808\r\n1809\r\n1810\r\n1811\r\n1812\r\n1813\r\n1814\r\n1815\r\n1816\r\n1817\r\n1818\r\n1819\r\n1820\r\n1821\r\n1822\r\n1823\r\n1824\r\n1825\r\n1826\r\n1827\r\n1828\r\n1829\r\n1830\r\n1831\r\n1832\r\n1833\r\n1834\r\n1835\r\n1836\r\n1837\r\n1838\r\n1839\r\n1840\r\n1841\r\n1842\r\n1843\r\n1844\r\n1845\r\n1846\r\n1847\r\n1848\r\n1849\r\n1850\r\n1851\r\n1852\r\n1853\r\n1854\r\n1855\r\n1856\r\n1857\r\n1858\r\n1859\r\n1860\r\n1861\r\n1862\r\n1863\r\n1864\r\n1865\r\n1866\r\n1867\r\n1868\r\n1869\r\n1870\r\n1871\r\n1872\r\n1873\r\n1874\r\n1875\r\n1876\r\n1877\r\n1878\r\n1879\r\n1880\r\n1881\r\n1882\r\n1883\r\n1884\r\n1885\r\n1886\r\n1887\r\n1888\r\n1889\r\n1890\r\n1891\r\n1892\r\n1893\r\n1894\r\n1895\r\n1896\r\n1897\r\n1898\r\n1899\r\n1900\r\n1901\r\n1902\r\n1903\r\n1904\r\n1905\r\n1906\r\n1907\r\n1908\r\n1909\r\n1910\r\n1911\r\n1912\r\n1913\r\n1914\r\n1915\r\n1916\r\n1917\r\n1918\r\n1919\r\n1920\r\n1921\r\n1922\r\n1923\r\n1924\r\n1925\r\n1926\r\n1927\r\n1928\r\n1929\r\n1930\r\n1931\r\n1932\r\n1933\r\n1934\r\n1935\r\n1936\r\n1937\r\n1938\r\n1939\r\n1940\r\n1941\r\n1942\r\n1943\r\n1944\r\n1945\r\n1946\r\n1947\r\n1948\r\n1949\r\n1950\r\n1951\r\n1952\r\n1953\r\n1954\r\n1955\r\n1956\r\n1957\r\n1958\r\n1959\r\n1960\r\n1961\r\n1962\r\n1963\r\n1964\r\n1965\r\n1966\r\n1967\r\n1968\r\n1969\r\n1970\r\n1971\r\n1972\r\n1973\r\n1974\r\n1975\r\n1976\r\n1977\r\n1978\r\n1979\r\n1980\r\n1981\r\n1982\r\n1983\r\n1984\r\n1985\r\n1986\r\n1987\r\n1988\r\n1989\r\n1990\r\n1991\r\n1992\r\n1993\r\n1994\r\n1995\r\n1996\r\n1997\r\n1998\r\n1999\r\n2000\r\n2001\r\n2002\r\n2003\r\n2004\r\n2005\r\n2006\r\n2007\r\n2008\r\n2009\r\n2010\r\n2011\r\n2012\r\n2013\r\n2014\r\n2015\r\n2016\r\n2017\r\n2018\r\n2019\r\n2020\r\n2021\r\n2022\r\n2023\r\n2024\r\n2025\r\n2026\r\n2027\r\n2028\r\n2029\r\n2030\r\n2031\r\n2032\r\n2033\r\n2034\r\n2035\r\n2036\r\n2037\r\n2038\r\n2039\r\n2040\r\n2041\r\n2042\r\n2043\r\n2044\r\n2045\r\n2046\r\n2047\r\n2048\r\n2049\r\n2050\r\n2051\r\n2052\r\n2053\r\n2054\r\n2055\r\n2056\r\n2057\r\n2058\r\n2059\r\n2060\r\n2061\r\n2062\r\n2063\r\n2064\r\n2065\r\n2066\r\n2067\r\n2068\r\n2069\r\n2070\r\n2071\r\n2072\r\n2073\r\n2074\r\n2075\r\n2076\r\n2077\r\n2078\r\n2079\r\n2080\r\n2081\r\n2082\r\n2083\r\n2084\r\n2085\r\n2086\r\n2087\r\n2088\r\n2089\r\n2090\r\n2091\r\n2092\r\n2093\r\n2094\r\n2095\r\n2096\r\n2097\r\n2098\r\n2099\r\n2100\r\n2101\r\n2102\r\n2103\r\n2104\r\n2105\r\n2106\r\n2107\r\n2108\r\n2109\r\n2110\r\n2111\r\n2112\r\n2113\r\n2114\r\n2115\r\n2116\r\n2117\r\n2118\r\n2119\r\n2120\r\n2121\r\n2122\r\n2123\r\n2124\r\n2125\r\n2126\r\n2127\r\n2128\r\n2129\r\n2130\r\n2131\r\n2132\r\n2133\r\n2134\r\n2135\r\n2136\r\n2137\r\n2138\r\n2139\r\n2140\r\n2141\r\n2142\r\n2143\r\n2144\r\n2145\r\n2146\r\n2147\r\n2148\r\n2149\r\n2150\r\n2151\r\n2152\r\n2153\r\n2154\r\n2155\r\n2156\r\n2157\r\n2158\r\n2159\r\n2160\r\n2161\r\n2162\r\n2163\r\n2164\r\n2165\r\n2166\r\n2167\r\n2168\r\n2169\r\n2170\r\n2171\r\n2172\r\n2173\r\n2174\r\n2175\r\n2176\r\n2177\r\n2178\r\n2179\r\n2180\r\n2181\r\n2182\r\n2183\r\n2184\r\n2185\r\n2186\r\n2187\r\n2188\r\n2189\r\n2190\r\n2191\r\n2192\r\n2193\r\n2194\r\n2195\r\n2196\r\n2197\r\n2198\r\n2199\r\n2200\r\n2201\r\n2202\r\n2203\r\n2204\r\n2205\r\n2206\r\n2207\r\n2208\r\n2209\r\n2210\r\n2211\r\n2212\r\n2213\r\n2214\r\n2215\r\n2216\r\n2217\r\n2218\r\n2219\r\n2220\r\n2221\r\n2222\r\n2223\r\n2224\r\n2225\r\n2226\r\n2227\r\n2228\r\n2229\r\n2230\r\n2231\r\n2232\r\n2233\r\n2234\r\n2235\r\n2236\r\n2237\r\n2238\r\n2239\r\n2240\r\n2241\r\n2242\r\n2243\r\n2244\r\n2245\r\n2246\r\n2247\r\n2248\r\n2249\r\n2250\r\n2251\r\n2252\r\n2253\r\n2254\r\n2255\r\n2256\r\n2257\r\n2258\r\n2259\r\n2260\r\n2261\r\n2262\r\n2263\r\n2264\r\n2265\r\n2266\r\n2267\r\n2268\r\n2269\r\n2270\r\n2271\r\n2272\r\n2273\r\n2274\r\n2275\r\n2276\r\n2277\r\n2278\r\n2279\r\n2280\r\n2281\r\n2282\r\n2283\r\n2284\r\n2285\r\n2286\r\n2287\r\n2288\r\n2289\r\n2290\r\n2291\r\n2292\r\n2293\r\n2294\r\n2295\r\n2296\r\n2297\r\n2298\r\n2299\r\n2300\r\n2301\r\n2302\r\n2303\r\n2304\r\n2305\r\n2306\r\n2307\r\n2308\r\n2309\r\n2310\r\n2311\r\n2312\r\n2313\r\n2314\r\n2315\r\n2316\r\n2317\r\n2318\r\n2319\r\n2320\r\n2321\r\n2322\r\n2323\r\n2324\r\n2325\r\n2326\r\n2327\r\n2328\r\n2329\r\n2330\r\n2331\r\n2332\r\n2333\r\n2334\r\n2335\r\n2336\r\n2337\r\n2338\r\n2339\r\n2340\r\n2341\r\n2342\r\n2343\r\n2344\r\n2345\r\n2346\r\n2347\r\n2348\r\n2349\r\n2350\r\n2351\r\n2352\r\n2353\r\n2354\r\n2355\r\n2356\r\n2357\r\n2358\r\n2359\r\n2360\r\n2361\r\n2362\r\n2363\r\n2364\r\n2365\r\n2366\r\n2367\r\n2368\r\n2369\r\n2370\r\n2371\r\n2372\r\n2373\r\n2374\r\n2375\r\n2376\r\n2377\r\n2378\r\n2379\r\n2380\r\n2381\r\n2382\r\n2383\r\n2384\r\n2385\r\n2386\r\n2387\r\n2388\r\n2389\r\n2390\r\n2391\r\n2392\r\n2393\r\n2394\r\n2395\r\n2396\r\n2397\r\n2398\r\n2399\r\n2400\r\n2401\r\n2402\r\n2403\r\n2404\r\n2405\r\n2406\r\n2407\r\n2408\r\n2409\r\n2410\r\n2411\r\n2412\r\n2413\r\n2414\r\n2415\r\n2416\r\n2417\r\n2418\r\n2419\r\n2420\r\n2421\r\n2422\r\n2423\r\n2424\r\n2425\r\n2426\r\n2427\r\n2428\r\n2429\r\n2430\r\n2431\r\n2432\r\n2433\r\n2434\r\n2435\r\n2436\r\n2437\r\n2438\r\n2439\r\n2440\r\n2441\r\n2442\r\n2443\r\n2444\r\n2445\r\n2446\r\n2447\r\n2448\r\n2449\r\n2450\r\n2451\r\n2452\r\n2453\r\n2454\r\n2455\r\n2456\r\n2457\r\n2458\r\n2459\r\n2460\r\n2461\r\n2462\r\n2463\r\n2464\r\n2465\r\n2466\r\n2467\r\n2468\r\n2469\r\n2470\r\n2471\r\n2472\r\n2473\r\n2474\r\n2475\r\n2476\r\n2477\r\n2478\r\n2479\r\n2480\r\n2481\r\n2482\r\n2483\r\n2484\r\n2485\r\n2486\r\n2487\r\n2488\r\n2489\r\n2490\r\n2491\r\n2492\r\n2493\r\n2494\r\n2495\r\n2496\r\n2497\r\n2498\r\n2499\r\n2500\r\n2501\r\n2502\r\n2503\r\n2504\r\n2505\r\n2506\r\n2507\r\n2508\r\n2509\r\n2510\r\n2511\r\n2512\r\n2513\r\n2514\r\n2515\r\n2516\r\n2517\r\n2518\r\n2519\r\n2520\r\n2521\r\n2522\r\n2523\r\n2524\r\n2525\r\n2526\r\n2527\r\n2528\r\n2529\r\n2530\r\n2531\r\n2532\r\n2533\r\n2534\r\n2535\r\n2536\r\n2537\r\n2538\r\n2539\r\n2540\r\n2541\r\n2542\r\n2543\r\n2544\r\n2545\r\n2546\r\n2547\r\n2548\r\n2549\r\n2550\r\n2551\r\n2552\r\n2553\r\n2554\r\n2555\r\n2556\r\n2557\r\n2558\r\n2559\r\n2560\r\n2561\r\n2562\r\n2563\r\n2564\r\n2565\r\n2566\r\n2567\r\n2568\r\n2569\r\n2570\r\n2571\r\n2572\r\n2573\r\n2574\r\n2575\r\n2576\r\n2577\r\n2578\r\n2579\r\n2580\r\n2581\r\n2582\r\n2583\r\n2584\r\n2585\r\n2586\r\n2587\r\n2588\r\n2589\r\n2590\r\n2591\r\n2592\r\n2593\r\n2594\r\n2595\r\n2596\r\n2597\r\n2598\r\n2599\r\n2600\r\n2601\r\n2602\r\n2603\r\n2604\r\n2605\r\n2606\r\n2607\r\n2608\r\n2609\r\n2610\r\n2611\r\n2612\r\n2613\r\n2614\r\n2615\r\n2616\r\n2617\r\n2618\r\n2619\r\n2620\r\n2621\r\n2622\r\n2623\r\n2624\r\n2625\r\n2626\r\n2627\r\n2628\r\n2629\r\n2630\r\n2631\r\n2632\r\n2633\r\n2634\r\n2635\r\n2636\r\n2637\r\n2638\r\n2639\r\n2640\r\n2641\r\n2642\r\n2643\r\n2644\r\n2645\r\n2646\r\n2647\r\n2648\r\n2649\r\n2650\r\n2651\r\n2652\r\n2653\r\n2654\r\n2655\r\n2656\r\n2657\r\n2658\r\n2659\r\n2660\r\n2661\r\n2662\r\n2663\r\n2664\r\n2665\r\n2666\r\n2667\r\n2668\r\n2669\r\n2670\r\n2671\r\n2672\r\n2673\r\n2674\r\n2675\r\n2676\r\n2677\r\n2678\r\n2679\r\n2680\r\n2681\r\n2682\r\n2683\r\n2684\r\n2685\r\n2686\r\n2687\r\n2688\r\n2689\r\n2690\r\n2691\r\n2692\r\n2693\r\n2694\r\n2695\r\n2696\r\n2697\r\n2698\r\n2699\r\n2700\r\n2701\r\n2702\r\n2703\r\n2704\r\n2705\r\n2706\r\n2707\r\n2708\r\n2709\r\n2710\r\n2711\r\n2712\r\n2713\r\n2714\r\n2715\r\n2716\r\n2717\r\n2718\r\n2719\r\n2720\r\n2721\r\n2722\r\n2723\r\n2724\r\n2725\r\n2726\r\n2727\r\n2728\r\n2729\r\n2730\r\n2731\r\n2732\r\n2733\r\n2734\r\n2735\r\n2736\r\n2737\r\n2738\r\n2739\r\n2740\r\n2741\r\n2742\r\n2743\r\n2744\r\n2745\r\n2746\r\n2747\r\n2748\r\n2749\r\n2750\r\n2751\r\n2752\r\n2753\r\n2754\r\n2755\r\n2756\r\n2757\r\n2758\r\n2759\r\n2760\r\n2761\r\n2762\r\n2763\r\n2764\r\n2765\r\n2766\r\n2767\r\n2768\r\n2769\r\n2770\r\n2771\r\n2772\r\n2773\r\n2774\r\n2775\r\n2776\r\n2777\r\n2778\r\n2779\r\n2780\r\n2781\r\n2782\r\n2783\r\n2784\r\n2785\r\n2786\r\n2787\r\n2788\r\n2789\r\n2790\r\n2791\r\n2792\r\n2793\r\n2794\r\n2795\r\n2796\r\n2797\r\n2798\r\n2799\r\n2800\r\n2801\r\n2802\r\n2803\r\n2804\r\n2805\r\n2806\r\n2807\r\n2808\r\n2809\r\n2810\r\n2811\r\n2812\r\n2813\r\n2814\r\n2815\r\n2816\r\n2817\r\n2818\r\n2819\r\n2820\r\n2821\r\n2822\r\n2823\r\n2824\r\n2825\r\n2826\r\n2827\r\n2828\r\n2829\r\n2830\r\n2831\r\n2832\r\n2833\r\n2834\r\n2835\r\n2836\r\n2837\r\n2838\r\n2839\r\n2840\r\n2841\r\n2842\r\n2843\r\n2844\r\n2845\r\n2846\r\n2847\r\n2848\r\n2849\r\n2850\r\n2851\r\n2852\r\n2853\r\n2854\r\n2855\r\n2856\r\n2857\r\n2858\r\n2859\r\n2860\r\n2861\r\n2862\r\n2863\r\n2864\r\n2865\r\n2866\r\n2867\r\n2868\r\n2869\r\n2870\r\n2871\r\n2872\r\n2873\r\n2874\r\n2875\r\n2876\r\n2877\r\n2878\r\n2879\r\n2880\r\n2881\r\n2882\r\n2883\r\n2884\r\n2885\r\n2886\r\n2887\r\n2888\r\n2889\r\n2890\r\n2891\r\n2892\r\n2893\r\n2894\r\n2895\r\n2896\r\n2897\r\n2898\r\n2899\r\n2900\r\n2901\r\n2902\r\n2903\r\n2904\r\n2905\r\n2906\r\n2907\r\n2908\r\n2909\r\n2910\r\n2911\r\n2912\r\n2913\r\n2914\r\n2915\r\n2916\r\n2917\r\n2918\r\n2919\r\n2920\r\n2921\r\n2922\r\n2923\r\n2924\r\n2925\r\n2926\r\n2927\r\n2928\r\n2929\r\n2930\r\n2931\r\n2932\r\n2933\r\n2934\r\n2935\r\n2936\r\n2937\r\n2938\r\n2939\r\n2940\r\n2941\r\n2942\r\n2943\r\n2944\r\n2945\r\n2946\r\n2947\r\n2948\r\n2949\r\n2950\r\n2951\r\n2952\r\n2953\r\n2954\r\n2955\r\n2956\r\n2957\r\n2958\r\n2959\r\n2960\r\n2961\r\n2962\r\n2963\r\n2964\r\n2965\r\n2966\r\n2967\r\n2968\r\n2969\r\n2970\r\n2971\r\n2972\r\n2973\r\n2974\r\n2975\r\n2976\r\n2977\r\n2978\r\n2979\r\n2980\r\n2981\r\n2982\r\n2983\r\n2984\r\n2985\r\n2986\r\n2987\r\n2988\r\n2989\r\n2990\r\n2991\r\n2992\r\n2993\r\n2994\r\n2995\r\n2996\r\n2997\r\n2998\r\n2999\r\n3000\r\n3001\r\n3002\r\n3003\r\n3004\r\n3005\r\n3006\r\n3007\r\n3008\r\n3009\r\n3010\r\n3011\r\n3012\r\n3013\r\n3014\r\n3015\r\n3016\r\n3017\r\n3018\r\n3019\r\n3020\r\n3021\r\n3022\r\n3023\r\n3024\r\n3025\r\n3026\r\n3027\r\n3028\r\n3029\r\n3030\r\n3031\r\n3032\r\n3033\r\n3034\r\n3035\r\n3036\r\n3037\r\n3038\r\n3039\r\n3040\r\n3041\r\n3042\r\n3043\r\n3044\r\n3045\r\n3046\r\n3047\r\n3048\r\n3049\r\n3050\r\n3051\r\n3052\r\n3053\r\n3054\r\n3055\r\n3056\r\n3057\r\n3058\r\n3059\r\n3060\r\n3061\r\n3062\r\n3063\r\n3064\r\n3065\r\n3066\r\n3067\r\n3068\r\n3069\r\n3070\r\n3071\r\n3072\r\n3073\r\n3074\r\n3075\r\n3076\r\n3077\r\n3078\r\n3079\r\n3080\r\n3081\r\n3082\r\n3083\r\n3084\r\n3085\r\n3086\r\n3087\r\n3088\r\n3089\r\n3090\r\n3091\r\n3092\r\n3093\r\n3094\r\n3095\r\n3096\r\n3097\r\n3098\r\n3099\r\n3100\r\n3101\r\n3102\r\n3103\r\n3104\r\n3105\r\n3106\r\n3107\r\n3108\r\n3109\r\n3110\r\n3111\r\n3112\r\n3113\r\n3114\r\n3115\r\n3116\r\n3117\r\n3118\r\n3119\r\n3120\r\n3121\r\n3122\r\n3123\r\n3124\r\n3125\r\n3126\r\n3127\r\n3128\r\n3129\r\n3130\r\n3131\r\n3132\r\n3133\r\n3134\r\n3135\r\n3136\r\n3137\r\n3138\r\n3139\r\n3140\r\n3141\r\n3142\r\n3143\r\n3144\r\n3145\r\n3146\r\n3147\r\n3148\r\n3149\r\n3150\r\n3151\r\n3152\r\n3153\r\n3154\r\n3155\r\n3156\r\n3157\r\n3158\r\n3159\r\n3160\r\n3161\r\n3162\r\n3163\r\n3164\r\n3165\r\n3166\r\n3167\r\n3168\r\n3169\r\n3170\r\n3171\r\n3172\r\n3173\r\n3174\r\n3175\r\n3176\r\n3177\r\n3178\r\n3179\r\n3180\r\n3181\r\n3182\r\n3183\r\n3184\r\n3185\r\n3186\r\n3187\r\n3188\r\n3189\r\n3190\r\n3191\r\n3192\r\n3193\r\n3194\r\n3195\r\n3196\r\n3197\r\n3198\r\n3199\r\n3200\r\n3201\r\n3202\r\n3203\r\n3204\r\n3205\r\n3206\r\n3207\r\n3208\r\n3209\r\n3210\r\n3211\r\n3212\r\n3213\r\n3214\r\n3215\r\n3216\r\n3217\r\n3218\r\n3219\r\n3220\r\n3221\r\n3222\r\n3223\r\n3224\r\n3225\r\n3226\r\n3227\r\n3228\r\n3229\r\n3230\r\n3231\r\n3232\r\n3233\r\n3234\r\n3235\r\n3236\r\n3237\r\n3238\r\n3239\r\n3240\r\n3241\r\n3242\r\n3243\r\n3244\r\n3245\r\n3246\r\n3247\r\n3248\r\n3249\r\n3250\r\n3251\r\n3252\r\n3253\r\n3254\r\n3255\r\n3256\r\n3257\r\n3258\r\n3259\r\n3260\r\n3261\r\n3262\r\n3263\r\n3264\r\n3265\r\n3266\r\n3267\r\n3268\r\n3269\r\n3270\r\n3271\r\n3272\r\n3273\r\n3274\r\n3275\r\n3276\r\n3277\r\n3278\r\n3279\r\n3280\r\n3281\r\n3282\r\n3283\r\n3284\r\n3285\r\n3286\r\n3287\r\n3288\r\n3289\r\n3290\r\n3291\r\n3292\r\n3293\r\n3294\r\n3295\r\n3296\r\n3297\r\n3298\r\n3299\r\n3300\r\n3301\r\n3302\r\n3303\r\n3304\r\n3305\r\n3306\r\n3307\r\n3308\r\n3309\r\n3310\r\n3311\r\n3312\r\n3313\r\n3314\r\n3315\r\n3316\r\n3317\r\n3318\r\n3319\r\n3320\r\n3321\r\n3322\r\n3323\r\n3324\r\n3325\r\n3326\r\n3327\r\n3328\r\n3329\r\n3330\r\n3331\r\n3332\r\n3333\r\n3334\r\n3335\r\n3336\r\n3337\r\n3338\r\n3339\r\n3340\r\n3341\r\n3342\r\n3343\r\n3344\r\n3345\r\n3346\r\n3347\r\n3348\r\n3349\r\n3350\r\n3351\r\n3352\r\n3353\r\n3354\r\n3355\r\n3356\r\n3357\r\n3358\r\n3359\r\n3360\r\n3361\r\n3362\r\n3363\r\n3364\r\n3365\r\n3366\r\n3367\r\n3368\r\n3369\r\n3370\r\n3371\r\n3372\r\n3373\r\n3374\r\n3375\r\n3376\r\n3377\r\n3378\r\n3379\r\n3380\r\n3381\r\n3382\r\n3383\r\n3384\r\n3385\r\n3386\r\n3387\r\n3388\r\n3389\r\n3390\r\n3391\r\n3392\r\n3393\r\n3394\r\n3395\r\n3396\r\n3397\r\n3398\r\n3399\r\n3400\r\n3401\r\n3402\r\n3403\r\n3404\r\n3405\r\n3406\r\n3407\r\n3408\r\n3409\r\n3410\r\n3411\r\n3412\r\n3413\r\n3414\r\n3415\r\n3416\r\n3417\r\n3418\r\n3419\r\n3420\r\n3421\r\n3422\r\n3423\r\n3424\r\n3425\r\n3426\r\n3427\r\n3428\r\n3429\r\n3430\r\n3431\r\n3432\r\n3433\r\n3434\r\n3435\r\n3436\r\n3437\r\n3438\r\n3439\r\n3440\r\n3441\r\n3442\r\n3443\r\n3444\r\n3445\r\n3446\r\n3447\r\n3448\r\n3449\r\n3450\r\n3451\r\n3452\r\n3453\r\n3454\r\n3455\r\n3456\r\n3457\r\n3458\r\n3459\r\n3460\r\n3461\r\n3462\r\n3463\r\n3464\r\n3465\r\n3466\r\n3467\r\n3468\r\n3469\r\n3470\r\n3471\r\n3472\r\n3473\r\n3474\r\n3475\r\n3476\r\n3477\r\n3478\r\n3479\r\n3480\r\n3481\r\n3482\r\n3483\r\n3484\r\n3485\r\n3486\r\n3487\r\n3488\r\n3489\r\n3490\r\n3491\r\n3492\r\n3493\r\n3494\r\n3495\r\n3496\r\n3497\r\n3498\r\n3499\r\n3500\r\n3501\r\n3502\r\n3503\r\n3504\r\n3505\r\n3506\r\n3507\r\n3508\r\n3509\r\n3510\r\n3511\r\n3512\r\n3513\r\n3514\r\n3515\r\n3516\r\n3517\r\n3518\r\n3519\r\n3520\r\n3521\r\n3522\r\n3523\r\n3524\r\n3525\r\n3526\r\n3527\r\n3528\r\n3529\r\n3530\r\n3531\r\n3532\r\n3533\r\n3534\r\n3535\r\n3536\r\n3537\r\n3538\r\n3539\r\n3540\r\n3541\r\n3542\r\n3543\r\n3544\r\n3545\r\n3546\r\n3547\r\n3548\r\n3549\r\n3550\r\n3551\r\n3552\r\n3553\r\n3554\r\n3555\r\n3556\r\n3557\r\n3558\r\n3559\r\n3560\r\n3561\r\n3562\r\n3563\r\n3564\r\n3565\r\n3566\r\n3567\r\n3568\r\n3569\r\n3570\r\n3571\r\n3572\r\n3573\r\n3574\r\n3575\r\n3576\r\n3577\r\n3578\r\n3579\r\n3580\r\n3581\r\n3582\r\n3583\r\n3584\r\n3585\r\n3586\r\n3587\r\n3588\r\n3589\r\n3590\r\n3591\r\n3592\r\n3593\r\n3594\r\n3595\r\n3596\r\n3597\r\n3598\r\n3599\r\n3600\r\n3601\r\n3602\r\n3603\r\n3604\r\n3605\r\n3606\r\n3607\r\n3608\r\n3609\r\n3610\r\n3611\r\n3612\r\n3613\r\n3614\r\n3615\r\n3616\r\n3617\r\n3618\r\n3619\r\n3620\r\n3621\r\n3622\r\n3623\r\n3624\r\n3625\r\n3626\r\n3627\r\n3628\r\n3629\r\n3630\r\n3631\r\n3632\r\n3633\r\n3634\r\n3635\r\n3636\r\n3637\r\n3638\r\n3639\r\n3640\r\n3641\r\n3642\r\n3643\r\n3644\r\n3645\r\n3646\r\n3647\r\n3648\r\n3649\r\n3650\r\n3651\r\n3652\r\n3653\r\n3654\r\n3655\r\n3656\r\n3657\r\n3658\r\n3659\r\n3660\r\n3661\r\n3662\r\n3663\r\n3664\r\n3665\r\n3666\r\n3667\r\n3668\r\n3669\r\n3670\r\n3671\r\n3672\r\n3673\r\n3674\r\n3675\r\n3676\r\n3677\r\n3678\r\n3679\r\n3680\r\n3681\r\n3682\r\n3683\r\n3684\r\n3685\r\n3686\r\n3687\r\n3688\r\n3689\r\n3690\r\n3691\r\n3692\r\n3693\r\n3694\r\n3695\r\n3696\r\n3697\r\n3698\r\n3699\r\n3700\r\n3701\r\n3702\r\n3703\r\n3704\r\n3705\r\n3706\r\n3707\r\n3708\r\n3709\r\n3710\r\n3711\r\n3712\r\n3713\r\n3714\r\n3715\r\n3716\r\n3717\r\n3718\r\n3719\r\n3720\r\n3721\r\n3722\r\n3723\r\n3724\r\n3725\r\n3726\r\n3727\r\n3728\r\n3729\r\n3730\r\n3731\r\n3732\r\n3733\r\n3734\r\n3735\r\n3736\r\n3737\r\n3738\r\n3739\r\n3740\r\n3741\r\n3742\r\n3743\r\n3744\r\n3745\r\n3746\r\n3747\r\n3748\r\n3749\r\n3750\r\n3751\r\n3752\r\n3753\r\n3754\r\n3755\r\n3756\r\n3757\r\n3758\r\n3759\r\n3760\r\n3761\r\n3762\r\n3763\r\n3764\r\n3765\r\n3766\r\n3767\r\n3768\r\n3769\r\n3770\r\n3771\r\n3772\r\n3773\r\n3774\r\n3775\r\n3776\r\n3777\r\n3778\r\n3779\r\n3780\r\n3781\r\n3782\r\n3783\r\n3784\r\n3785\r\n3786\r\n3787\r\n3788\r\n3789\r\n3790\r\n3791\r\n3792\r\n3793\r\n3794\r\n3795\r\n3796\r\n3797\r\n3798\r\n3799\r\n3800\r\n3801\r\n3802\r\n3803\r\n3804\r\n3805\r\n3806\r\n3807\r\n3808\r\n3809\r\n3810\r\n3811\r\n3812\r\n3813\r\n3814\r\n3815\r\n3816\r\n3817\r\n3818\r\n3819\r\n3820\r\n3821\r\n3822\r\n3823\r\n3824\r\n3825\r\n3826\r\n3827\r\n3828\r\n3829\r\n3830\r\n3831\r\n3832\r\n3833\r\n3834\r\n3835\r\n3836\r\n3837\r\n3838\r\n3839\r\n3840\r\n3841\r\n3842\r\n3843\r\n3844\r\n3845\r\n3846\r\n3847\r\n3848\r\n3849\r\n3850\r\n3851\r\n3852\r\n3853\r\n3854\r\n3855\r\n3856\r\n3857\r\n3858\r\n3859\r\n3860\r\n3861\r\n3862\r\n3863\r\n3864\r\n3865\r\n3866\r\n3867\r\n3868\r\n3869\r\n3870\r\n3871\r\n3872\r\n3873\r\n3874\r\n3875\r\n3876\r\n3877\r\n3878\r\n3879\r\n3880\r\n3881\r\n3882\r\n3883\r\n3884\r\n3885\r\n3886\r\n3887\r\n3888\r\n3889\r\n3890\r\n3891\r\n3892\r\n3893\r\n3894\r\n3895\r\n3896\r\n3897\r\n3898\r\n3899\r\n3900\r\n3901\r\n3902\r\n3903\r\n3904\r\n3905\r\n3906\r\n3907\r\n3908\r\n3909\r\n3910\r\n3911\r\n3912\r\n3913\r\n3914\r\n3915\r\n3916\r\n3917\r\n3918\r\n3919\r\n3920\r\n3921\r\n3922\r\n3923\r\n3924\r\n3925\r\n3926\r\n3927\r\n3928\r\n3929\r\n3930\r\n3931\r\n3932\r\n3933\r\n3934\r\n3935\r\n3936\r\n3937\r\n3938\r\n3939\r\n3940\r\n3941\r\n3942\r\n3943\r\n3944\r\n3945\r\n3946\r\n3947\r\n3948\r\n3949\r\n3950\r\n3951\r\n3952\r\n3953\r\n3954\r\n3955\r\n3956\r\n3957\r\n3958\r\n3959\r\n3960\r\n3961\r\n3962\r\n3963\r\n3964\r\n3965\r\n3966\r\n3967\r\n3968\r\n3969\r\n3970\r\n3971\r\n3972\r\n3973\r\n3974\r\n3975\r\n3976\r\n3977\r\n3978\r\n3979\r\n3980\r\n3981\r\n3982\r\n3983\r\n3984\r\n3985\r\n3986\r\n3987\r\n3988\r\n3989\r\n3990\r\n3991\r\n3992\r\n3993\r\n3994\r\n3995\r\n3996\r\n3997\r\n3998\r\n3999\r\n4000\r\n4001\r\n4002\r\n4003\r\n4004\r\n4005\r\n4006\r\n4007\r\n4008\r\n4009\r\n4010\r\n4011\r\n4012\r\n4013\r\n4014\r\n4015\r\n4016\r\n4017\r\n4018\r\n4019\r\n4020\r\n4021\r\n4022\r\n4023\r\n4024\r\n4025\r\n4026\r\n4027\r\n4028\r\n4029\r\n4030\r\n4031\r\n4032\r\n4033\r\n4034\r\n4035\r\n4036\r\n4037\r\n4038\r\n4039\r\n4040\r\n4041\r\n4042\r\n4043\r\n4044\r\n4045\r\n4046\r\n4047\r\n4048\r\n4049\r\n4050\r\n4051\r\n4052\r\n4053\r\n4054\r\n4055\r\n4056\r\n4057\r\n4058\r\n4059\r\n4060\r\n4061\r\n4062\r\n4063\r\n4064\r\n4065\r\n4066\r\n4067\r\n4068\r\n4069\r\n4070\r\n4071\r\n4072\r\n4073\r\n4074\r\n4075\r\n4076\r\n4077\r\n4078\r\n4079\r\n4080\r\n4081\r\n4082\r\n4083\r\n4084\r\n4085\r\n4086\r\n4087\r\n4088\r\n4089\r\n4090\r\n4091\r\n4092\r\n4093\r\n4094\r\n4095\r\n4096\r\n4097\r\n4098\r\n4099\r\n4100\r\n4101\r\n4102\r\n4103\r\n4104\r\n4105\r\n4106\r\n4107\r\n4108\r\n4109\r\n4110\r\n4111\r\n4112\r\n4113\r\n4114\r\n4115\r\n4116\r\n4117\r\n4118\r\n4119\r\n4120\r\n4121\r\n4122\r\n4123\r\n4124\r\n4125\r\n4126\r\n4127\r\n4128\r\n4129\r\n4130\r\n4131\r\n4132\r\n4133\r\n4134\r\n4135\r\n4136\r\n4137\r\n4138\r\n4139\r\n4140\r\n4141\r\n4142\r\n4143\r\n4144\r\n4145\r\n4146\r\n4147\r\n4148\r\n4149\r\n4150\r\n4151\r\n4152\r\n4153\r\n4154\r\n4155\r\n4156\r\n4157\r\n4158\r\n4159\r\n4160\r\n4161\r\n4162\r\n4163\r\n4164\r\n4165\r\n4166\r\n4167\r\n4168\r\n4169\r\n4170\r\n4171\r\n4172\r\n4173\r\n4174\r\n4175\r\n4176\r\n4177\r\n4178\r\n4179\r\n4180\r\n4181\r\n4182\r\n4183\r\n4184\r\n4185\r\n4186\r\n4187\r\n4188\r\n4189\r\n4190\r\n4191\r\n4192\r\n4193\r\n4194\r\n4195\r\n4196\r\n4197\r\n4198\r\n4199\r\n4200\r\n4201\r\n4202\r\n4203\r\n4204\r\n4205\r\n4206\r\n4207\r\n4208\r\n4209\r\n4210\r\n4211\r\n4212\r\n4213\r\n4214\r\n4215\r\n4216\r\n4217\r\n4218\r\n4219\r\n4220\r\n4221\r\n4222\r\n4223\r\n4224\r\n4225\r\n4226\r\n4227\r\n4228\r\n4229\r\n4230\r\n4231\r\n4232\r\n4233\r\n4234\r\n4235\r\n4236\r\n4237\r\n4238\r\n4239\r\n4240\r\n4241\r\n4242\r\n4243\r\n4244\r\n4245\r\n4246\r\n4247\r\n4248\r\n4249\r\n4250\r\n4251\r\n4252\r\n4253\r\n4254\r\n4255\r\n4256\r\n4257\r\n4258\r\n4259\r\n4260\r\n4261\r\n4262\r\n4263\r\n4264\r\n4265\r\n4266\r\n4267\r\n4268\r\n4269\r\n4270\r\n4271\r\n4272\r\n4273\r\n4274\r\n4275\r\n4276\r\n4277\r\n4278\r\n4279\r\n4280\r\n4281\r\n4282\r\n4283\r\n4284\r\n4285\r\n4286\r\n4287\r\n4288\r\n4289\r\n4290\r\n4291\r\n4292\r\n4293\r\n4294\r\n4295\r\n4296\r\n4297\r\n4298\r\n4299\r\n4300\r\n4301\r\n4302\r\n4303\r\n4304\r\n4305\r\n4306\r\n4307\r\n4308\r\n4309\r\n4310\r\n4311\r\n4312\r\n4313\r\n4314\r\n4315\r\n4316\r\n4317\r\n4318\r\n4319\r\n4320\r\n4321\r\n4322\r\n4323\r\n4324\r\n4325\r\n4326\r\n4327\r\n4328\r\n4329\r\n4330\r\n4331\r\n4332\r\n4333\r\n4334\r\n4335\r\n4336\r\n4337\r\n4338\r\n4339\r\n4340\r\n4341\r\n4342\r\n4343\r\n4344\r\n4345\r\n4346\r\n4347\r\n4348\r\n4349\r\n4350\r\n4351\r\n4352\r\n4353\r\n4354\r\n4355\r\n4356\r\n4357\r\n4358\r\n4359\r\n4360\r\n4361\r\n4362\r\n4363\r\n4364\r\n4365\r\n4366\r\n4367\r\n4368\r\n4369\r\n4370\r\n4371\r\n4372\r\n4373\r\n4374\r\n4375\r\n4376\r\n4377\r\n4378\r\n4379\r\n4380\r\n4381\r\n4382\r\n4383\r\n4384\r\n4385\r\n4386\r\n4387\r\n4388\r\n4389\r\n4390\r\n4391\r\n4392\r\n4393\r\n4394\r\n4395\r\n4396\r\n4397\r\n4398\r\n4399\r\n4400\r\n4401\r\n4402\r\n4403\r\n4404\r\n4405\r\n4406\r\n4407\r\n4408\r\n4409\r\n4410\r\n4411\r\n4412\r\n4413\r\n4414\r\n4415\r\n4416\r\n4417\r\n4418\r\n4419\r\n4420\r\n4421\r\n4422\r\n4423\r\n4424\r\n4425\r\n4426\r\n4427\r\n4428\r\n4429\r\n4430\r\n4431\r\n4432\r\n4433\r\n4434\r\n4435\r\n4436\r\n4437\r\n4438\r\n4439\r\n4440\r\n4441\r\n4442\r\n4443\r\n4444\r\n4445\r\n4446\r\n4447\r\n4448\r\n4449\r\n4450\r\n4451\r\n4452\r\n4453\r\n4454\r\n4455\r\n4456\r\n4457\r\n4458\r\n4459\r\n4460\r\n4461\r\n4462\r\n4463\r\n4464\r\n4465\r\n4466\r\n4467\r\n4468\r\n4469\r\n4470\r\n4471\r\n4472\r\n4473\r\n4474\r\n4475\r\n4476\r\n4477\r\n4478\r\n4479\r\n4480\r\n4481\r\n4482\r\n4483\r\n4484\r\n4485\r\n4486\r\n4487\r\n4488\r\n4489\r\n4490\r\n4491\r\n4492\r\n4493\r\n4494\r\n4495\r\n4496\r\n4497\r\n4498\r\n4499\r\n4500\r\n4501\r\n4502\r\n4503\r\n4504\r\n4505\r\n4506\r\n4507\r\n4508\r\n4509\r\n4510\r\n4511\r\n4512\r\n4513\r\n4514\r\n4515\r\n4516\r\n4517\r\n4518\r\n4519\r\n4520\r\n4521\r\n4522\r\n4523\r\n4524\r\n4525\r\n4526\r\n4527\r\n4528\r\n4529\r\n4530\r\n4531\r\n4532\r\n4533\r\n4534\r\n4535\r\n4536\r\n4537\r\n4538\r\n4539\r\n4540\r\n4541\r\n4542\r\n4543\r\n4544\r\n4545\r\n4546\r\n4547\r\n4548\r\n4549\r\n4550\r\n4551\r\n4552\r\n4553\r\n4554\r\n4555\r\n4556\r\n4557\r\n4558\r\n4559\r\n4560\r\n4561\r\n4562\r\n4563\r\n4564\r\n4565\r\n4566\r\n4567\r\n4568\r\n4569\r\n4570\r\n4571\r\n4572\r\n4573\r\n4574\r\n4575\r\n4576\r\n4577\r\n4578\r\n4579\r\n4580\r\n4581\r\n4582\r\n4583\r\n4584\r\n4585\r\n4586\r\n4587\r\n4588\r\n4589\r\n4590\r\n4591\r\n4592\r\n4593\r\n4594\r\n4595\r\n4596\r\n4597\r\n4598\r\n4599\r\n4600\r\n4601\r\n4602\r\n4603\r\n4604\r\n4605\r\n4606\r\n4607\r\n4608\r\n4609\r\n4610\r\n4611\r\n4612\r\n4613\r\n4614\r\n4615\r\n4616\r\n4617\r\n4618\r\n4619\r\n4620\r\n4621\r\n4622\r\n4623\r\n4624\r\n4625\r\n4626\r\n4627\r\n4628\r\n4629\r\n4630\r\n4631\r\n4632\r\n4633\r\n4634\r\n4635\r\n4636\r\n4637\r\n4638\r\n4639\r\n4640\r\n4641\r\n4642\r\n4643\r\n4644\r\n4645\r\n4646\r\n4647\r\n4648\r\n4649\r\n4650\r\n4651\r\n4652\r\n4653\r\n4654\r\n4655\r\n4656\r\n4657\r\n4658\r\n4659\r\n4660\r\n4661\r\n4662\r\n4663\r\n4664\r\n4665\r\n4666\r\n4667\r\n4668\r\n4669\r\n4670\r\n4671\r\n4672\r\n4673\r\n4674\r\n4675\r\n4676\r\n4677\r\n4678\r\n4679\r\n4680\r\n4681\r\n4682\r\n4683\r\n4684\r\n4685\r\n4686\r\n4687\r\n4688\r\n4689\r\n4690\r\n4691\r\n4692\r\n4693\r\n4694\r\n4695\r\n4696\r\n4697\r\n4698\r\n4699\r\n4700\r\n4701\r\n4702\r\n4703\r\n4704\r\n4705\r\n4706\r\n4707\r\n4708\r\n4709\r\n4710\r\n4711\r\n4712\r\n4713\r\n4714\r\n4715\r\n4716\r\n4717\r\n4718\r\n4719\r\n4720\r\n4721\r\n4722\r\n4723\r\n4724\r\n4725\r\n4726\r\n4727\r\n4728\r\n4729\r\n4730\r\n4731\r\n4732\r\n4733\r\n4734\r\n4735\r\n4736\r\n4737\r\n4738\r\n4739\r\n4740\r\n4741\r\n4742\r\n4743\r\n4744\r\n4745\r\n4746\r\n4747\r\n4748\r\n4749\r\n4750\r\n4751\r\n4752\r\n4753\r\n4754\r\n4755\r\n4756\r\n4757\r\n4758\r\n4759\r\n4760\r\n4761\r\n4762\r\n4763\r\n4764\r\n4765\r\n4766\r\n4767\r\n4768\r\n4769\r\n4770\r\n4771\r\n4772\r\n4773\r\n4774\r\n4775\r\n4776\r\n4777\r\n4778\r\n4779\r\n4780\r\n4781\r\n4782\r\n4783\r\n4784\r\n4785\r\n4786\r\n4787\r\n4788\r\n4789\r\n4790\r\n4791\r\n4792\r\n4793\r\n4794\r\n4795\r\n4796\r\n4797\r\n4798\r\n4799\r\n4800\r\n4801\r\n4802\r\n4803\r\n4804\r\n4805\r\n4806\r\n4807\r\n4808\r\n4809\r\n4810\r\n4811\r\n4812\r\n4813\r\n4814\r\n4815\r\n4816\r\n4817\r\n4818\r\n4819\r\n4820\r\n4821\r\n4822\r\n4823\r\n4824\r\n4825\r\n4826\r\n4827\r\n4828\r\n4829\r\n4830\r\n4831\r\n4832\r\n4833\r\n4834\r\n4835\r\n4836\r\n4837\r\n4838\r\n4839\r\n4840\r\n4841\r\n4842\r\n4843\r\n4844\r\n4845\r\n4846\r\n4847\r\n4848\r\n4849\r\n4850\r\n4851\r\n4852\r\n4853\r\n4854\r\n4855\r\n4856\r\n4857\r\n4858\r\n4859\r\n4860\r\n4861\r\n4862\r\n4863\r\n4864\r\n4865\r\n4866\r\n4867\r\n4868\r\n4869\r\n4870\r\n4871\r\n4872\r\n4873\r\n4874\r\n4875\r\n4876\r\n4877\r\n4878\r\n4879\r\n4880\r\n4881\r\n4882\r\n4883\r\n4884\r\n4885\r\n4886\r\n4887\r\n4888\r\n4889\r\n4890\r\n4891\r\n4892\r\n4893\r\n4894\r\n4895\r\n4896\r\n4897\r\n4898\r\n4899\r\n4900\r\n4901\r\n4902\r\n4903\r\n4904\r\n4905\r\n4906\r\n4907\r\n4908\r\n4909\r\n4910\r\n4911\r\n4912\r\n4913\r\n4914\r\n4915\r\n4916\r\n4917\r\n4918\r\n4919\r\n4920\r\n4921\r\n4922\r\n4923\r\n4924\r\n4925\r\n4926\r\n4927\r\n4928\r\n4929\r\n4930\r\n4931\r\n4932\r\n4933\r\n4934\r\n4935\r\n4936\r\n4937\r\n4938\r\n4939\r\n4940\r\n4941\r\n4942\r\n4943\r\n4944\r\n4945\r\n4946\r\n4947\r\n4948\r\n4949\r\n4950\r\n4951\r\n4952\r\n4953\r\n4954\r\n4955\r\n4956\r\n4957\r\n4958\r\n4959\r\n4960\r\n4961\r\n4962\r\n4963\r\n4964\r\n4965\r\n4966\r\n4967\r\n4968\r\n4969\r\n4970\r\n4971\r\n4972\r\n4973\r\n4974\r\n4975\r\n4976\r\n4977\r\n4978\r\n4979\r\n4980\r\n4981\r\n4982\r\n4983\r\n4984\r\n4985\r\n4986\r\n4987\r\n4988\r\n4989\r\n4990\r\n4991\r\n4992\r\n4993\r\n4994\r\n4995\r\n4996\r\n4997\r\n4998\r\n4999\r\n5000\r\n5001\r\n5002\r\n5003\r\n5004\r\n5005\r\n5006\r\n5007\r\n5008\r\n5009\r\n5010\r\n5011\r\n5012\r\n5013\r\n5014\r\n5015\r\n5016\r\n5017\r\n5018\r\n5019\r\n5020\r\n5021\r\n5022\r\n5023\r\n5024\r\n5025\r\n5026\r\n5027\r\n5028\r\n5029\r\n5030\r\n5031\r\n5032\r\n5033\r\n5034\r\n5035\r\n5036\r\n5037\r\n5038\r\n5039\r\n5040\r\n5041\r\n5042\r\n5043\r\n5044\r\n5045\r\n5046\r\n5047\r\n5048\r\n5049\r\n5050\r\n5051\r\n5052\r\n5053\r\n5054\r\n5055\r\n5056\r\n5057\r\n5058\r\n5059\r\n5060\r\n5061\r\n5062\r\n5063\r\n5064\r\n5065\r\n5066\r\n5067\r\n5068\r\n5069\r\n5070\r\n5071\r\n5072\r\n5073\r\n5074\r\n5075\r\n5076\r\n5077\r\n5078\r\n5079\r\n5080\r\n5081\r\n5082\r\n5083\r\n5084\r\n5085\r\n5086\r\n5087\r\n5088\r\n5089\r\n5090\r\n5091\r\n5092\r\n5093\r\n5094\r\n5095\r\n5096\r\n5097\r\n5098\r\n5099\r\n5100\r\n5101\r\n5102\r\n5103\r\n5104\r\n5105\r\n5106\r\n5107\r\n5108\r\n5109\r\n5110\r\n5111\r\n5112\r\n5113\r\n5114\r\n5115\r\n5116\r\n5117\r\n5118\r\n5119\r\n5120\r\n5121\r\n5122\r\n5123\r\n5124\r\n5125\r\n5126\r\n5127\r\n5128\r\n5129\r\n5130\r\n5131\r\n5132\r\n5133\r\n5134\r\n5135\r\n5136\r\n5137\r\n5138\r\n5139\r\n5140\r\n5141\r\n5142\r\n5143\r\n5144\r\n5145\r\n5146\r\n5147\r\n5148\r\n5149\r\n5150\r\n5151\r\n5152\r\n5153\r\n5154\r\n5155\r\n5156\r\n5157\r\n5158\r\n5159\r\n5160\r\n5161\r\n5162\r\n5163\r\n5164\r\n5165\r\n5166\r\n5167\r\n5168\r\n5169\r\n5170\r\n5171\r\n5172\r\n5173\r\n5174\r\n5175\r\n5176\r\n5177\r\n5178\r\n5179\r\n5180\r\n5181\r\n5182\r\n5183\r\n5184\r\n5185\r\n5186\r\n5187\r\n5188\r\n5189\r\n5190\r\n5191\r\n5192\r\n5193\r\n5194\r\n5195\r\n5196\r\n5197\r\n5198\r\n5199\r\n5200\r\n5201\r\n5202\r\n5203\r\n5204\r\n5205\r\n5206\r\n5207\r\n5208\r\n5209\r\n5210\r\n5211\r\n5212\r\n5213\r\n5214\r\n5215\r\n5216\r\n5217\r\n5218\r\n5219\r\n5220\r\n5221\r\n5222\r\n5223\r\n5224\r\n5225\r\n5226\r\n5227\r\n5228\r\n5229\r\n5230\r\n5231\r\n5232\r\n5233\r\n5234\r\n5235\r\n5236\r\n5237\r\n5238\r\n5239\r\n5240\r\n5241\r\n5242\r\n5243\r\n5244\r\n5245\r\n5246\r\n5247\r\n5248\r\n5249\r\n5250\r\n5251\r\n5252\r\n5253\r\n5254\r\n5255\r\n5256\r\n5257\r\n5258\r\n5259\r\n5260\r\n5261\r\n5262\r\n5263\r\n5264\r\n5265\r\n5266\r\n5267\r\n5268\r\n5269\r\n5270\r\n5271\r\n5272\r\n5273\r\n5274\r\n5275\r\n5276\r\n5277\r\n5278\r\n5279\r\n5280\r\n5281\r\n5282\r\n5283\r\n5284\r\n5285\r\n5286\r\n5287\r\n5288\r\n5289\r\n5290\r\n5291\r\n5292\r\n5293\r\n5294\r\n5295\r\n5296\r\n5297\r\n5298\r\n5299\r\n5300\r\n5301\r\n5302\r\n5303\r\n5304\r\n5305\r\n5306\r\n5307\r\n5308\r\n5309\r\n5310\r\n5311\r\n5312\r\n5313\r\n5314\r\n5315\r\n5316\r\n5317\r\n5318\r\n5319\r\n5320\r\n5321\r\n5322\r\n5323\r\n5324\r\n5325\r\n5326\r\n5327\r\n5328\r\n5329\r\n5330\r\n5331\r\n5332\r\n5333\r\n5334\r\n5335\r\n5336\r\n5337\r\n5338\r\n5339\r\n5340\r\n5341\r\n5342\r\n5343\r\n5344\r\n5345\r\n5346\r\n5347\r\n5348\r\n5349\r\n5350\r\n5351\r\n5352\r\n5353\r\n5354\r\n5355\r\n5356\r\n5357\r\n5358\r\n5359\r\n5360\r\n5361\r\n5362\r\n5363\r\n5364\r\n5365\r\n5366\r\n5367\r\n5368\r\n5369\r\n5370\r\n5371\r\n5372\r\n5373\r\n5374\r\n5375\r\n5376\r\n5377\r\n5378\r\n5379\r\n5380\r\n5381\r\n5382\r\n5383\r\n5384\r\n5385\r\n5386\r\n5387\r\n5388\r\n5389\r\n5390\r\n5391\r\n5392\r\n5393\r\n5394\r\n5395\r\n5396\r\n5397\r\n5398\r\n5399\r\n5400\r\n5401\r\n5402\r\n5403\r\n5404\r\n5405\r\n5406\r\n5407\r\n5408\r\n5409\r\n5410\r\n5411\r\n5412\r\n5413\r\n5414\r\n5415\r\n5416\r\n5417\r\n5418\r\n5419\r\n5420\r\n5421\r\n5422\r\n5423\r\n5424\r\n5425\r\n5426\r\n5427\r\n5428\r\n5429\r\n5430\r\n5431\r\n5432\r\n5433\r\n5434\r\n5435\r\n5436\r\n5437\r\n5438\r\n5439\r\n5440\r\n5441\r\n5442\r\n5443\r\n5444\r\n5445\r\n5446\r\n5447\r\n5448\r\n5449\r\n5450\r\n5451\r\n5452\r\n5453\r\n5454\r\n5455\r\n5456\r\n5457\r\n5458\r\n5459\r\n5460\r\n5461\r\n5462\r\n5463\r\n5464\r\n5465\r\n5466\r\n5467\r\n5468\r\n5469\r\n5470\r\n5471\r\n5472\r\n5473\r\n5474\r\n5475\r\n5476\r\n5477\r\n5478\r\n5479\r\n5480\r\n5481\r\n5482\r\n5483\r\n5484\r\n5485\r\n5486\r\n5487\r\n5488\r\n5489\r\n5490\r\n5491\r\n5492\r\n5493\r\n5494\r\n5495\r\n5496\r\n5497\r\n5498\r\n5499\r\n5500\r\n5501\r\n5502\r\n5503\r\n5504\r\n5505\r\n5506\r\n5507\r\n5508\r\n5509\r\n5510\r\n5511\r\n5512\r\n5513\r\n5514\r\n5515\r\n5516\r\n5517\r\n5518\r\n5519\r\n5520\r\n5521\r\n5522\r\n5523\r\n5524\r\n5525\r\n5526\r\n5527\r\n5528\r\n5529\r\n5530\r\n5531\r\n5532\r\n5533\r\n5534\r\n5535\r\n5536\r\n5537\r\n5538\r\n5539\r\n5540\r\n5541\r\n5542\r\n5543\r\n5544\r\n5545\r\n5546\r\n5547\r\n5548\r\n5549\r\n5550\r\n5551\r\n5552\r\n5553\r\n5554\r\n5555\r\n5556\r\n5557\r\n5558\r\n5559\r\n5560\r\n5561\r\n5562\r\n5563\r\n5564\r\n5565\r\n5566\r\n5567\r\n5568\r\n5569\r\n5570\r\n5571\r\n5572\r\n5573\r\n5574\r\n5575\r\n5576\r\n5577\r\n5578\r\n5579\r\n5580\r\n5581\r\n5582\r\n5583\r\n5584\r\n5585\r\n5586\r\n5587\r\n5588\r\n5589\r\n5590\r\n5591\r\n5592\r\n5593\r\n5594\r\n5595\r\n5596\r\n5597\r\n5598\r\n5599\r\n5600\r\n5601\r\n5602\r\n5603\r\n5604\r\n5605\r\n5606\r\n5607\r\n5608\r\n5609\r\n5610\r\n5611\r\n5612\r\n5613\r\n5614\r\n5615\r\n5616\r\n5617\r\n5618\r\n5619\r\n5620\r\n5621\r\n5622\r\n5623\r\n5624\r\n5625\r\n5626\r\n5627\r\n5628\r\n5629\r\n5630\r\n5631\r\n5632\r\n5633\r\n5634\r\n5635\r\n5636\r\n5637\r\n5638\r\n5639\r\n5640\r\n5641\r\n5642\r\n5643\r\n5644\r\n5645\r\n5646\r\n5647\r\n5648\r\n5649\r\n5650\r\n5651\r\n5652\r\n5653\r\n5654\r\n5655\r\n5656\r\n5657\r\n5658\r\n5659\r\n5660\r\n5661\r\n5662\r\n5663\r\n5664\r\n5665\r\n5666\r\n5667\r\n5668\r\n5669\r\n5670\r\n5671\r\n5672\r\n5673\r\n5674\r\n5675\r\n5676\r\n5677\r\n5678\r\n5679\r\n5680\r\n5681\r\n5682\r\n5683\r\n5684\r\n5685\r\n5686\r\n5687\r\n5688\r\n5689\r\n5690\r\n5691\r\n5692\r\n5693\r\n5694\r\n5695\r\n5696\r\n5697\r\n5698\r\n5699\r\n5700\r\n5701\r\n5702\r\n5703\r\n5704\r\n5705\r\n5706\r\n5707\r\n5708\r\n5709\r\n5710\r\n5711\r\n5712\r\n5713\r\n5714\r\n5715\r\n5716\r\n5717\r\n5718\r\n5719\r\n5720\r\n5721\r\n5722\r\n5723\r\n5724\r\n5725\r\n5726\r\n5727\r\n5728\r\n5729\r\n5730\r\n5731\r\n5732\r\n5733\r\n5734\r\n5735\r\n5736\r\n5737\r\n5738\r\n5739\r\n5740\r\n5741\r\n5742\r\n5743\r\n5744\r\n5745\r\n5746\r\n5747\r\n5748\r\n5749\r\n5750\r\n5751\r\n5752\r\n5753\r\n5754\r\n5755\r\n5756\r\n5757\r\n5758\r\n5759\r\n5760\r\n5761\r\n5762\r\n5763\r\n5764\r\n5765\r\n5766\r\n5767\r\n5768\r\n5769\r\n5770\r\n5771\r\n5772\r\n5773\r\n5774\r\n5775\r\n5776\r\n5777\r\n5778\r\n5779\r\n5780\r\n5781\r\n5782\r\n5783\r\n5784\r\n5785\r\n5786\r\n5787\r\n5788\r\n5789\r\n5790\r\n5791\r\n5792\r\n5793\r\n5794\r\n5795\r\n5796\r\n5797\r\n5798\r\n5799\r\n5800\r\n5801\r\n5802\r\n5803\r\n5804\r\n5805\r\n5806\r\n5807\r\n5808\r\n5809\r\n5810\r\n5811\r\n5812\r\n5813\r\n5814\r\n5815\r\n5816\r\n5817\r\n5818\r\n5819\r\n5820\r\n5821\r\n5822\r\n5823\r\n5824\r\n5825\r\n5826\r\n5827\r\n5828\r\n5829\r\n5830\r\n5831\r\n5832\r\n5833\r\n5834\r\n5835\r\n5836\r\n5837\r\n5838\r\n5839\r\n5840\r\n5841\r\n5842\r\n5843\r\n5844\r\n5845\r\n5846\r\n5847\r\n5848\r\n5849\r\n5850\r\n5851\r\n5852\r\n5853\r\n5854\r\n5855\r\n5856\r\n5857\r\n5858\r\n5859\r\n5860\r\n5861\r\n5862\r\n5863\r\n5864\r\n5865\r\n5866\r\n5867\r\n5868\r\n5869\r\n5870\r\n5871\r\n5872\r\n5873\r\n5874\r\n5875\r\n5876\r\n5877\r\n5878\r\n5879\r\n5880\r\n5881\r\n5882\r\n5883\r\n5884\r\n5885\r\n5886\r\n5887\r\n5888\r\n5889\r\n5890\r\n5891\r\n5892\r\n5893\r\n5894\r\n5895\r\n5896\r\n5897\r\n5898\r\n5899\r\n5900\r\n5901\r\n5902\r\n5903\r\n5904\r\n5905\r\n5906\r\n5907\r\n5908\r\n5909\r\n5910\r\n5911\r\n5912\r\n5913\r\n5914\r\n5915\r\n5916\r\n5917\r\n5918\r\n5919\r\n5920\r\n5921\r\n5922\r\n5923\r\n5924\r\n5925\r\n5926\r\n5927\r\n5928\r\n5929\r\n5930\r\n5931\r\n5932\r\n5933\r\n5934\r\n5935\r\n5936\r\n5937\r\n5938\r\n5939\r\n5940\r\n5941\r\n5942\r\n5943\r\n5944\r\n5945\r\n5946\r\n5947\r\n5948\r\n5949\r\n5950\r\n5951\r\n5952\r\n5953\r\n5954\r\n5955\r\n5956\r\n5957\r\n5958\r\n5959\r\n5960\r\n5961\r\n5962\r\n5963\r\n5964\r\n5965\r\n5966\r\n5967\r\n5968\r\n5969\r\n5970\r\n5971\r\n5972\r\n5973\r\n5974\r\n5975\r\n5976\r\n5977\r\n5978\r\n5979\r\n5980\r\n5981\r\n5982\r\n5983\r\n5984\r\n5985\r\n5986\r\n5987\r\n5988\r\n5989\r\n5990\r\n5991\r\n5992\r\n5993\r\n5994\r\n5995\r\n5996\r\n5997\r\n5998\r\n5999\r\n6000\r\n6001\r\n6002\r\n6003\r\n6004\r\n6005\r\n6006\r\n6007\r\n6008\r\n6009\r\n6010\r\n6011\r\n6012\r\n6013\r\n6014\r\n6015\r\n6016\r\n6017\r\n6018\r\n6019\r\n6020\r\n6021\r\n6022\r\n6023\r\n6024\r\n6025\r\n6026\r\n6027\r\n6028\r\n6029\r\n6030\r\n6031\r\n6032\r\n6033\r\n6034\r\n6035\r\n6036\r\n6037\r\n6038\r\n6039\r\n6040\r\n6041\r\n6042\r\n6043\r\n6044\r\n6045\r\n6046\r\n6047\r\n6048\r\n6049\r\n6050\r\n6051\r\n6052\r\n6053\r\n6054\r\n6055\r\n6056\r\n6057\r\n6058\r\n6059\r\n6060\r\n6061\r\n6062\r\n6063\r\n6064\r\n6065\r\n6066\r\n6067\r\n6068\r\n6069\r\n6070\r\n6071\r\n6072\r\n6073\r\n6074\r\n6075\r\n6076\r\n6077\r\n6078\r\n6079\r\n6080\r\n6081\r\n6082\r\n6083\r\n6084\r\n6085\r\n6086\r\n6087\r\n6088\r\n6089\r\n6090\r\n6091\r\n6092\r\n6093\r\n6094\r\n6095\r\n6096\r\n6097\r\n6098\r\n6099\r\n6100\r\n6101\r\n6102\r\n6103\r\n6104\r\n6105\r\n6106\r\n6107\r\n6108\r\n6109\r\n6110\r\n6111\r\n6112\r\n6113\r\n6114\r\n6115\r\n6116\r\n6117\r\n6118\r\n6119\r\n6120\r\n6121\r\n6122\r\n6123\r\n6124\r\n6125\r\n6126\r\n6127\r\n6128\r\n6129\r\n6130\r\n6131\r\n6132\r\n6133\r\n6134\r\n6135\r\n6136\r\n6137\r\n6138\r\n6139\r\n6140\r\n6141\r\n6142\r\n6143\r\n6144\r\n6145\r\n6146\r\n6147\r\n6148\r\n6149\r\n6150\r\n6151\r\n6152\r\n6153\r\n6154\r\n6155\r\n6156\r\n6157\r\n6158\r\n6159\r\n6160\r\n6161\r\n6162\r\n6163\r\n6164\r\n6165\r\n6166\r\n6167\r\n6168\r\n6169\r\n6170\r\n6171\r\n6172\r\n6173\r\n6174\r\n6175\r\n6176\r\n6177\r\n6178\r\n6179\r\n6180\r\n6181\r\n6182\r\n6183\r\n6184\r\n6185\r\n6186\r\n6187\r\n6188\r\n6189\r\n6190\r\n6191\r\n6192\r\n6193\r\n6194\r\n6195\r\n6196\r\n6197\r\n6198\r\n6199\r\n6200\r\n6201\r\n6202\r\n6203\r\n6204\r\n6205\r\n6206\r\n6207\r\n6208\r\n6209\r\n6210\r\n6211\r\n6212\r\n6213\r\n6214\r\n6215\r\n6216\r\n6217\r\n6218\r\n6219\r\n6220\r\n6221\r\n6222\r\n6223\r\n6224\r\n6225\r\n6226\r\n6227\r\n6228\r\n6229\r\n6230\r\n6231\r\n6232\r\n6233\r\n6234\r\n6235\r\n6236\r\n6237\r\n6238\r\n6239\r\n6240\r\n6241\r\n6242\r\n6243\r\n6244\r\n6245\r\n6246\r\n6247\r\n6248\r\n6249\r\n6250\r\n6251\r\n6252\r\n6253\r\n6254\r\n6255\r\n6256\r\n6257\r\n6258\r\n6259\r\n6260\r\n6261\r\n6262\r\n6263\r\n6264\r\n6265\r\n6266\r\n6267\r\n6268\r\n6269\r\n6270\r\n6271\r\n6272\r\n6273\r\n6274\r\n6275\r\n6276\r\n6277\r\n6278\r\n6279\r\n6280\r\n6281\r\n6282\r\n6283\r\n6284\r\n6285\r\n6286\r\n6287\r\n6288\r\n6289\r\n6290\r\n6291\r\n6292\r\n6293\r\n6294\r\n6295\r\n6296\r\n6297\r\n6298\r\n6299\r\n6300\r\n6301\r\n6302\r\n6303\r\n6304\r\n6305\r\n6306\r\n6307\r\n6308\r\n6309\r\n6310\r\n6311\r\n6312\r\n6313\r\n6314\r\n6315\r\n6316\r\n6317\r\n6318\r\n6319\r\n6320\r\n6321\r\n6322\r\n6323\r\n6324\r\n6325\r\n6326\r\n6327\r\n6328\r\n6329\r\n6330\r\n6331\r\n6332\r\n6333\r\n6334\r\n6335\r\n6336\r\n6337\r\n6338\r\n6339\r\n6340\r\n6341\r\n6342\r\n6343\r\n6344\r\n6345\r\n6346\r\n6347\r\n6348\r\n6349\r\n6350\r\n6351\r\n6352\r\n6353\r\n6354\r\n6355\r\n6356\r\n6357\r\n6358\r\n6359\r\n6360\r\n6361\r\n6362\r\n6363\r\n6364\r\n6365\r\n6366\r\n6367\r\n6368\r\n6369\r\n6370\r\n6371\r\n6372\r\n6373\r\n6374\r\n6375\r\n6376\r\n6377\r\n6378\r\n6379\r\n6380\r\n6381\r\n6382\r\n6383\r\n6384\r\n6385\r\n6386\r\n6387\r\n6388\r\n6389\r\n6390\r\n6391\r\n6392\r\n6393\r\n6394\r\n6395\r\n6396\r\n6397\r\n6398\r\n6399\r\n6400\r\n6401\r\n6402\r\n6403\r\n6404\r\n6405\r\n6406\r\n6407\r\n6408\r\n6409\r\n6410\r\n6411\r\n6412\r\n6413\r\n6414\r\n6415\r\n6416\r\n6417\r\n6418\r\n6419\r\n6420\r\n6421\r\n6422\r\n6423\r\n6424\r\n6425\r\n6426\r\n6427\r\n6428\r\n6429\r\n6430\r\n6431\r\n6432\r\n6433\r\n6434\r\n6435\r\n6436\r\n6437\r\n6438\r\n6439\r\n6440\r\n6441\r\n6442\r\n6443\r\n6444\r\n6445\r\n6446\r\n6447\r\n6448\r\n6449\r\n6450\r\n6451\r\n6452\r\n6453\r\n6454\r\n6455\r\n6456\r\n6457\r\n6458\r\n6459\r\n6460\r\n6461\r\n6462\r\n6463\r\n6464\r\n6465\r\n6466\r\n6467\r\n6468\r\n6469\r\n6470\r\n6471\r\n6472\r\n6473\r\n6474\r\n6475\r\n6476\r\n6477\r\n6478\r\n6479\r\n6480\r\n6481\r\n6482\r\n6483\r\n6484\r\n6485\r\n6486\r\n6487\r\n6488\r\n6489\r\n6490\r\n6491\r\n6492\r\n6493\r\n6494\r\n6495\r\n6496\r\n6497\r\n6498\r\n6499\r\n6500\r\n6501\r\n6502\r\n6503\r\n6504\r\n6505\r\n6506\r\n6507\r\n6508\r\n6509\r\n6510\r\n6511\r\n6512\r\n6513\r\n6514\r\n6515\r\n6516\r\n6517\r\n6518\r\n6519\r\n6520\r\n6521\r\n6522\r\n6523\r\n6524\r\n6525\r\n6526\r\n6527\r\n6528\r\n6529\r\n6530\r\n6531\r\n6532\r\n6533\r\n6534\r\n6535\r\n6536\r\n6537\r\n6538\r\n6539\r\n6540\r\n6541\r\n6542\r\n6543\r\n6544\r\n6545\r\n6546\r\n6547\r\n6548\r\n6549\r\n6550\r\n6551\r\n6552\r\n6553\r\n6554\r\n6555\r\n6556\r\n6557\r\n6558\r\n6559\r\n6560\r\n6561\r\n6562\r\n6563\r\n6564\r\n6565\r\n6566\r\n6567\r\n6568\r\n6569\r\n6570\r\n6571\r\n6572\r\n6573\r\n6574\r\n6575\r\n6576\r\n6577\r\n6578\r\n6579\r\n6580\r\n6581\r\n6582\r\n6583\r\n6584\r\n6585\r\n6586\r\n6587\r\n6588\r\n6589\r\n6590\r\n6591\r\n6592\r\n6593\r\n6594\r\n6595\r\n6596\r\n6597\r\n6598\r\n6599\r\n6600\r\n6601\r\n6602\r\n6603\r\n6604\r\n6605\r\n6606\r\n6607\r\n6608\r\n6609\r\n6610\r\n6611\r\n6612\r\n6613\r\n6614\r\n6615\r\n6616\r\n6617\r\n6618\r\n6619\r\n6620\r\n6621\r\n6622\r\n6623\r\n6624\r\n6625\r\n6626\r\n6627\r\n6628\r\n6629\r\n6630\r\n6631\r\n6632\r\n6633\r\n6634\r\n6635\r\n6636\r\n6637\r\n6638\r\n6639\r\n6640\r\n6641\r\n6642\r\n6643\r\n6644\r\n6645\r\n6646\r\n6647\r\n6648\r\n6649\r\n6650\r\n6651\r\n6652\r\n6653\r\n6654\r\n6655\r\n6656\r\n6657\r\n6658\r\n6659\r\n6660\r\n6661\r\n6662\r\n6663\r\n6664\r\n6665\r\n6666\r\n6667\r\n6668\r\n6669\r\n6670\r\n6671\r\n6672\r\n6673\r\n6674\r\n6675\r\n6676\r\n6677\r\n6678\r\n6679\r\n6680\r\n6681\r\n6682\r\n6683\r\n6684\r\n6685\r\n6686\r\n6687\r\n6688\r\n6689\r\n6690\r\n6691\r\n6692\r\n6693\r\n6694\r\n6695\r\n6696\r\n6697\r\n6698\r\n6699\r\n6700\r\n6701\r\n6702\r\n6703\r\n6704\r\n6705\r\n6706\r\n6707\r\n6708\r\n6709\r\n6710\r\n6711\r\n6712\r\n6713\r\n6714\r\n6715\r\n6716\r\n6717\r\n6718\r\n6719\r\n6720\r\n6721\r\n6722\r\n6723\r\n6724\r\n6725\r\n6726\r\n6727\r\n6728\r\n6729\r\n6730\r\n6731\r\n6732\r\n6733\r\n6734\r\n6735\r\n6736\r\n6737\r\n6738\r\n6739\r\n6740\r\n6741\r\n6742\r\n6743\r\n6744\r\n6745\r\n6746\r\n6747\r\n6748\r\n6749\r\n6750\r\n6751\r\n6752\r\n6753\r\n6754\r\n6755\r\n6756\r\n6757\r\n6758\r\n6759\r\n6760\r\n6761\r\n6762\r\n6763\r\n6764\r\n6765\r\n6766\r\n6767\r\n6768\r\n6769\r\n6770\r\n6771\r\n6772\r\n6773\r\n6774\r\n6775\r\n6776\r\n6777\r\n6778\r\n6779\r\n6780\r\n6781\r\n6782\r\n6783\r\n6784\r\n6785\r\n6786\r\n6787\r\n6788\r\n6789\r\n6790\r\n6791\r\n6792\r\n6793\r\n6794\r\n6795\r\n6796\r\n6797\r\n6798\r\n6799\r\n6800\r\n6801\r\n6802\r\n6803\r\n6804\r\n6805\r\n6806\r\n6807\r\n6808\r\n6809\r\n6810\r\n6811\r\n6812\r\n6813\r\n6814\r\n6815\r\n6816\r\n6817\r\n6818\r\n6819\r\n6820\r\n6821\r\n6822\r\n6823\r\n6824\r\n6825\r\n6826\r\n6827\r\n6828\r\n6829\r\n6830\r\n6831\r\n6832\r\n6833\r\n6834\r\n6835\r\n6836\r\n6837\r\n6838\r\n6839\r\n6840\r\n6841\r\n6842\r\n6843\r\n6844\r\n6845\r\n6846\r\n6847\r\n6848\r\n6849\r\n6850\r\n6851\r\n6852\r\n6853\r\n6854\r\n6855\r\n6856\r\n6857\r\n6858\r\n6859\r\n6860\r\n6861\r\n6862\r\n6863\r\n6864\r\n6865\r\n6866\r\n6867\r\n6868\r\n6869\r\n6870\r\n6871\r\n6872\r\n6873\r\n6874\r\n6875\r\n6876\r\n6877\r\n6878\r\n6879\r\n6880\r\n6881\r\n6882\r\n6883\r\n6884\r\n6885\r\n6886\r\n6887\r\n6888\r\n6889\r\n6890\r\n6891\r\n6892\r\n6893\r\n6894\r\n6895\r\n6896\r\n6897\r\n6898\r\n6899\r\n6900\r\n6901\r\n6902\r\n6903\r\n6904\r\n6905\r\n6906\r\n6907\r\n6908\r\n6909\r\n6910\r\n6911\r\n6912\r\n6913\r\n6914\r\n6915\r\n6916\r\n6917\r\n6918\r\n6919\r\n6920\r\n6921\r\n6922\r\n6923\r\n6924\r\n6925\r\n6926\r\n6927\r\n6928\r\n6929\r\n6930\r\n6931\r\n6932\r\n6933\r\n6934\r\n6935\r\n6936\r\n6937\r\n6938\r\n6939\r\n6940\r\n6941\r\n6942\r\n6943\r\n6944\r\n6945\r\n6946\r\n6947\r\n6948\r\n6949\r\n6950\r\n6951\r\n6952\r\n6953\r\n6954\r\n6955\r\n6956\r\n6957\r\n6958\r\n6959\r\n6960\r\n6961\r\n6962\r\n6963\r\n6964\r\n6965\r\n6966\r\n6967\r\n6968\r\n6969\r\n6970\r\n6971\r\n6972\r\n6973\r\n6974\r\n6975\r\n6976\r\n6977\r\n6978\r\n6979\r\n6980\r\n6981\r\n6982\r\n6983\r\n6984\r\n6985\r\n6986\r\n6987\r\n6988\r\n6989\r\n6990\r\n6991\r\n6992\r\n6993\r\n6994\r\n6995\r\n6996\r\n6997\r\n6998\r\n6999\r\n7000\r\n7001\r\n7002\r\n7003\r\n7004\r\n7005\r\n7006\r\n7007\r\n7008\r\n7009\r\n7010\r\n7011\r\n7012\r\n7013\r\n7014\r\n7015\r\n7016\r\n7017\r\n7018\r\n7019\r\n7020\r\n7021\r\n7022\r\n7023\r\n7024\r\n7025\r\n7026\r\n7027\r\n7028\r\n7029\r\n7030\r\n7031\r\n7032\r\n7033\r\n7034\r\n7035\r\n7036\r\n7037\r\n7038\r\n7039\r\n7040\r\n7041\r\n7042\r\n7043\r\n7044\r\n7045\r\n7046\r\n7047\r\n7048\r\n7049\r\n7050\r\n7051\r\n7052\r\n7053\r\n7054\r\n7055\r\n7056\r\n7057\r\n7058\r\n7059\r\n7060\r\n7061\r\n7062\r\n7063\r\n7064\r\n7065\r\n7066\r\n7067\r\n7068\r\n7069\r\n7070\r\n7071\r\n7072\r\n7073\r\n7074\r\n7075\r\n7076\r\n7077\r\n7078\r\n7079\r\n7080\r\n7081\r\n7082\r\n7083\r\n7084\r\n7085\r\n7086\r\n7087\r\n7088\r\n7089\r\n7090\r\n7091\r\n7092\r\n7093\r\n7094\r\n7095\r\n7096\r\n7097\r\n7098\r\n7099\r\n7100\r\n7101\r\n7102\r\n7103\r\n7104\r\n7105\r\n7106\r\n7107\r\n7108\r\n7109\r\n7110\r\n7111\r\n7112\r\n7113\r\n7114\r\n7115\r\n7116\r\n7117\r\n7118\r\n7119\r\n7120\r\n7121\r\n7122\r\n7123\r\n7124\r\n7125\r\n7126\r\n7127\r\n7128\r\n7129\r\n7130\r\n7131\r\n7132\r\n7133\r\n7134\r\n7135\r\n7136\r\n7137\r\n7138\r\n7139\r\n7140\r\n7141\r\n7142\r\n7143\r\n7144\r\n7145\r\n7146\r\n7147\r\n7148\r\n7149\r\n7150\r\n7151\r\n7152\r\n7153\r\n7154\r\n7155\r\n7156\r\n7157\r\n7158\r\n7159\r\n7160\r\n7161\r\n7162\r\n7163\r\n7164\r\n7165\r\n7166\r\n7167\r\n7168\r\n7169\r\n7170\r\n7171\r\n7172\r\n7173\r\n7174\r\n7175\r\n7176\r\n7177\r\n7178\r\n7179\r\n7180\r\n7181\r\n7182\r\n7183\r\n7184\r\n7185\r\n7186\r\n7187\r\n7188\r\n7189\r\n7190\r\n7191\r\n7192\r\n7193\r\n7194\r\n7195\r\n7196\r\n7197\r\n7198\r\n7199\r\n7200\r\n7201\r\n7202\r\n7203\r\n7204\r\n7205\r\n7206\r\n7207\r\n7208\r\n7209\r\n7210\r\n7211\r\n7212\r\n7213\r\n7214\r\n7215\r\n7216\r\n7217\r\n7218\r\n7219\r\n7220\r\n7221\r\n7222\r\n7223\r\n7224\r\n7225\r\n7226\r\n7227\r\n7228\r\n7229\r\n7230\r\n7231\r\n7232\r\n7233\r\n7234\r\n7235\r\n7236\r\n7237\r\n7238\r\n7239\r\n7240\r\n7241\r\n7242\r\n7243\r\n7244\r\n7245\r\n7246\r\n7247\r\n7248\r\n7249\r\n7250\r\n7251\r\n7252\r\n7253\r\n7254\r\n7255\r\n7256\r\n7257\r\n7258\r\n7259\r\n7260\r\n7261\r\n7262\r\n7263\r\n7264\r\n7265\r\n7266\r\n7267\r\n7268\r\n7269\r\n7270\r\n7271\r\n7272\r\n7273\r\n7274\r\n7275\r\n7276\r\n7277\r\n7278\r\n7279\r\n7280\r\n7281\r\n7282\r\n7283\r\n7284\r\n7285\r\n7286\r\n7287\r\n7288\r\n7289\r\n7290\r\n7291\r\n7292\r\n7293\r\n7294\r\n7295\r\n7296\r\n7297\r\n7298\r\n7299\r\n7300\r\n7301\r\n7302\r\n7303\r\n7304\r\n7305\r\n7306\r\n7307\r\n7308\r\n7309\r\n7310\r\n7311\r\n7312\r\n7313\r\n7314\r\n7315\r\n7316\r\n7317\r\n7318\r\n7319\r\n7320\r\n7321\r\n7322\r\n7323\r\n7324\r\n7325\r\n7326\r\n7327\r\n7328\r\n7329\r\n7330\r\n7331\r\n7332\r\n7333\r\n7334\r\n7335\r\n7336\r\n7337\r\n7338\r\n7339\r\n7340\r\n7341\r\n7342\r\n7343\r\n7344\r\n7345\r\n7346\r\n7347\r\n7348\r\n7349\r\n7350\r\n7351\r\n7352\r\n7353\r\n7354\r\n7355\r\n7356\r\n7357\r\n7358\r\n7359\r\n7360\r\n7361\r\n7362\r\n7363\r\n7364\r\n7365\r\n7366\r\n7367\r\n7368\r\n7369\r\n7370\r\n7371\r\n7372\r\n7373\r\n7374\r\n7375\r\n7376\r\n7377\r\n7378\r\n7379\r\n7380\r\n7381\r\n7382\r\n7383\r\n7384\r\n7385\r\n7386\r\n7387\r\n7388\r\n7389\r\n7390\r\n7391\r\n7392\r\n7393\r\n7394\r\n7395\r\n7396\r\n7397\r\n7398\r\n7399\r\n7400\r\n7401\r\n7402\r\n7403\r\n7404\r\n7405\r\n7406\r\n7407\r\n7408\r\n7409\r\n7410\r\n7411\r\n7412\r\n7413\r\n7414\r\n7415\r\n7416\r\n7417\r\n7418\r\n7419\r\n7420\r\n7421\r\n7422\r\n7423\r\n7424\r\n7425\r\n7426\r\n7427\r\n7428\r\n7429\r\n7430\r\n7431\r\n7432\r\n7433\r\n7434\r\n7435\r\n7436\r\n7437\r\n7438\r\n7439\r\n7440\r\n7441\r\n7442\r\n7443\r\n7444\r\n7445\r\n7446\r\n7447\r\n7448\r\n7449\r\n7450\r\n7451\r\n7452\r\n7453\r\n7454\r\n7455\r\n7456\r\n7457\r\n7458\r\n7459\r\n7460\r\n7461\r\n7462\r\n7463\r\n7464\r\n7465\r\n7466\r\n7467\r\n7468\r\n7469\r\n7470\r\n7471\r\n7472\r\n7473\r\n7474\r\n7475\r\n7476\r\n7477\r\n7478\r\n7479\r\n7480\r\n7481\r\n7482\r\n7483\r\n7484\r\n7485\r\n7486\r\n7487\r\n7488\r\n7489\r\n7490\r\n7491\r\n7492\r\n7493\r\n7494\r\n7495\r\n7496\r\n7497\r\n7498\r\n7499\r\n7500\r\n7501\r\n7502\r\n7503\r\n7504\r\n7505\r\n7506\r\n7507\r\n7508\r\n7509\r\n7510\r\n7511\r\n7512\r\n7513\r\n7514\r\n7515\r\n7516\r\n7517\r\n7518\r\n7519\r\n7520\r\n7521\r\n7522\r\n7523\r\n7524\r\n7525\r\n7526\r\n7527\r\n7528\r\n7529\r\n7530\r\n7531\r\n7532\r\n7533\r\n7534\r\n7535\r\n7536\r\n7537\r\n7538\r\n7539\r\n7540\r\n7541\r\n7542\r\n7543\r\n7544\r\n7545\r\n7546\r\n7547\r\n7548\r\n7549\r\n7550\r\n7551\r\n7552\r\n7553\r\n7554\r\n7555\r\n7556\r\n7557\r\n7558\r\n7559\r\n7560\r\n7561\r\n7562\r\n7563\r\n7564\r\n7565\r\n7566\r\n7567\r\n7568\r\n7569\r\n7570\r\n7571\r\n7572\r\n7573\r\n7574\r\n7575\r\n7576\r\n7577\r\n7578\r\n7579\r\n7580\r\n7581\r\n7582\r\n7583\r\n7584\r\n7585\r\n7586\r\n7587\r\n7588\r\n7589\r\n7590\r\n7591\r\n7592\r\n7593\r\n7594\r\n7595\r\n7596\r\n7597\r\n7598\r\n7599\r\n7600\r\n7601\r\n7602\r\n7603\r\n7604\r\n7605\r\n7606\r\n7607\r\n7608\r\n7609\r\n7610\r\n7611\r\n7612\r\n7613\r\n7614\r\n7615\r\n7616\r\n7617\r\n7618\r\n7619\r\n7620\r\n7621\r\n7622\r\n7623\r\n7624\r\n7625\r\n7626\r\n7627\r\n7628\r\n7629\r\n7630\r\n7631\r\n7632\r\n7633\r\n7634\r\n7635\r\n7636\r\n7637\r\n7638\r\n7639\r\n7640\r\n7641\r\n7642\r\n7643\r\n7644\r\n7645\r\n7646\r\n7647\r\n7648\r\n7649\r\n7650\r\n7651\r\n7652\r\n7653\r\n7654\r\n7655\r\n7656\r\n7657\r\n7658\r\n7659\r\n7660\r\n7661\r\n7662\r\n7663\r\n7664\r\n7665\r\n7666\r\n7667\r\n7668\r\n7669\r\n7670\r\n7671\r\n7672\r\n7673\r\n7674\r\n7675\r\n7676\r\n7677\r\n7678\r\n7679\r\n7680\r\n7681\r\n7682\r\n7683\r\n7684\r\n7685\r\n7686\r\n7687\r\n7688\r\n7689\r\n7690\r\n7691\r\n7692\r\n7693\r\n7694\r\n7695\r\n7696\r\n7697\r\n7698\r\n7699\r\n7700\r\n7701\r\n7702\r\n7703\r\n7704\r\n7705\r\n7706\r\n7707\r\n7708\r\n7709\r\n7710\r\n7711\r\n7712\r\n7713\r\n7714\r\n7715\r\n7716\r\n7717\r\n7718\r\n7719\r\n7720\r\n7721\r\n7722\r\n7723\r\n7724\r\n7725\r\n7726\r\n7727\r\n7728\r\n7729\r\n7730\r\n7731\r\n7732\r\n7733\r\n7734\r\n7735\r\n7736\r\n7737\r\n7738\r\n7739\r\n7740\r\n7741\r\n7742\r\n7743\r\n7744\r\n7745\r\n7746\r\n7747\r\n7748\r\n7749\r\n7750\r\n7751\r\n7752\r\n7753\r\n7754\r\n7755\r\n7756\r\n7757\r\n7758\r\n7759\r\n7760\r\n7761\r\n7762\r\n7763\r\n7764\r\n7765\r\n7766\r\n7767\r\n7768\r\n7769\r\n7770\r\n7771\r\n7772\r\n7773\r\n7774\r\n7775\r\n7776\r\n7777\r\n7778\r\n7779\r\n7780\r\n7781\r\n7782\r\n7783\r\n7784\r\n7785\r\n7786\r\n7787\r\n7788\r\n7789\r\n7790\r\n7791\r\n7792\r\n7793\r\n7794\r\n7795\r\n7796\r\n7797\r\n7798\r\n7799\r\n7800\r\n7801\r\n7802\r\n7803\r\n7804\r\n7805\r\n7806\r\n7807\r\n7808\r\n7809\r\n7810\r\n7811\r\n7812\r\n7813\r\n7814\r\n7815\r\n7816\r\n7817\r\n7818\r\n7819\r\n7820\r\n7821\r\n7822\r\n7823\r\n7824\r\n7825\r\n7826\r\n7827\r\n7828\r\n7829\r\n7830\r\n7831\r\n7832\r\n7833\r\n7834\r\n7835\r\n7836\r\n7837\r\n7838\r\n7839\r\n7840\r\n7841\r\n7842\r\n7843\r\n7844\r\n7845\r\n7846\r\n7847\r\n7848\r\n7849\r\n7850\r\n7851\r\n7852\r\n7853\r\n7854\r\n7855\r\n7856\r\n7857\r\n7858\r\n7859\r\n7860\r\n7861\r\n7862\r\n7863\r\n7864\r\n7865\r\n7866\r\n7867\r\n7868\r\n7869\r\n7870\r\n7871\r\n7872\r\n7873\r\n7874\r\n7875\r\n7876\r\n7877\r\n7878\r\n7879\r\n7880\r\n7881\r\n7882\r\n7883\r\n7884\r\n7885\r\n7886\r\n7887\r\n7888\r\n7889\r\n7890\r\n7891\r\n7892\r\n7893\r\n7894\r\n7895\r\n7896\r\n7897\r\n7898\r\n7899\r\n7900\r\n7901\r\n7902\r\n7903\r\n7904\r\n7905\r\n7906\r\n7907\r\n7908\r\n7909\r\n7910\r\n7911\r\n7912\r\n7913\r\n7914\r\n7915\r\n7916\r\n7917\r\n7918\r\n7919\r\n7920\r\n7921\r\n7922\r\n7923\r\n7924\r\n7925\r\n7926\r\n7927\r\n7928\r\n7929\r\n7930\r\n7931\r\n7932\r\n7933\r\n7934\r\n7935\r\n7936\r\n7937\r\n7938\r\n7939\r\n7940\r\n7941\r\n7942\r\n7943\r\n7944\r\n7945\r\n7946\r\n7947\r\n7948\r\n7949\r\n7950\r\n7951\r\n7952\r\n7953\r\n7954\r\n7955\r\n7956\r\n7957\r\n7958\r\n7959\r\n7960\r\n7961\r\n7962\r\n7963\r\n7964\r\n7965\r\n7966\r\n7967\r\n7968\r\n7969\r\n7970\r\n7971\r\n7972\r\n7973\r\n7974\r\n7975\r\n7976\r\n7977\r\n7978\r\n7979\r\n7980\r\n7981\r\n7982\r\n7983\r\n7984\r\n7985\r\n7986\r\n7987\r\n7988\r\n7989\r\n7990\r\n7991\r\n7992\r\n7993\r\n7994\r\n7995\r\n7996\r\n7997\r\n7998\r\n7999\r\n8000\r\n8001\r\n8002\r\n8003\r\n8004\r\n8005\r\n8006\r\n8007\r\n8008\r\n8009\r\n8010\r\n8011\r\n8012\r\n8013\r\n8014\r\n8015\r\n8016\r\n8017\r\n8018\r\n8019\r\n8020\r\n8021\r\n8022\r\n8023\r\n8024\r\n8025\r\n8026\r\n8027\r\n8028\r\n8029\r\n8030\r\n8031\r\n8032\r\n8033\r\n8034\r\n8035\r\n8036\r\n8037\r\n8038\r\n8039\r\n8040\r\n8041\r\n8042\r\n8043\r\n8044\r\n8045\r\n8046\r\n8047\r\n8048\r\n8049\r\n8050\r\n8051\r\n8052\r\n8053\r\n8054\r\n8055\r\n8056\r\n8057\r\n8058\r\n8059\r\n8060\r\n8061\r\n8062\r\n8063\r\n8064\r\n8065\r\n8066\r\n8067\r\n8068\r\n8069\r\n8070\r\n8071\r\n8072\r\n8073\r\n8074\r\n8075\r\n8076\r\n8077\r\n8078\r\n8079\r\n8080\r\n8081\r\n8082\r\n8083\r\n8084\r\n8085\r\n8086\r\n8087\r\n8088\r\n8089\r\n8090\r\n8091\r\n8092\r\n8093\r\n8094\r\n8095\r\n8096\r\n8097\r\n8098\r\n8099\r\n8100\r\n8101\r\n8102\r\n8103\r\n8104\r\n8105\r\n8106\r\n8107\r\n8108\r\n8109\r\n8110\r\n8111\r\n8112\r\n8113\r\n8114\r\n8115\r\n8116\r\n8117\r\n8118\r\n8119\r\n8120\r\n8121\r\n8122\r\n8123\r\n8124\r\n8125\r\n8126\r\n8127\r\n8128\r\n8129\r\n8130\r\n8131\r\n8132\r\n8133\r\n8134\r\n8135\r\n8136\r\n8137\r\n8138\r\n8139\r\n8140\r\n8141\r\n8142\r\n8143\r\n8144\r\n8145\r\n8146\r\n8147\r\n8148\r\n8149\r\n8150\r\n8151\r\n8152\r\n8153\r\n8154\r\n8155\r\n8156\r\n8157\r\n8158\r\n8159\r\n8160\r\n8161\r\n8162\r\n8163\r\n8164\r\n8165\r\n8166\r\n8167\r\n8168\r\n8169\r\n8170\r\n8171\r\n8172\r\n8173\r\n8174\r\n8175\r\n8176\r\n8177\r\n8178\r\n8179\r\n8180\r\n8181\r\n8182\r\n8183\r\n8184\r\n8185\r\n8186\r\n8187\r\n8188\r\n8189\r\n8190\r\n8191\r\n8192\r\n8193\r\n8194\r\n8195\r\n8196\r\n8197\r\n8198\r\n8199\r\n8200\r\n8201\r\n8202\r\n8203\r\n8204\r\n8205\r\n8206\r\n8207\r\n8208\r\n8209\r\n8210\r\n8211\r\n8212\r\n8213\r\n8214\r\n8215\r\n8216\r\n8217\r\n8218\r\n8219\r\n8220\r\n8221\r\n8222\r\n8223\r\n8224\r\n8225\r\n8226\r\n8227\r\n8228\r\n8229\r\n8230\r\n8231\r\n8232\r\n8233\r\n8234\r\n8235\r\n8236\r\n8237\r\n8238\r\n8239\r\n8240\r\n8241\r\n8242\r\n8243\r\n8244\r\n8245\r\n8246\r\n8247\r\n8248\r\n8249\r\n8250\r\n8251\r\n8252\r\n8253\r\n8254\r\n8255\r\n8256\r\n8257\r\n8258\r\n8259\r\n8260\r\n8261\r\n8262\r\n8263\r\n8264\r\n8265\r\n8266\r\n8267\r\n8268\r\n8269\r\n8270\r\n8271\r\n8272\r\n8273\r\n8274\r\n8275\r\n8276\r\n8277\r\n8278\r\n8279\r\n8280\r\n8281\r\n8282\r\n8283\r\n8284\r\n8285\r\n8286\r\n8287\r\n8288\r\n8289\r\n8290\r\n8291\r\n8292\r\n8293\r\n8294\r\n8295\r\n8296\r\n8297\r\n8298\r\n8299\r\n8300\r\n8301\r\n8302\r\n8303\r\n8304\r\n8305\r\n8306\r\n8307\r\n8308\r\n8309\r\n8310\r\n8311\r\n8312\r\n8313\r\n8314\r\n8315\r\n8316\r\n8317\r\n8318\r\n8319\r\n8320\r\n8321\r\n8322\r\n8323\r\n8324\r\n8325\r\n8326\r\n8327\r\n8328\r\n8329\r\n8330\r\n8331\r\n8332\r\n8333\r\n8334\r\n8335\r\n8336\r\n8337\r\n8338\r\n8339\r\n8340\r\n8341\r\n8342\r\n8343\r\n8344\r\n8345\r\n8346\r\n8347\r\n8348\r\n8349\r\n8350\r\n8351\r\n8352\r\n8353\r\n8354\r\n8355\r\n8356\r\n8357\r\n8358\r\n8359\r\n8360\r\n8361\r\n8362\r\n8363\r\n8364\r\n8365\r\n8366\r\n8367\r\n8368\r\n8369\r\n8370\r\n8371\r\n8372\r\n8373\r\n8374\r\n8375\r\n8376\r\n8377\r\n8378\r\n8379\r\n8380\r\n8381\r\n8382\r\n8383\r\n8384\r\n8385\r\n8386\r\n8387\r\n8388\r\n8389\r\n8390\r\n8391\r\n8392\r\n8393\r\n8394\r\n8395\r\n8396\r\n8397\r\n8398\r\n8399\r\n8400\r\n8401\r\n8402\r\n8403\r\n8404\r\n8405\r\n8406\r\n8407\r\n8408\r\n8409\r\n8410\r\n8411\r\n8412\r\n8413\r\n8414\r\n8415\r\n8416\r\n8417\r\n8418\r\n8419\r\n8420\r\n8421\r\n8422\r\n8423\r\n8424\r\n8425\r\n8426\r\n8427\r\n8428\r\n8429\r\n8430\r\n8431\r\n8432\r\n8433\r\n8434\r\n8435\r\n8436\r\n8437\r\n8438\r\n8439\r\n8440\r\n8441\r\n8442\r\n8443\r\n8444\r\n8445\r\n8446\r\n8447\r\n8448\r\n8449\r\n8450\r\n8451\r\n8452\r\n8453\r\n8454\r\n8455\r\n8456\r\n8457\r\n8458\r\n8459\r\n8460\r\n8461\r\n8462\r\n8463\r\n8464\r\n8465\r\n8466\r\n8467\r\n8468\r\n8469\r\n8470\r\n8471\r\n8472\r\n8473\r\n8474\r\n8475\r\n8476\r\n8477\r\n8478\r\n8479\r\n8480\r\n8481\r\n8482\r\n8483\r\n8484\r\n8485\r\n8486\r\n8487\r\n8488\r\n8489\r\n8490\r\n8491\r\n8492\r\n8493\r\n8494\r\n8495\r\n8496\r\n8497\r\n8498\r\n8499\r\n8500\r\n8501\r\n8502\r\n8503\r\n8504\r\n8505\r\n8506\r\n8507\r\n8508\r\n8509\r\n8510\r\n8511\r\n8512\r\n8513\r\n8514\r\n8515\r\n8516\r\n8517\r\n8518\r\n8519\r\n8520\r\n8521\r\n8522\r\n8523\r\n8524\r\n8525\r\n8526\r\n8527\r\n8528\r\n8529\r\n8530\r\n8531\r\n8532\r\n8533\r\n8534\r\n8535\r\n8536\r\n8537\r\n8538\r\n8539\r\n8540\r\n8541\r\n8542\r\n8543\r\n8544\r\n8545\r\n8546\r\n8547\r\n8548\r\n8549\r\n8550\r\n8551\r\n8552\r\n8553\r\n8554\r\n8555\r\n8556\r\n8557\r\n8558\r\n8559\r\n8560\r\n8561\r\n8562\r\n8563\r\n8564\r\n8565\r\n8566\r\n8567\r\n8568\r\n8569\r\n8570\r\n8571\r\n8572\r\n8573\r\n8574\r\n8575\r\n8576\r\n8577\r\n8578\r\n8579\r\n8580\r\n8581\r\n8582\r\n8583\r\n8584\r\n8585\r\n8586\r\n8587\r\n8588\r\n8589\r\n8590\r\n8591\r\n8592\r\n8593\r\n8594\r\n8595\r\n8596\r\n8597\r\n8598\r\n8599\r\n8600\r\n8601\r\n8602\r\n8603\r\n8604\r\n8605\r\n8606\r\n8607\r\n8608\r\n8609\r\n8610\r\n8611\r\n8612\r\n8613\r\n8614\r\n8615\r\n8616\r\n8617\r\n8618\r\n8619\r\n8620\r\n8621\r\n8622\r\n8623\r\n8624\r\n8625\r\n8626\r\n8627\r\n8628\r\n8629\r\n8630\r\n8631\r\n8632\r\n8633\r\n8634\r\n8635\r\n8636\r\n8637\r\n8638\r\n8639\r\n8640\r\n8641\r\n8642\r\n8643\r\n8644\r\n8645\r\n8646\r\n8647\r\n8648\r\n8649\r\n8650\r\n8651\r\n8652\r\n8653\r\n8654\r\n8655\r\n8656\r\n8657\r\n8658\r\n8659\r\n8660\r\n8661\r\n8662\r\n8663\r\n8664\r\n8665\r\n8666\r\n8667\r\n8668\r\n8669\r\n8670\r\n8671\r\n8672\r\n8673\r\n8674\r\n8675\r\n8676\r\n8677\r\n8678\r\n8679\r\n8680\r\n8681\r\n8682\r\n8683\r\n8684\r\n8685\r\n8686\r\n8687\r\n8688\r\n8689\r\n8690\r\n8691\r\n8692\r\n8693\r\n8694\r\n8695\r\n8696\r\n8697\r\n8698\r\n8699\r\n8700\r\n8701\r\n8702\r\n8703\r\n8704\r\n8705\r\n8706\r\n8707\r\n8708\r\n8709\r\n8710\r\n8711\r\n8712\r\n8713\r\n8714\r\n8715\r\n8716\r\n8717\r\n8718\r\n8719\r\n8720\r\n8721\r\n8722\r\n8723\r\n8724\r\n8725\r\n8726\r\n8727\r\n8728\r\n8729\r\n8730\r\n8731\r\n8732\r\n8733\r\n8734\r\n8735\r\n8736\r\n8737\r\n8738\r\n8739\r\n8740\r\n8741\r\n8742\r\n8743\r\n8744\r\n8745\r\n8746\r\n8747\r\n8748\r\n8749\r\n8750\r\n8751\r\n8752\r\n8753\r\n8754\r\n8755\r\n8756\r\n8757\r\n8758\r\n8759\r\n8760\r\n8761\r\n8762\r\n8763\r\n8764\r\n8765\r\n8766\r\n8767\r\n8768\r\n8769\r\n8770\r\n8771\r\n8772\r\n8773\r\n8774\r\n8775\r\n8776\r\n8777\r\n8778\r\n8779\r\n8780\r\n8781\r\n8782\r\n8783\r\n8784\r\n8785\r\n8786\r\n8787\r\n8788\r\n8789\r\n8790\r\n8791\r\n8792\r\n8793\r\n8794\r\n8795\r\n8796\r\n8797\r\n8798\r\n8799\r\n8800\r\n8801\r\n8802\r\n8803\r\n8804\r\n8805\r\n8806\r\n8807\r\n8808\r\n8809\r\n8810\r\n8811\r\n8812\r\n8813\r\n8814\r\n8815\r\n8816\r\n8817\r\n8818\r\n8819\r\n8820\r\n8821\r\n8822\r\n8823\r\n8824\r\n8825\r\n8826\r\n8827\r\n8828\r\n8829\r\n8830\r\n8831\r\n8832\r\n8833\r\n8834\r\n8835\r\n8836\r\n8837\r\n8838\r\n8839\r\n8840\r\n8841\r\n8842\r\n8843\r\n8844\r\n8845\r\n8846\r\n8847\r\n8848\r\n8849\r\n8850\r\n8851\r\n8852\r\n8853\r\n8854\r\n8855\r\n8856\r\n8857\r\n8858\r\n8859\r\n8860\r\n8861\r\n8862\r\n8863\r\n8864\r\n8865\r\n8866\r\n8867\r\n8868\r\n8869\r\n8870\r\n8871\r\n8872\r\n8873\r\n8874\r\n8875\r\n8876\r\n8877\r\n8878\r\n8879\r\n8880\r\n8881\r\n8882\r\n8883\r\n8884\r\n8885\r\n8886\r\n8887\r\n8888\r\n8889\r\n8890\r\n8891\r\n8892\r\n8893\r\n8894\r\n8895\r\n8896\r\n8897\r\n8898\r\n8899\r\n8900\r\n8901\r\n8902\r\n8903\r\n8904\r\n8905\r\n8906\r\n8907\r\n8908\r\n8909\r\n8910\r\n8911\r\n8912\r\n8913\r\n8914\r\n8915\r\n8916\r\n8917\r\n8918\r\n8919\r\n8920\r\n8921\r\n8922\r\n8923\r\n8924\r\n8925\r\n8926\r\n8927\r\n8928\r\n8929\r\n8930\r\n8931\r\n8932\r\n8933\r\n8934\r\n8935\r\n8936\r\n8937\r\n8938\r\n8939\r\n8940\r\n8941\r\n8942\r\n8943\r\n8944\r\n8945\r\n8946\r\n8947\r\n8948\r\n8949\r\n8950\r\n8951\r\n8952\r\n8953\r\n8954\r\n8955\r\n8956\r\n8957\r\n8958\r\n8959\r\n8960\r\n8961\r\n8962\r\n8963\r\n8964\r\n8965\r\n8966\r\n8967\r\n8968\r\n8969\r\n8970\r\n8971\r\n8972\r\n8973\r\n8974\r\n8975\r\n8976\r\n8977\r\n8978\r\n8979\r\n8980\r\n8981\r\n8982\r\n8983\r\n8984\r\n8985\r\n8986\r\n8987\r\n8988\r\n8989\r\n8990\r\n8991\r\n8992\r\n8993\r\n8994\r\n8995\r\n8996\r\n8997\r\n8998\r\n8999\r\n9000\r\n9001\r\n9002\r\n9003\r\n9004\r\n9005\r\n9006\r\n9007\r\n9008\r\n9009\r\n9010\r\n9011\r\n9012\r\n9013\r\n9014\r\n9015\r\n9016\r\n9017\r\n9018\r\n9019\r\n9020\r\n9021\r\n9022\r\n9023\r\n9024\r\n9025\r\n9026\r\n9027\r\n9028\r\n9029\r\n9030\r\n9031\r\n9032\r\n9033\r\n9034\r\n9035\r\n9036\r\n9037\r\n9038\r\n9039\r\n9040\r\n9041\r\n9042\r\n9043\r\n9044\r\n9045\r\n9046\r\n9047\r\n9048\r\n9049\r\n9050\r\n9051\r\n9052\r\n9053\r\n9054\r\n9055\r\n9056\r\n9057\r\n9058\r\n9059\r\n9060\r\n9061\r\n9062\r\n9063\r\n9064\r\n9065\r\n9066\r\n9067\r\n9068\r\n9069\r\n9070\r\n9071\r\n9072\r\n9073\r\n9074\r\n9075\r\n9076\r\n9077\r\n9078\r\n9079\r\n9080\r\n9081\r\n9082\r\n9083\r\n9084\r\n9085\r\n9086\r\n9087\r\n9088\r\n9089\r\n9090\r\n9091\r\n9092\r\n9093\r\n9094\r\n9095\r\n9096\r\n9097\r\n9098\r\n9099\r\n9100\r\n9101\r\n9102\r\n9103\r\n9104\r\n9105\r\n9106\r\n9107\r\n9108\r\n9109\r\n9110\r\n9111\r\n9112\r\n9113\r\n9114\r\n9115\r\n9116\r\n9117\r\n9118\r\n9119\r\n9120\r\n9121\r\n9122\r\n9123\r\n9124\r\n9125\r\n9126\r\n9127\r\n9128\r\n9129\r\n9130\r\n9131\r\n9132\r\n9133\r\n9134\r\n9135\r\n9136\r\n9137\r\n9138\r\n9139\r\n9140\r\n9141\r\n9142\r\n9143\r\n9144\r\n9145\r\n9146\r\n9147\r\n9148\r\n9149\r\n9150\r\n9151\r\n9152\r\n9153\r\n9154\r\n9155\r\n9156\r\n9157\r\n9158\r\n9159\r\n9160\r\n9161\r\n9162\r\n9163\r\n9164\r\n9165\r\n9166\r\n9167\r\n9168\r\n9169\r\n9170\r\n9171\r\n9172\r\n9173\r\n9174\r\n9175\r\n9176\r\n9177\r\n9178\r\n9179\r\n9180\r\n9181\r\n9182\r\n9183\r\n9184\r\n9185\r\n9186\r\n9187\r\n9188\r\n9189\r\n9190\r\n9191\r\n9192\r\n9193\r\n9194\r\n9195\r\n9196\r\n9197\r\n9198\r\n9199\r\n9200\r\n9201\r\n9202\r\n9203\r\n9204\r\n9205\r\n9206\r\n9207\r\n9208\r\n9209\r\n9210\r\n9211\r\n9212\r\n9213\r\n9214\r\n9215\r\n9216\r\n9217\r\n9218\r\n9219\r\n9220\r\n9221\r\n9222\r\n9223\r\n9224\r\n9225\r\n9226\r\n9227\r\n9228\r\n9229\r\n9230\r\n9231\r\n9232\r\n9233\r\n9234\r\n9235\r\n9236\r\n9237\r\n9238\r\n9239\r\n9240\r\n9241\r\n9242\r\n9243\r\n9244\r\n9245\r\n9246\r\n9247\r\n9248\r\n9249\r\n9250\r\n9251\r\n9252\r\n9253\r\n9254\r\n9255\r\n9256\r\n9257\r\n9258\r\n9259\r\n9260\r\n9261\r\n9262\r\n9263\r\n9264\r\n9265\r\n9266\r\n9267\r\n9268\r\n9269\r\n9270\r\n9271\r\n9272\r\n9273\r\n9274\r\n9275\r\n9276\r\n9277\r\n9278\r\n9279\r\n9280\r\n9281\r\n9282\r\n9283\r\n9284\r\n9285\r\n9286\r\n9287\r\n9288\r\n9289\r\n9290\r\n9291\r\n9292\r\n9293\r\n9294\r\n9295\r\n9296\r\n9297\r\n9298\r\n9299\r\n9300\r\n9301\r\n9302\r\n9303\r\n9304\r\n9305\r\n9306\r\n9307\r\n9308\r\n9309\r\n9310\r\n9311\r\n9312\r\n9313\r\n9314\r\n9315\r\n9316\r\n9317\r\n9318\r\n9319\r\n9320\r\n9321\r\n9322\r\n9323\r\n9324\r\n9325\r\n9326\r\n9327\r\n9328\r\n9329\r\n9330\r\n9331\r\n9332\r\n9333\r\n9334\r\n9335\r\n9336\r\n9337\r\n9338\r\n9339\r\n9340\r\n9341\r\n9342\r\n9343\r\n9344\r\n9345\r\n9346\r\n9347\r\n9348\r\n9349\r\n9350\r\n9351\r\n9352\r\n9353\r\n9354\r\n9355\r\n9356\r\n9357\r\n9358\r\n9359\r\n9360\r\n9361\r\n9362\r\n9363\r\n9364\r\n9365\r\n9366\r\n9367\r\n9368\r\n9369\r\n9370\r\n9371\r\n9372\r\n9373\r\n9374\r\n9375\r\n9376\r\n9377\r\n9378\r\n9379\r\n9380\r\n9381\r\n9382\r\n9383\r\n9384\r\n9385\r\n9386\r\n9387\r\n9388\r\n9389\r\n9390\r\n9391\r\n9392\r\n9393\r\n9394\r\n9395\r\n9396\r\n9397\r\n9398\r\n9399\r\n9400\r\n9401\r\n9402\r\n9403\r\n9404\r\n9405\r\n9406\r\n9407\r\n9408\r\n9409\r\n9410\r\n9411\r\n9412\r\n9413\r\n9414\r\n9415\r\n9416\r\n9417\r\n9418\r\n9419\r\n9420\r\n9421\r\n9422\r\n9423\r\n9424\r\n9425\r\n9426\r\n9427\r\n9428\r\n9429\r\n9430\r\n9431\r\n9432\r\n9433\r\n9434\r\n9435\r\n9436\r\n9437\r\n9438\r\n9439\r\n9440\r\n9441\r\n9442\r\n9443\r\n9444\r\n9445\r\n9446\r\n9447\r\n9448\r\n9449\r\n9450\r\n9451\r\n9452\r\n9453\r\n9454\r\n9455\r\n9456\r\n9457\r\n9458\r\n9459\r\n9460\r\n9461\r\n9462\r\n9463\r\n9464\r\n9465\r\n9466\r\n9467\r\n9468\r\n9469\r\n9470\r\n9471\r\n9472\r\n9473\r\n9474\r\n9475\r\n9476\r\n9477\r\n9478\r\n9479\r\n9480\r\n9481\r\n9482\r\n9483\r\n9484\r\n9485\r\n9486\r\n9487\r\n9488\r\n9489\r\n9490\r\n9491\r\n9492\r\n9493\r\n9494\r\n9495\r\n9496\r\n9497\r\n9498\r\n9499\r\n9500\r\n9501\r\n9502\r\n9503\r\n9504\r\n9505\r\n9506\r\n9507\r\n9508\r\n9509\r\n9510\r\n9511\r\n9512\r\n9513\r\n9514\r\n9515\r\n9516\r\n9517\r\n9518\r\n9519\r\n9520\r\n9521\r\n9522\r\n9523\r\n9524\r\n9525\r\n9526\r\n9527\r\n9528\r\n9529\r\n9530\r\n9531\r\n9532\r\n9533\r\n9534\r\n9535\r\n9536\r\n9537\r\n9538\r\n9539\r\n9540\r\n9541\r\n9542\r\n9543\r\n9544\r\n9545\r\n9546\r\n9547\r\n9548\r\n9549\r\n9550\r\n9551\r\n9552\r\n9553\r\n9554\r\n9555\r\n9556\r\n9557\r\n9558\r\n9559\r\n9560\r\n9561\r\n9562\r\n9563\r\n9564\r\n9565\r\n9566\r\n9567\r\n9568\r\n9569\r\n9570\r\n9571\r\n9572\r\n9573\r\n9574\r\n9575\r\n9576\r\n9577\r\n9578\r\n9579\r\n9580\r\n9581\r\n9582\r\n9583\r\n9584\r\n9585\r\n9586\r\n9587\r\n9588\r\n9589\r\n9590\r\n9591\r\n9592\r\n9593\r\n9594\r\n9595\r\n9596\r\n9597\r\n9598\r\n9599\r\n9600\r\n9601\r\n9602\r\n9603\r\n9604\r\n9605\r\n9606\r\n9607\r\n9608\r\n9609\r\n9610\r\n9611\r\n9612\r\n9613\r\n9614\r\n9615\r\n9616\r\n9617\r\n9618\r\n9619\r\n9620\r\n9621\r\n9622\r\n9623\r\n9624\r\n9625\r\n9626\r\n9627\r\n9628\r\n9629\r\n9630\r\n9631\r\n9632\r\n9633\r\n9634\r\n9635\r\n9636\r\n9637\r\n9638\r\n9639\r\n9640\r\n9641\r\n9642\r\n9643\r\n9644\r\n9645\r\n9646\r\n9647\r\n9648\r\n9649\r\n9650\r\n9651\r\n9652\r\n9653\r\n9654\r\n9655\r\n9656\r\n9657\r\n9658\r\n9659\r\n9660\r\n9661\r\n9662\r\n9663\r\n9664\r\n9665\r\n9666\r\n9667\r\n9668\r\n9669\r\n9670\r\n9671\r\n9672\r\n9673\r\n9674\r\n9675\r\n9676\r\n9677\r\n9678\r\n9679\r\n9680\r\n9681\r\n9682\r\n9683\r\n9684\r\n9685\r\n9686\r\n9687\r\n9688\r\n9689\r\n9690\r\n9691\r\n9692\r\n9693\r\n9694\r\n9695\r\n9696\r\n9697\r\n9698\r\n9699\r\n9700\r\n9701\r\n9702\r\n9703\r\n9704\r\n9705\r\n9706\r\n9707\r\n9708\r\n9709\r\n9710\r\n9711\r\n9712\r\n9713\r\n9714\r\n9715\r\n9716\r\n9717\r\n9718\r\n9719\r\n9720\r\n9721\r\n9722\r\n9723\r\n9724\r\n9725\r\n9726\r\n9727\r\n9728\r\n9729\r\n9730\r\n9731\r\n9732\r\n9733\r\n9734\r\n9735\r\n9736\r\n9737\r\n9738\r\n9739\r\n9740\r\n9741\r\n9742\r\n9743\r\n9744\r\n9745\r\n9746\r\n9747\r\n9748\r\n9749\r\n9750\r\n9751\r\n9752\r\n9753\r\n9754\r\n9755\r\n9756\r\n9757\r\n9758\r\n9759\r\n9760\r\n9761\r\n9762\r\n9763\r\n9764\r\n9765\r\n9766\r\n9767\r\n9768\r\n9769\r\n9770\r\n9771\r\n9772\r\n9773\r\n9774\r\n9775\r\n9776\r\n9777\r\n9778\r\n9779\r\n9780\r\n9781\r\n9782\r\n9783\r\n9784\r\n9785\r\n9786\r\n9787\r\n9788\r\n9789\r\n9790\r\n9791\r\n9792\r\n9793\r\n9794\r\n9795\r\n9796\r\n9797\r\n9798\r\n9799\r\n9800\r\n9801\r\n9802\r\n9803\r\n9804\r\n9805\r\n9806\r\n9807\r\n9808\r\n9809\r\n9810\r\n9811\r\n9812\r\n9813\r\n9814\r\n9815\r\n9816\r\n9817\r\n9818\r\n9819\r\n9820\r\n9821\r\n9822\r\n9823\r\n9824\r\n9825\r\n9826\r\n9827\r\n9828\r\n9829\r\n9830\r\n9831\r\n9832\r\n9833\r\n9834\r\n9835\r\n9836\r\n9837\r\n9838\r\n9839\r\n9840\r\n9841\r\n9842\r\n9843\r\n9844\r\n9845\r\n9846\r\n9847\r\n9848\r\n9849\r\n9850\r\n9851\r\n9852\r\n9853\r\n9854\r\n9855\r\n9856\r\n9857\r\n9858\r\n9859\r\n9860\r\n9861\r\n9862\r\n9863\r\n9864\r\n9865\r\n9866\r\n9867\r\n9868\r\n9869\r\n9870\r\n9871\r\n9872\r\n9873\r\n9874\r\n9875\r\n9876\r\n9877\r\n9878\r\n9879\r\n9880\r\n9881\r\n9882\r\n9883\r\n9884\r\n9885\r\n9886\r\n9887\r\n9888\r\n9889\r\n9890\r\n9891\r\n9892\r\n9893\r\n9894\r\n9895\r\n9896\r\n9897\r\n9898\r\n9899\r\n9900\r\n9901\r\n9902\r\n9903\r\n9904\r\n9905\r\n9906\r\n9907\r\n9908\r\n9909\r\n9910\r\n9911\r\n9912\r\n9913\r\n9914\r\n9915\r\n9916\r\n9917\r\n9918\r\n9919\r\n9920\r\n9921\r\n9922\r\n9923\r\n9924\r\n9925\r\n9926\r\n9927\r\n9928\r\n9929\r\n9930\r\n9931\r\n9932\r\n9933\r\n9934\r\n9935\r\n9936\r\n9937\r\n9938\r\n9939\r\n9940\r\n9941\r\n9942\r\n9943\r\n9944\r\n9945\r\n9946\r\n9947\r\n9948\r\n9949\r\n9950\r\n9951\r\n9952\r\n9953\r\n9954\r\n9955\r\n9956\r\n9957\r\n9958\r\n9959\r\n9960\r\n9961\r\n9962\r\n9963\r\n9964\r\n9965\r\n9966\r\n9967\r\n9968\r\n9969\r\n9970\r\n9971\r\n9972\r\n9973\r\n9974\r\n9975\r\n9976\r\n9977\r\n9978\r\n9979\r\n9980\r\n9981\r\n9982\r\n9983\r\n9984\r\n9985\r\n9986\r\n9987\r\n9988\r\n9989\r\n9990\r\n9991\r\n9992\r\n9993\r\n9994\r\n9995\r\n9996\r\n9997\r\n9998\r\n9999\r\n10000\r\n10001\r\n10002\r\n10003\r\n10004\r\n10005\r\n10006\r\n10007\r\n10008\r\n10009\r\n10010\r\n10011\r\n10012\r\n10013\r\n10014\r\n10015\r\n10016\r\n10017\r\n10018\r\n10019\r\n10020\r\n10021\r\n10022\r\n10023\r\n10024\r\n10025\r\n10026\r\n10027\r\n10028\r\n10029\r\n10030\r\n10031\r\n10032\r\n10033\r\n10034\r\n10035\r\n10036\r\n10037\r\n10038\r\n10039\r\n10040\r\n10041\r\n10042\r\n10043\r\n10044\r\n10045\r\n10046\r\n10047\r\n10048\r\n10049\r\n10050\r\n10051\r\n10052\r\n10053\r\n10054\r\n10055\r\n10056\r\n10057\r\n10058\r\n10059\r\n10060\r\n10061\r\n10062\r\n10063\r\n10064\r\n10065\r\n10066\r\n10067\r\n10068\r\n10069\r\n10070\r\n10071\r\n10072\r\n10073\r\n10074\r\n10075\r\n10076\r\n10077\r\n10078\r\n10079\r\n10080\r\n10081\r\n10082\r\n10083\r\n10084\r\n10085\r\n10086\r\n10087\r\n10088\r\n10089\r\n10090\r\n10091\r\n10092\r\n10093\r\n10094\r\n10095\r\n10096\r\n10097\r\n10098\r\n10099\r\n10100\r\n10101\r\n10102\r\n10103\r\n10104\r\n10105\r\n10106\r\n10107\r\n10108\r\n10109\r\n10110\r\n10111\r\n10112\r\n10113\r\n10114\r\n10115\r\n10116\r\n10117\r\n10118\r\n10119\r\n10120\r\n10121\r\n10122\r\n10123\r\n10124\r\n10125\r\n10126\r\n10127\r\n10128\r\n10129\r\n10130\r\n10131\r\n10132\r\n10133\r\n10134\r\n10135\r\n10136\r\n10137\r\n10138\r\n10139\r\n10140\r\n10141\r\n10142\r\n10143\r\n10144\r\n10145\r\n10146\r\n10147\r\n10148\r\n10149\r\n10150\r\n10151\r\n10152\r\n10153\r\n10154\r\n10155\r\n10156\r\n10157\r\n10158\r\n10159\r\n10160\r\n10161\r\n10162\r\n10163\r\n10164\r\n10165\r\n10166\r\n10167\r\n10168\r\n10169\r\n10170\r\n10171\r\n10172\r\n10173\r\n10174\r\n10175\r\n10176\r\n10177\r\n10178\r\n10179\r\n10180\r\n10181\r\n10182\r\n10183\r\n10184\r\n10185\r\n10186\r\n10187\r\n10188\r\n10189\r\n10190\r\n10191\r\n10192\r\n10193\r\n10194\r\n10195\r\n10196\r\n10197\r\n10198\r\n10199\r\n10200\r\n10201\r\n10202\r\n10203\r\n10204\r\n10205\r\n10206\r\n10207\r\n10208\r\n10209\r\n10210\r\n10211\r\n10212\r\n10213\r\n10214\r\n10215\r\n10216\r\n10217\r\n10218\r\n10219\r\n10220\r\n10221\r\n10222\r\n10223\r\n10224\r\n10225\r\n10226\r\n10227\r\n10228\r\n10229\r\n10230\r\n10231\r\n10232\r\n10233\r\n10234\r\n10235\r\n10236\r\n10237\r\n10238\r\n10239\r\n10240\r\n10241\r\n10242\r\n10243\r\n10244\r\n10245\r\n10246\r\n10247\r\n10248\r\n10249\r\n10250\r\n10251\r\n10252\r\n10253\r\n10254\r\n10255\r\n10256\r\n10257\r\n10258\r\n10259\r\n10260\r\n10261\r\n10262\r\n10263\r\n10264\r\n10265\r\n10266\r\n10267\r\n10268\r\n10269\r\n10270\r\n10271\r\n10272\r\n10273\r\n10274\r\n10275\r\n10276\r\n10277\r\n10278\r\n10279\r\n10280\r\n10281\r\n10282\r\n10283\r\n10284\r\n10285\r\n10286\r\n10287\r\n10288\r\n10289\r\n10290\r\n10291\r\n10292\r\n10293\r\n10294\r\n10295\r\n10296\r\n10297\r\n10298\r\n10299\r\n10300\r\n10301\r\n10302\r\n10303\r\n10304\r\n10305\r\n10306\r\n10307\r\n10308\r\n10309\r\n10310\r\n10311\r\n10312\r\n10313\r\n10314\r\n10315\r\n10316\r\n10317\r\n10318\r\n10319\r\n10320\r\n10321\r\n10322\r\n10323\r\n10324\r\n10325\r\n10326\r\n10327\r\n10328\r\n10329\r\n10330\r\n10331\r\n10332\r\n10333\r\n10334\r\n10335\r\n10336\r\n10337\r\n10338\r\n10339\r\n10340\r\n10341\r\n10342\r\n10343\r\n10344\r\n10345\r\n10346\r\n10347\r\n10348\r\n10349\r\n10350\r\n10351\r\n10352\r\n10353\r\n10354\r\n10355\r\n10356\r\n10357\r\n10358\r\n10359\r\n10360\r\n10361\r\n10362\r\n10363\r\n10364\r\n10365\r\n10366\r\n10367\r\n10368\r\n10369\r\n10370\r\n10371\r\n10372\r\n10373\r\n10374\r\n10375\r\n10376\r\n10377\r\n10378\r\n10379\r\n10380\r\n10381\r\n10382\r\n10383\r\n10384\r\n10385\r\n10386\r\n10387\r\n10388\r\n10389\r\n10390\r\n10391\r\n10392\r\n10393\r\n10394\r\n10395\r\n10396\r\n10397\r\n10398\r\n10399\r\n10400\r\n10401\r\n10402\r\n10403\r\n10404\r\n10405\r\n10406\r\n10407\r\n10408\r\n10409\r\n10410\r\n10411\r\n10412\r\n10413\r\n10414\r\n10415\r\n10416\r\n10417\r\n10418\r\n10419\r\n10420\r\n10421\r\n10422\r\n10423\r\n10424\r\n10425\r\n10426\r\n10427\r\n10428\r\n10429\r\n10430\r\n10431\r\n10432\r\n10433\r\n10434\r\n10435\r\n10436\r\n10437\r\n10438\r\n10439\r\n10440\r\n10441\r\n10442\r\n10443\r\n10444\r\n10445\r\n10446\r\n10447\r\n10448\r\n10449\r\n10450\r\n10451\r\n10452\r\n10453\r\n10454\r\n10455\r\n10456\r\n10457\r\n10458\r\n10459\r\n10460\r\n10461\r\n10462\r\n10463\r\n10464\r\n10465\r\n10466\r\n10467\r\n10468\r\n10469\r\n10470\r\n10471\r\n10472\r\n10473\r\n10474\r\n10475\r\n10476\r\n10477\r\n10478\r\n10479\r\n10480\r\n10481\r\n10482\r\n10483\r\n10484\r\n10485\r\n10486\r\n10487\r\n10488\r\n10489\r\n10490\r\n10491\r\n10492\r\n10493\r\n10494\r\n10495\r\n10496\r\n10497\r\n10498\r\n10499\r\n10500\r\n10501\r\n10502\r\n10503\r\n10504\r\n10505\r\n10506\r\n10507\r\n10508\r\n10509\r\n10510\r\n10511\r\n10512\r\n10513\r\n10514\r\n10515\r\n10516\r\n10517\r\n10518\r\n10519\r\n10520\r\n10521\r\n10522\r\n10523\r\n10524\r\n10525\r\n10526\r\n10527\r\n10528\r\n10529\r\n10530\r\n10531\r\n10532\r\n10533\r\n10534\r\n10535\r\n10536\r\n10537\r\n10538\r\n10539\r\n10540\r\n10541\r\n10542\r\n10543\r\n10544\r\n10545\r\n10546\r\n10547\r\n10548\r\n10549\r\n10550\r\n10551\r\n10552\r\n10553\r\n10554\r\n10555\r\n10556\r\n10557\r\n10558\r\n10559\r\n10560\r\n10561\r\n10562\r\n10563\r\n10564\r\n10565\r\n10566\r\n10567\r\n10568\r\n10569\r\n10570\r\n10571\r\n10572\r\n10573\r\n10574\r\n10575\r\n10576\r\n10577\r\n10578\r\n10579\r\n10580\r\n10581\r\n10582\r\n10583\r\n10584\r\n10585\r\n10586\r\n10587\r\n10588\r\n10589\r\n10590\r\n10591\r\n10592\r\n10593\r\n10594\r\n10595\r\n10596\r\n10597\r\n10598\r\n10599\r\n10600\r\n10601\r\n10602\r\n10603\r\n10604\r\n10605\r\n10606\r\n10607\r\n10608\r\n10609\r\n10610\r\n10611\r\n10612\r\n10613\r\n10614\r\n10615\r\n10616\r\n10617\r\n10618\r\n10619\r\n10620\r\n10621\r\n10622\r\n10623\r\n10624\r\n10625\r\n10626\r\n10627\r\n10628\r\n10629\r\n10630\r\n10631\r\n10632\r\n10633\r\n10634\r\n10635\r\n10636\r\n10637\r\n10638\r\n10639\r\n10640\r\n10641\r\n10642\r\n10643\r\n10644\r\n10645\r\n10646\r\n10647\r\n10648\r\n10649\r\n10650\r\n10651\r\n10652\r\n10653\r\n10654\r\n10655\r\n10656\r\n10657\r\n10658\r\n10659\r\n10660\r\n10661\r\n10662\r\n10663\r\n10664\r\n10665\r\n10666\r\n10667\r\n10668\r\n10669\r\n10670\r\n10671\r\n10672\r\n10673\r\n10674\r\n10675\r\n10676\r\n10677\r\n10678\r\n10679\r\n10680\r\n10681\r\n10682\r\n10683\r\n10684\r\n10685\r\n10686\r\n10687\r\n10688\r\n10689\r\n10690\r\n10691\r\n10692\r\n10693\r\n10694\r\n10695\r\n10696\r\n10697\r\n10698\r\n10699\r\n10700\r\n10701\r\n10702\r\n10703\r\n10704\r\n10705\r\n10706\r\n10707\r\n10708\r\n10709\r\n10710\r\n10711\r\n10712\r\n10713\r\n10714\r\n10715\r\n10716\r\n10717\r\n10718\r\n10719\r\n10720\r\n10721\r\n10722\r\n10723\r\n10724\r\n10725\r\n10726\r\n10727\r\n10728\r\n10729\r\n10730\r\n10731\r\n10732\r\n10733\r\n10734\r\n10735\r\n10736\r\n10737\r\n10738\r\n10739\r\n10740\r\n10741\r\n10742\r\n10743\r\n10744\r\n10745\r\n10746\r\n10747\r\n10748\r\n10749\r\n10750\r\n10751\r\n10752\r\n10753\r\n10754\r\n10755\r\n10756\r\n10757\r\n10758\r\n10759\r\n10760\r\n10761\r\n10762\r\n10763\r\n10764\r\n10765\r\n10766\r\n10767\r\n10768\r\n10769\r\n10770\r\n10771\r\n10772\r\n10773\r\n10774\r\n10775\r\n10776\r\n10777\r\n10778\r\n10779\r\n10780\r\n10781\r\n10782\r\n10783\r\n10784\r\n10785\r\n10786\r\n10787\r\n10788\r\n10789\r\n10790\r\n10791\r\n10792\r\n10793\r\n10794\r\n10795\r\n10796\r\n10797\r\n10798\r\n10799\r\n10800\r\n10801\r\n10802\r\n10803\r\n10804\r\n10805\r\n10806\r\n10807\r\n10808\r\n10809\r\n10810\r\n10811\r\n10812\r\n10813\r\n10814\r\n10815\r\n10816\r\n10817\r\n10818\r\n10819\r\n10820\r\n10821\r\n10822\r\n10823\r\n10824\r\n10825\r\n10826\r\n10827\r\n10828\r\n10829\r\n10830\r\n10831\r\n10832\r\n10833\r\n10834\r\n10835\r\n10836\r\n10837\r\n10838\r\n10839\r\n10840\r\n10841\r\n10842\r\n10843\r\n10844\r\n10845\r\n10846\r\n10847\r\n10848\r\n10849\r\n10850\r\n10851\r\n10852\r\n10853\r\n10854\r\n10855\r\n10856\r\n10857\r\n10858\r\n10859\r\n10860\r\n10861\r\n10862\r\n10863\r\n10864\r\n10865\r\n10866\r\n10867\r\n10868\r\n10869\r\n10870\r\n10871\r\n10872\r\n10873\r\n10874\r\n10875\r\n10876\r\n10877\r\n10878\r\n10879\r\n10880\r\n10881\r\n10882\r\n10883\r\n10884\r\n10885\r\n10886\r\n10887\r\n10888\r\n10889\r\n10890\r\n10891\r\n10892\r\n10893\r\n10894\r\n10895\r\n10896\r\n10897\r\n10898\r\n10899\r\n10900\r\n10901\r\n10902\r\n10903\r\n10904\r\n10905\r\n10906\r\n10907\r\n10908\r\n10909\r\n10910\r\n10911\r\n10912\r\n10913\r\n10914\r\n10915\r\n10916\r\n10917\r\n10918\r\n10919\r\n10920\r\n10921\r\n10922\r\n10923\r\n10924\r\n10925\r\n10926\r\n10927\r\n10928\r\n10929\r\n10930\r\n10931\r\n10932\r\n10933\r\n10934\r\n10935\r\n10936\r\n10937\r\n10938\r\n10939\r\n10940\r\n10941\r\n10942\r\n10943\r\n10944\r\n10945\r\n10946\r\n10947\r\n10948\r\n10949\r\n10950\r\n10951\r\n10952\r\n10953\r\n10954\r\n10955\r\n10956\r\n10957\r\n10958\r\n10959\r\n10960\r\n10961\r\n10962\r\n10963\r\n10964\r\n10965\r\n10966\r\n10967\r\n10968\r\n10969\r\n10970\r\n10971\r\n10972\r\n10973\r\n10974\r\n10975\r\n10976\r\n10977\r\n10978\r\n10979\r\n10980\r\n10981\r\n10982\r\n10983\r\n10984\r\n10985\r\n10986\r\n10987\r\n10988\r\n10989\r\n10990\r\n10991\r\n10992\r\n10993\r\n10994\r\n10995\r\n10996\r\n10997\r\n10998\r\n10999\r\n11000\r\n11001\r\n11002\r\n11003\r\n11004\r\n11005\r\n11006\r\n11007\r\n11008\r\n11009\r\n11010\r\n11011\r\n11012\r\n11013\r\n11014\r\n11015\r\n11016\r\n11017\r\n11018\r\n11019\r\n11020\r\n11021\r\n11022\r\n11023\r\n11024\r\n11025\r\n11026\r\n11027\r\n11028\r\n11029\r\n11030\r\n11031\r\n11032\r\n11033\r\n11034\r\n11035\r\n11036\r\n11037\r\n11038\r\n11039\r\n11040\r\n11041\r\n11042\r\n11043\r\n11044\r\n11045\r\n11046\r\n11047\r\n11048\r\n11049\r\n11050\r\n11051\r\n11052\r\n11053\r\n11054\r\n11055\r\n11056\r\n11057\r\n11058\r\n11059\r\n11060\r\n11061\r\n11062\r\n11063\r\n11064\r\n11065\r\n11066\r\n11067\r\n11068\r\n11069\r\n11070\r\n11071\r\n11072\r\n11073\r\n11074\r\n11075\r\n11076\r\n11077\r\n11078\r\n11079\r\n11080\r\n11081\r\n11082\r\n11083\r\n11084\r\n11085\r\n11086\r\n11087\r\n11088\r\n11089\r\n11090\r\n11091\r\n11092\r\n11093\r\n11094\r\n11095\r\n11096\r\n11097\r\n11098\r\n11099\r\n11100\r\n11101\r\n11102\r\n11103\r\n11104\r\n11105\r\n11106\r\n11107\r\n11108\r\n11109\r\n11110\r\n11111\r\n11112\r\n11113\r\n11114\r\n11115\r\n11116\r\n11117\r\n11118\r\n11119\r\n11120\r\n11121\r\n11122\r\n11123\r\n11124\r\n11125\r\n11126\r\n11127\r\n11128\r\n11129\r\n11130\r\n11131\r\n11132\r\n11133\r\n11134\r\n11135\r\n11136\r\n11137\r\n11138\r\n11139\r\n11140\r\n11141\r\n11142\r\n11143\r\n11144\r\n11145\r\n11146\r\n11147\r\n11148\r\n11149\r\n11150\r\n11151\r\n11152\r\n11153\r\n11154\r\n11155\r\n11156\r\n11157\r\n11158\r\n11159\r\n11160\r\n11161\r\n11162\r\n11163\r\n11164\r\n11165\r\n11166\r\n11167\r\n11168\r\n11169\r\n11170\r\n11171\r\n11172\r\n11173\r\n11174\r\n11175\r\n11176\r\n11177\r\n11178\r\n11179\r\n11180\r\n11181\r\n11182\r\n11183\r\n11184\r\n11185\r\n11186\r\n11187\r\n11188\r\n11189\r\n11190\r\n11191\r\n11192\r\n11193\r\n11194\r\n11195\r\n11196\r\n11197\r\n11198\r\n11199\r\n11200\r\n11201\r\n11202\r\n11203\r\n11204\r\n11205\r\n11206\r\n11207\r\n11208\r\n11209\r\n11210\r\n11211\r\n11212\r\n11213\r\n11214\r\n11215\r\n11216\r\n11217\r\n11218\r\n11219\r\n11220\r\n11221\r\n11222\r\n11223\r\n11224\r\n11225\r\n11226\r\n11227\r\n11228\r\n11229\r\n11230\r\n11231\r\n11232\r\n11233\r\n11234\r\n11235\r\n11236\r\n11237\r\n11238\r\n11239\r\n11240\r\n11241\r\n11242\r\n11243\r\n11244\r\n11245\r\n11246\r\n11247\r\n11248\r\n11249\r\n11250\r\n11251\r\n11252\r\n11253\r\n11254\r\n11255\r\n11256\r\n11257\r\n11258\r\n11259\r\n11260\r\n11261\r\n11262\r\n11263\r\n11264\r\n11265\r\n11266\r\n11267\r\n11268\r\n11269\r\n11270\r\n11271\r\n11272\r\n11273\r\n11274\r\n11275\r\n11276\r\n11277\r\n11278\r\n11279\r\n11280\r\n11281\r\n11282\r\n11283\r\n11284\r\n11285\r\n11286\r\n11287\r\n11288\r\n11289\r\n11290\r\n11291\r\n11292\r\n11293\r\n11294\r\n11295\r\n11296\r\n11297\r\n11298\r\n11299\r\n11300\r\n11301\r\n11302\r\n11303\r\n11304\r\n11305\r\n11306\r\n11307\r\n11308\r\n11309\r\n11310\r\n11311\r\n11312\r\n11313\r\n11314\r\n11315\r\n11316\r\n11317\r\n11318\r\n11319\r\n11320\r\n11321\r\n11322\r\n11323\r\n11324\r\n11325\r\n11326\r\n11327\r\n11328\r\n11329\r\n11330\r\n11331\r\n11332\r\n11333\r\n11334\r\n11335\r\n11336\r\n11337\r\n11338\r\n11339\r\n11340\r\n11341\r\n11342\r\n11343\r\n11344\r\n11345\r\n11346\r\n11347\r\n11348\r\n11349\r\n11350\r\n11351\r\n11352\r\n11353\r\n11354\r\n11355\r\n11356\r\n11357\r\n11358\r\n11359\r\n11360\r\n11361\r\n11362\r\n11363\r\n11364\r\n11365\r\n11366\r\n11367\r\n11368\r\n11369\r\n11370\r\n11371\r\n11372\r\n11373\r\n11374\r\n11375\r\n11376\r\n11377\r\n11378\r\n11379\r\n11380\r\n11381\r\n11382\r\n11383\r\n11384\r\n11385\r\n11386\r\n11387\r\n11388\r\n11389\r\n11390\r\n11391\r\n11392\r\n11393\r\n11394\r\n11395\r\n11396\r\n11397\r\n11398\r\n11399\r\n11400\r\n11401\r\n11402\r\n11403\r\n11404\r\n11405\r\n11406\r\n11407\r\n11408\r\n11409\r\n11410\r\n11411\r\n11412\r\n11413\r\n11414\r\n11415\r\n11416\r\n11417\r\n11418\r\n11419\r\n11420\r\n11421\r\n11422\r\n11423\r\n11424\r\n11425\r\n11426\r\n11427\r\n11428\r\n11429\r\n11430\r\n11431\r\n11432\r\n11433\r\n11434\r\n11435\r\n11436\r\n11437\r\n11438\r\n11439\r\n11440\r\n11441\r\n11442\r\n11443\r\n11444\r\n11445\r\n11446\r\n11447\r\n11448\r\n11449\r\n11450\r\n11451\r\n11452\r\n11453\r\n11454\r\n11455\r\n11456\r\n11457\r\n11458\r\n11459\r\n11460\r\n11461\r\n11462\r\n11463\r\n11464\r\n11465\r\n11466\r\n11467\r\n11468\r\n11469\r\n11470\r\n11471\r\n11472\r\n11473\r\n11474\r\n11475\r\n11476\r\n11477\r\n11478\r\n11479\r\n11480\r\n11481\r\n11482\r\n11483\r\n11484\r\n11485\r\n11486\r\n11487\r\n11488\r\n11489\r\n11490\r\n11491\r\n11492\r\n11493\r\n11494\r\n11495\r\n11496\r\n11497\r\n11498\r\n11499\r\n11500\r\n11501\r\n11502\r\n11503\r\n11504\r\n11505\r\n11506\r\n11507\r\n11508\r\n11509\r\n11510\r\n11511\r\n11512\r\n11513\r\n11514\r\n11515\r\n11516\r\n11517\r\n11518\r\n11519\r\n11520\r\n11521\r\n11522\r\n11523\r\n11524\r\n11525\r\n11526\r\n11527\r\n11528\r\n11529\r\n11530\r\n11531\r\n11532\r\n11533\r\n11534\r\n11535\r\n11536\r\n11537\r\n11538\r\n11539\r\n11540\r\n11541\r\n11542\r\n11543\r\n11544\r\n11545\r\n11546\r\n11547\r\n11548\r\n11549\r\n11550\r\n11551\r\n11552\r\n11553\r\n11554\r\n11555\r\n11556\r\n11557\r\n11558\r\n11559\r\n11560\r\n11561\r\n11562\r\n11563\r\n11564\r\n11565\r\n11566\r\n11567\r\n11568\r\n11569\r\n11570\r\n11571\r\n11572\r\n11573\r\n11574\r\n11575\r\n11576\r\n11577\r\n11578\r\n11579\r\n11580\r\n11581\r\n11582\r\n11583\r\n11584\r\n11585\r\n11586\r\n11587\r\n11588\r\n11589\r\n11590\r\n11591\r\n11592\r\n11593\r\n11594\r\n11595\r\n11596\r\n11597\r\n11598\r\n11599\r\n11600\r\n11601\r\n11602\r\n11603\r\n11604\r\n11605\r\n11606\r\n11607\r\n11608\r\n11609\r\n11610\r\n11611\r\n11612\r\n11613\r\n11614\r\n11615\r\n11616\r\n11617\r\n11618\r\n11619\r\n11620\r\n11621\r\n11622\r\n11623\r\n11624\r\n11625\r\n11626\r\n11627\r\n11628\r\n11629\r\n11630\r\n11631\r\n11632\r\n11633\r\n11634\r\n11635\r\n11636\r\n11637\r\n11638\r\n11639\r\n11640\r\n11641\r\n11642\r\n11643\r\n11644\r\n11645\r\n11646\r\n11647\r\n11648\r\n11649\r\n11650\r\n11651\r\n11652\r\n11653\r\n11654\r\n11655\r\n11656\r\n11657\r\n11658\r\n11659\r\n11660\r\n11661\r\n11662\r\n11663\r\n11664\r\n11665\r\n11666\r\n11667\r\n11668\r\n11669\r\n11670\r\n11671\r\n11672\r\n11673\r\n11674\r\n11675\r\n11676\r\n11677\r\n11678\r\n11679\r\n11680\r\n11681\r\n11682\r\n11683\r\n11684\r\n11685\r\n11686\r\n11687\r\n11688\r\n11689\r\n11690\r\n11691\r\n11692\r\n11693\r\n11694\r\n11695\r\n11696\r\n11697\r\n11698\r\n11699\r\n11700\r\n11701\r\n11702\r\n11703\r\n11704\r\n11705\r\n11706\r\n11707\r\n11708\r\n11709\r\n11710\r\n11711\r\n11712\r\n11713\r\n11714\r\n11715\r\n11716\r\n11717\r\n11718\r\n11719\r\n11720\r\n11721\r\n11722\r\n11723\r\n11724\r\n11725\r\n11726\r\n11727\r\n11728\r\n11729\r\n11730\r\n11731\r\n11732\r\n11733\r\n11734\r\n11735\r\n11736\r\n11737\r\n11738\r\n11739\r\n11740\r\n11741\r\n11742\r\n11743\r\n11744\r\n11745\r\n11746\r\n11747\r\n11748\r\n11749\r\n11750\r\n11751\r\n11752\r\n11753\r\n11754\r\n11755\r\n11756\r\n11757\r\n11758\r\n11759\r\n11760\r\n11761\r\n11762\r\n11763\r\n11764\r\n11765\r\n11766\r\n11767\r\n11768\r\n11769\r\n11770\r\n11771\r\n11772\r\n11773\r\n11774\r\n11775\r\n11776\r\n11777\r\n11778\r\n11779\r\n11780\r\n11781\r\n11782\r\n11783\r\n11784\r\n11785\r\n11786\r\n11787\r\n11788\r\n11789\r\n11790\r\n11791\r\n11792\r\n11793\r\n11794\r\n11795\r\n11796\r\n11797\r\n11798\r\n11799\r\n11800\r\n11801\r\n11802\r\n11803\r\n11804\r\n11805\r\n11806\r\n11807\r\n11808\r\n11809\r\n11810\r\n11811\r\n11812\r\n11813\r\n11814\r\n11815\r\n11816\r\n11817\r\n11818\r\n11819\r\n11820\r\n11821\r\n11822\r\n11823\r\n11824\r\n11825\r\n11826\r\n11827\r\n11828\r\n11829\r\n11830\r\n11831\r\n11832\r\n11833\r\n11834\r\n11835\r\n11836\r\n11837\r\n11838\r\n11839\r\n11840\r\n11841\r\n11842\r\n11843\r\n11844\r\n11845\r\n11846\r\n11847\r\n11848\r\n11849\r\n11850\r\n11851\r\n11852\r\n11853\r\n11854\r\n11855\r\n11856\r\n11857\r\n11858\r\n11859\r\n11860\r\n11861\r\n11862\r\n11863\r\n11864\r\n11865\r\n11866\r\n11867\r\n11868\r\n11869\r\n11870\r\n11871\r\n11872\r\n11873\r\n11874\r\n11875\r\n11876\r\n11877\r\n11878\r\n11879\r\n11880\r\n11881\r\n11882\r\n11883\r\n11884\r\n11885\r\n11886\r\n11887\r\n11888\r\n11889\r\n11890\r\n11891\r\n11892\r\n11893\r\n11894\r\n11895\r\n11896\r\n11897\r\n11898\r\n11899\r\n11900\r\n11901\r\n11902\r\n11903\r\n11904\r\n11905\r\n11906\r\n11907\r\n11908\r\n11909\r\n11910\r\n11911\r\n11912\r\n11913\r\n11914\r\n11915\r\n11916\r\n11917\r\n11918\r\n11919\r\n11920\r\n11921\r\n11922\r\n11923\r\n11924\r\n11925\r\n11926\r\n11927\r\n11928\r\n11929\r\n11930\r\n11931\r\n11932\r\n11933\r\n11934\r\n11935\r\n11936\r\n11937\r\n11938\r\n11939\r\n11940\r\n11941\r\n11942\r\n11943\r\n11944\r\n11945\r\n11946\r\n11947\r\n11948\r\n11949\r\n11950\r\n11951\r\n11952\r\n11953\r\n11954\r\n11955\r\n11956\r\n11957\r\n11958\r\n11959\r\n11960\r\n11961\r\n11962\r\n11963\r\n11964\r\n11965\r\n11966\r\n11967\r\n11968\r\n11969\r\n11970\r\n11971\r\n11972\r\n11973\r\n11974\r\n11975\r\n11976\r\n11977\r\n11978\r\n11979\r\n11980\r\n11981\r\n11982\r\n11983\r\n11984\r\n11985\r\n11986\r\n11987\r\n11988\r\n11989\r\n11990\r\n11991\r\n11992\r\n11993\r\n11994\r\n11995\r\n11996\r\n11997\r\n11998\r\n11999\r\n12000\r\n12001\r\n12002\r\n12003\r\n12004\r\n12005\r\n12006\r\n12007\r\n12008\r\n12009\r\n12010\r\n12011\r\n12012\r\n12013\r\n12014\r\n12015\r\n12016\r\n12017\r\n12018\r\n12019\r\n12020\r\n12021\r\n12022\r\n12023\r\n12024\r\n12025\r\n12026\r\n12027\r\n12028\r\n12029\r\n12030\r\n12031\r\n12032\r\n12033\r\n12034\r\n12035\r\n12036\r\n12037\r\n12038\r\n12039\r\n12040\r\n12041\r\n12042\r\n12043\r\n12044\r\n12045\r\n12046\r\n12047\r\n12048\r\n12049\r\n12050\r\n12051\r\n12052\r\n12053\r\n12054\r\n12055\r\n12056\r\n12057\r\n12058\r\n12059\r\n12060\r\n12061\r\n12062\r\n12063\r\n12064\r\n12065\r\n12066\r\n12067\r\n12068\r\n12069\r\n12070\r\n12071\r\n12072\r\n12073\r\n12074\r\n12075\r\n12076\r\n12077\r\n12078\r\n12079\r\n12080\r\n12081\r\n12082\r\n12083\r\n12084\r\n12085\r\n12086\r\n12087\r\n12088\r\n12089\r\n12090\r\n12091\r\n12092\r\n12093\r\n12094\r\n12095\r\n12096\r\n12097\r\n12098\r\n12099\r\n12100\r\n12101\r\n12102\r\n12103\r\n12104\r\n12105\r\n12106\r\n12107\r\n12108\r\n12109\r\n12110\r\n12111\r\n12112\r\n12113\r\n12114\r\n12115\r\n12116\r\n12117\r\n12118\r\n12119\r\n12120\r\n12121\r\n12122\r\n12123\r\n12124\r\n12125\r\n12126\r\n12127\r\n12128\r\n12129\r\n12130\r\n12131\r\n12132\r\n12133\r\n12134\r\n12135\r\n12136\r\n12137\r\n12138\r\n12139\r\n12140\r\n12141\r\n12142\r\n12143\r\n12144\r\n12145\r\n12146\r\n12147\r\n12148\r\n12149\r\n12150\r\n12151\r\n12152\r\n12153\r\n12154\r\n12155\r\n12156\r\n12157\r\n12158\r\n12159\r\n12160\r\n12161\r\n12162\r\n12163\r\n12164\r\n12165\r\n12166\r\n12167\r\n12168\r\n12169\r\n12170\r\n12171\r\n12172\r\n12173\r\n12174\r\n12175\r\n12176\r\n12177\r\n12178\r\n12179\r\n12180\r\n12181\r\n12182\r\n12183\r\n12184\r\n12185\r\n12186\r\n12187\r\n12188\r\n12189\r\n12190\r\n12191\r\n12192\r\n12193\r\n12194\r\n12195\r\n12196\r\n12197\r\n12198\r\n12199\r\n12200\r\n12201\r\n12202\r\n12203\r\n12204\r\n12205\r\n12206\r\n12207\r\n12208\r\n12209\r\n12210\r\n12211\r\n12212\r\n12213\r\n12214\r\n12215\r\n12216\r\n12217\r\n12218\r\n12219\r\n12220\r\n12221\r\n12222\r\n12223\r\n12224\r\n12225\r\n12226\r\n12227\r\n12228\r\n12229\r\n12230\r\n12231\r\n12232\r\n12233\r\n12234\r\n12235\r\n12236\r\n12237\r\n12238\r\n12239\r\n12240\r\n12241\r\n12242\r\n12243\r\n12244\r\n12245\r\n12246\r\n12247\r\n12248\r\n12249\r\n12250\r\n12251\r\n12252\r\n12253\r\n12254\r\n12255\r\n12256\r\n12257\r\n12258\r\n12259\r\n12260\r\n12261\r\n12262\r\n12263\r\n12264\r\n12265\r\n12266\r\n12267\r\n12268\r\n12269\r\n12270\r\n12271\r\n12272\r\n12273\r\n12274\r\n12275\r\n12276\r\n12277\r\n12278\r\n12279\r\n12280\r\n12281\r\n12282\r\n12283\r\n12284\r\n12285\r\n12286\r\n12287\r\n12288\r\n12289\r\n12290\r\n12291\r\n12292\r\n12293\r\n12294\r\n12295\r\n12296\r\n12297\r\n12298\r\n12299\r\n12300\r\n12301\r\n12302\r\n12303\r\n12304\r\n12305\r\n12306\r\n12307\r\n12308\r\n12309\r\n12310\r\n12311\r\n12312\r\n12313\r\n12314\r\n12315\r\n12316\r\n12317\r\n12318\r\n12319\r\n12320\r\n12321\r\n12322\r\n12323\r\n12324\r\n12325\r\n12326\r\n12327\r\n12328\r\n12329\r\n12330\r\n12331\r\n12332\r\n12333\r\n12334\r\n12335\r\n12336\r\n12337\r\n12338\r\n12339\r\n12340\r\n12341\r\n12342\r\n12343\r\n12344\r\n12345\r\n12346\r\n12347\r\n12348\r\n12349\r\n12350\r\n12351\r\n12352\r\n12353\r\n12354\r\n12355\r\n12356\r\n12357\r\n12358\r\n12359\r\n12360\r\n12361\r\n12362\r\n12363\r\n12364\r\n12365\r\n12366\r\n12367\r\n12368\r\n12369\r\n12370\r\n12371\r\n12372\r\n12373\r\n12374\r\n12375\r\n12376\r\n12377\r\n12378\r\n12379\r\n12380\r\n12381\r\n12382\r\n12383\r\n12384\r\n12385\r\n12386\r\n12387\r\n12388\r\n12389\r\n12390\r\n12391\r\n12392\r\n12393\r\n12394\r\n12395\r\n12396\r\n12397\r\n12398\r\n12399\r\n12400\r\n12401\r\n12402\r\n12403\r\n12404\r\n12405\r\n12406\r\n12407\r\n12408\r\n12409\r\n12410\r\n12411\r\n12412\r\n12413\r\n12414\r\n12415\r\n12416\r\n12417\r\n12418\r\n12419\r\n12420\r\n12421\r\n12422\r\n12423\r\n12424\r\n12425\r\n12426\r\n12427\r\n12428\r\n12429\r\n12430\r\n12431\r\n12432\r\n12433\r\n12434\r\n12435\r\n12436\r\n12437\r\n12438\r\n12439\r\n12440\r\n12441\r\n12442\r\n12443\r\n12444\r\n12445\r\n12446\r\n12447\r\n12448\r\n12449\r\n12450\r\n12451\r\n12452\r\n12453\r\n12454\r\n12455\r\n12456\r\n12457\r\n12458\r\n12459\r\n12460\r\n12461\r\n12462\r\n12463\r\n12464\r\n12465\r\n12466\r\n12467\r\n12468\r\n12469\r\n12470\r\n12471\r\n12472\r\n12473\r\n12474\r\n12475\r\n12476\r\n12477\r\n12478\r\n12479\r\n12480\r\n12481\r\n12482\r\n12483\r\n12484\r\n12485\r\n12486\r\n12487\r\n12488\r\n12489\r\n12490\r\n12491\r\n12492\r\n12493\r\n12494\r\n12495\r\n12496\r\n12497\r\n12498\r\n12499\r\n12500\r\n12501\r\n12502\r\n12503\r\n12504\r\n12505\r\n12506\r\n12507\r\n12508\r\n12509\r\n12510\r\n12511\r\n12512\r\n12513\r\n12514\r\n12515\r\n12516\r\n12517\r\n12518\r\n12519\r\n12520\r\n12521\r\n12522\r\n12523\r\n12524\r\n12525\r\n12526\r\n12527\r\n12528\r\n12529\r\n12530\r\n12531\r\n12532\r\n12533\r\n12534\r\n12535\r\n12536\r\n12537\r\n12538\r\n12539\r\n12540\r\n12541\r\n12542\r\n12543\r\n12544\r\n12545\r\n12546\r\n12547\r\n12548\r\n12549\r\n12550\r\n12551\r\n12552\r\n12553\r\n12554\r\n12555\r\n12556\r\n12557\r\n12558\r\n12559\r\n12560\r\n12561\r\n12562\r\n12563\r\n12564\r\n12565\r\n12566\r\n12567\r\n12568\r\n12569\r\n12570\r\n12571\r\n12572\r\n12573\r\n12574\r\n12575\r\n12576\r\n12577\r\n12578\r\n12579\r\n12580\r\n12581\r\n12582\r\n12583\r\n12584\r\n12585\r\n12586\r\n12587\r\n12588\r\n12589\r\n12590\r\n12591\r\n12592\r\n12593\r\n12594\r\n12595\r\n12596\r\n12597\r\n12598\r\n12599\r\n12600\r\n12601\r\n12602\r\n12603\r\n12604\r\n12605\r\n12606\r\n12607\r\n12608\r\n12609\r\n12610\r\n12611\r\n12612\r\n12613\r\n12614\r\n12615\r\n12616\r\n12617\r\n12618\r\n12619\r\n12620\r\n12621\r\n12622\r\n12623\r\n12624\r\n12625\r\n12626\r\n12627\r\n12628\r\n12629\r\n12630\r\n12631\r\n12632\r\n12633\r\n12634\r\n12635\r\n12636\r\n12637\r\n12638\r\n12639\r\n12640\r\n12641\r\n12642\r\n12643\r\n12644\r\n12645\r\n12646\r\n12647\r\n12648\r\n12649\r\n12650\r\n12651\r\n12652\r\n12653\r\n12654\r\n12655\r\n12656\r\n12657\r\n12658\r\n12659\r\n12660\r\n12661\r\n12662\r\n12663\r\n12664\r\n12665\r\n12666\r\n12667\r\n12668\r\n12669\r\n12670\r\n12671\r\n12672\r\n12673\r\n12674\r\n12675\r\n12676\r\n12677\r\n12678\r\n12679\r\n12680\r\n12681\r\n12682\r\n12683\r\n12684\r\n12685\r\n12686\r\n12687\r\n12688\r\n12689\r\n12690\r\n12691\r\n12692\r\n12693\r\n12694\r\n12695\r\n12696\r\n12697\r\n12698\r\n12699\r\n12700\r\n12701\r\n12702\r\n12703\r\n12704\r\n12705\r\n12706\r\n12707\r\n12708\r\n12709\r\n12710\r\n12711\r\n12712\r\n12713\r\n12714\r\n12715\r\n12716\r\n12717\r\n12718\r\n12719\r\n12720\r\n12721\r\n12722\r\n12723\r\n12724\r\n12725\r\n12726\r\n12727\r\n12728\r\n12729\r\n12730\r\n12731\r\n12732\r\n12733\r\n12734\r\n12735\r\n12736\r\n12737\r\n12738\r\n12739\r\n12740\r\n12741\r\n12742\r\n12743\r\n12744\r\n12745\r\n12746\r\n12747\r\n12748\r\n12749\r\n12750\r\n12751\r\n12752\r\n12753\r\n12754\r\n12755\r\n12756\r\n12757\r\n12758\r\n12759\r\n12760\r\n12761\r\n12762\r\n12763\r\n12764\r\n12765\r\n12766\r\n12767\r\n12768\r\n12769\r\n12770\r\n12771\r\n12772\r\n12773\r\n12774\r\n12775\r\n12776\r\n12777\r\n12778\r\n12779\r\n12780\r\n12781\r\n12782\r\n12783\r\n12784\r\n12785\r\n12786\r\n12787\r\n12788\r\n12789\r\n12790\r\n12791\r\n12792\r\n12793\r\n12794\r\n12795\r\n12796\r\n12797\r\n12798\r\n12799\r\n12800\r\n12801\r\n12802\r\n12803\r\n12804\r\n12805\r\n12806\r\n12807\r\n12808\r\n12809\r\n12810\r\n12811\r\n12812\r\n12813\r\n12814\r\n12815\r\n12816\r\n12817\r\n12818\r\n12819\r\n12820\r\n12821\r\n12822\r\n12823\r\n12824\r\n12825\r\n12826\r\n12827\r\n12828\r\n12829\r\n12830\r\n12831\r\n12832\r\n12833\r\n12834\r\n12835\r\n12836\r\n12837\r\n12838\r\n12839\r\n12840\r\n12841\r\n12842\r\n12843\r\n12844\r\n12845\r\n12846\r\n12847\r\n12848\r\n12849\r\n12850\r\n12851\r\n12852\r\n12853\r\n12854\r\n12855\r\n12856\r\n12857\r\n12858\r\n12859\r\n12860\r\n12861\r\n12862\r\n12863\r\n12864\r\n12865\r\n12866\r\n12867\r\n12868\r\n12869\r\n12870\r\n12871\r\n12872\r\n12873\r\n12874\r\n12875\r\n12876\r\n12877\r\n12878\r\n12879\r\n12880\r\n12881\r\n12882\r\n12883\r\n12884\r\n12885\r\n12886\r\n12887\r\n12888\r\n12889\r\n12890\r\n12891\r\n12892\r\n12893\r\n12894\r\n12895\r\n12896\r\n12897\r\n12898\r\n12899\r\n12900\r\n12901\r\n12902\r\n12903\r\n12904\r\n12905\r\n12906\r\n12907\r\n12908\r\n12909\r\n12910\r\n12911\r\n12912\r\n12913\r\n12914\r\n12915\r\n12916\r\n12917\r\n12918\r\n12919\r\n12920\r\n12921\r\n12922\r\n12923\r\n12924\r\n12925\r\n12926\r\n12927\r\n12928\r\n12929\r\n12930\r\n12931\r\n12932\r\n12933\r\n12934\r\n12935\r\n12936\r\n12937\r\n12938\r\n12939\r\n12940\r\n12941\r\n12942\r\n12943\r\n12944\r\n12945\r\n12946\r\n12947\r\n12948\r\n12949\r\n12950\r\n12951\r\n12952\r\n12953\r\n12954\r\n12955\r\n12956\r\n12957\r\n12958\r\n12959\r\n12960\r\n12961\r\n12962\r\n12963\r\n12964\r\n12965\r\n12966\r\n12967\r\n12968\r\n12969\r\n12970\r\n12971\r\n12972\r\n12973\r\n12974\r\n12975\r\n12976\r\n12977\r\n12978\r\n12979\r\n12980\r\n12981\r\n12982\r\n12983\r\n12984\r\n12985\r\n12986\r\n12987\r\n12988\r\n12989\r\n12990\r\n12991\r\n12992\r\n12993\r\n12994\r\n12995\r\n12996\r\n12997\r\n12998\r\n12999\r\n13000\r\n13001\r\n13002\r\n13003\r\n13004\r\n13005\r\n13006\r\n13007\r\n13008\r\n13009\r\n13010\r\n13011\r\n13012\r\n13013\r\n13014\r\n13015\r\n13016\r\n13017\r\n13018\r\n13019\r\n13020\r\n13021\r\n13022\r\n13023\r\n13024\r\n13025\r\n13026\r\n13027\r\n13028\r\n13029\r\n13030\r\n13031\r\n13032\r\n13033\r\n13034\r\n13035\r\n13036\r\n13037\r\n13038\r\n13039\r\n13040\r\n13041\r\n13042\r\n13043\r\n13044\r\n13045\r\n13046\r\n13047\r\n13048\r\n13049\r\n13050\r\n13051\r\n13052\r\n13053\r\n13054\r\n13055\r\n13056\r\n13057\r\n13058\r\n13059\r\n13060\r\n13061\r\n13062\r\n13063\r\n13064\r\n13065\r\n13066\r\n13067\r\n13068\r\n13069\r\n13070\r\n13071\r\n13072\r\n13073\r\n13074\r\n13075\r\n13076\r\n13077\r\n13078\r\n13079\r\n13080\r\n13081\r\n13082\r\n13083\r\n13084\r\n13085\r\n13086\r\n13087\r\n13088\r\n13089\r\n13090\r\n13091\r\n13092\r\n13093\r\n13094\r\n13095\r\n13096\r\n13097\r\n13098\r\n13099\r\n13100\r\n13101\r\n13102\r\n13103\r\n13104\r\n13105\r\n13106\r\n13107\r\n13108\r\n13109\r\n13110\r\n13111\r\n13112\r\n13113\r\n13114\r\n13115\r\n13116\r\n13117\r\n13118\r\n13119\r\n13120\r\n13121\r\n13122\r\n13123\r\n13124\r\n13125\r\n13126\r\n13127\r\n13128\r\n13129\r\n13130\r\n13131\r\n13132\r\n13133\r\n13134\r\n13135\r\n13136\r\n13137\r\n13138\r\n13139\r\n13140\r\n13141\r\n13142\r\n13143\r\n13144\r\n13145\r\n13146\r\n13147\r\n13148\r\n13149\r\n13150\r\n13151\r\n13152\r\n13153\r\n13154\r\n13155\r\n13156\r\n13157\r\n13158\r\n13159\r\n13160\r\n13161\r\n13162\r\n13163\r\n13164\r\n13165\r\n13166\r\n13167\r\n13168\r\n13169\r\n13170\r\n13171\r\n13172\r\n13173\r\n13174\r\n13175\r\n13176\r\n13177\r\n13178\r\n13179\r\n13180\r\n13181\r\n13182\r\n13183\r\n13184\r\n13185\r\n13186\r\n13187\r\n13188\r\n13189\r\n13190\r\n13191\r\n13192\r\n13193\r\n13194\r\n13195\r\n13196\r\n13197\r\n13198\r\n13199\r\n13200\r\n13201\r\n13202\r\n13203\r\n13204\r\n13205\r\n13206\r\n13207\r\n13208\r\n13209\r\n13210\r\n13211\r\n13212\r\n13213\r\n13214\r\n13215\r\n13216\r\n13217\r\n13218\r\n13219\r\n13220\r\n13221\r\n13222\r\n13223\r\n13224\r\n13225\r\n13226\r\n13227\r\n13228\r\n13229\r\n13230\r\n13231\r\n13232\r\n13233\r\n13234\r\n13235\r\n13236\r\n13237\r\n13238\r\n13239\r\n13240\r\n13241\r\n13242\r\n13243\r\n13244\r\n13245\r\n13246\r\n13247\r\n13248\r\n13249\r\n13250\r\n13251\r\n13252\r\n13253\r\n13254\r\n13255\r\n13256\r\n13257\r\n13258\r\n13259\r\n13260\r\n13261\r\n13262\r\n13263\r\n13264\r\n13265\r\n13266\r\n13267\r\n13268\r\n13269\r\n13270\r\n13271\r\n13272\r\n13273\r\n13274\r\n13275\r\n13276\r\n13277\r\n13278\r\n13279\r\n13280\r\n13281\r\n13282\r\n13283\r\n13284\r\n13285\r\n13286\r\n13287\r\n13288\r\n13289\r\n13290\r\n13291\r\n13292\r\n13293\r\n13294\r\n13295\r\n13296\r\n13297\r\n13298\r\n13299\r\n13300\r\n13301\r\n13302\r\n13303\r\n13304\r\n13305\r\n13306\r\n13307\r\n13308\r\n13309\r\n13310\r\n13311\r\n13312\r\n13313\r\n13314\r\n13315\r\n13316\r\n13317\r\n13318\r\n13319\r\n13320\r\n13321\r\n13322\r\n13323\r\n13324\r\n13325\r\n13326\r\n13327\r\n13328\r\n13329\r\n13330\r\n13331\r\n13332\r\n13333\r\n13334\r\n13335\r\n13336\r\n13337\r\n13338\r\n13339\r\n13340\r\n13341\r\n13342\r\n13343\r\n13344\r\n13345\r\n13346\r\n13347\r\n13348\r\n13349\r\n13350\r\n13351\r\n13352\r\n13353\r\n13354\r\n13355\r\n13356\r\n13357\r\n13358\r\n13359\r\n13360\r\n13361\r\n13362\r\n13363\r\n13364\r\n13365\r\n13366\r\n13367\r\n13368\r\n13369\r\n13370\r\n13371\r\n13372\r\n13373\r\n13374\r\n13375\r\n13376\r\n13377\r\n13378\r\n13379\r\n13380\r\n13381\r\n13382\r\n13383\r\n13384\r\n13385\r\n13386\r\n13387\r\n13388\r\n13389\r\n13390\r\n13391\r\n13392\r\n13393\r\n13394\r\n13395\r\n13396\r\n13397\r\n13398\r\n13399\r\n13400\r\n13401\r\n13402\r\n13403\r\n13404\r\n13405\r\n13406\r\n13407\r\n13408\r\n13409\r\n13410\r\n13411\r\n13412\r\n13413\r\n13414\r\n13415\r\n13416\r\n13417\r\n13418\r\n13419\r\n13420\r\n13421\r\n13422\r\n13423\r\n13424\r\n13425\r\n13426\r\n13427\r\n13428\r\n13429\r\n13430\r\n13431\r\n13432\r\n13433\r\n13434\r\n13435\r\n13436\r\n13437\r\n13438\r\n13439\r\n13440\r\n13441\r\n13442\r\n13443\r\n13444\r\n13445\r\n13446\r\n13447\r\n13448\r\n13449\r\n13450\r\n13451\r\n13452\r\n13453\r\n13454\r\n13455\r\n13456\r\n13457\r\n13458\r\n13459\r\n13460\r\n13461\r\n13462\r\n13463\r\n13464\r\n13465\r\n13466\r\n13467\r\n13468\r\n13469\r\n13470\r\n13471\r\n13472\r\n13473\r\n13474\r\n13475\r\n13476\r\n13477\r\n13478\r\n13479\r\n13480\r\n13481\r\n13482\r\n13483\r\n13484\r\n13485\r\n13486\r\n13487\r\n13488\r\n13489\r\n13490\r\n13491\r\n13492\r\n13493\r\n13494\r\n13495\r\n13496\r\n13497\r\n13498\r\n13499\r\n13500\r\n13501\r\n13502\r\n13503\r\n13504\r\n13505\r\n13506\r\n13507\r\n13508\r\n13509\r\n13510\r\n13511\r\n13512\r\n13513\r\n13514\r\n13515\r\n13516\r\n13517\r\n13518\r\n13519\r\n13520\r\n13521\r\n13522\r\n13523\r\n13524\r\n13525\r\n13526\r\n13527\r\n13528\r\n13529\r\n13530\r\n13531\r\n13532\r\n13533\r\n13534\r\n13535\r\n13536\r\n13537\r\n13538\r\n13539\r\n13540\r\n13541\r\n13542\r\n13543\r\n13544\r\n13545\r\n13546\r\n13547\r\n13548\r\n13549\r\n13550\r\n13551\r\n13552\r\n13553\r\n13554\r\n13555\r\n13556\r\n13557\r\n13558\r\n13559\r\n13560\r\n13561\r\n13562\r\n13563\r\n13564\r\n13565\r\n13566\r\n13567\r\n13568\r\n13569\r\n13570\r\n13571\r\n13572\r\n13573\r\n13574\r\n13575\r\n13576\r\n13577\r\n13578\r\n13579\r\n13580\r\n13581\r\n13582\r\n13583\r\n13584\r\n13585\r\n13586\r\n13587\r\n13588\r\n13589\r\n13590\r\n13591\r\n13592\r\n13593\r\n13594\r\n13595\r\n13596\r\n13597\r\n13598\r\n13599\r\n13600\r\n13601\r\n13602\r\n13603\r\n13604\r\n13605\r\n13606\r\n13607\r\n13608\r\n13609\r\n13610\r\n13611\r\n13612\r\n13613\r\n13614\r\n13615\r\n13616\r\n13617\r\n13618\r\n13619\r\n13620\r\n13621\r\n13622\r\n13623\r\n13624\r\n13625\r\n13626\r\n13627\r\n13628\r\n13629\r\n13630\r\n13631\r\n13632\r\n13633\r\n13634\r\n13635\r\n13636\r\n13637\r\n13638\r\n13639\r\n13640\r\n13641\r\n13642\r\n13643\r\n13644\r\n13645\r\n13646\r\n13647\r\n13648\r\n13649\r\n13650\r\n13651\r\n13652\r\n13653\r\n13654\r\n13655\r\n13656\r\n13657\r\n13658\r\n13659\r\n13660\r\n13661\r\n13662\r\n13663\r\n13664\r\n13665\r\n13666\r\n13667\r\n13668\r\n13669\r\n13670\r\n13671\r\n13672\r\n13673\r\n13674\r\n13675\r\n13676\r\n13677\r\n13678\r\n13679\r\n13680\r\n13681\r\n13682\r\n13683\r\n13684\r\n13685\r\n13686\r\n13687\r\n13688\r\n13689\r\n13690\r\n13691\r\n13692\r\n13693\r\n13694\r\n13695\r\n13696\r\n13697\r\n13698\r\n13699\r\n13700\r\n13701\r\n13702\r\n13703\r\n13704\r\n13705\r\n13706\r\n13707\r\n13708\r\n13709\r\n13710\r\n13711\r\n13712\r\n13713\r\n13714\r\n13715\r\n13716\r\n13717\r\n13718\r\n13719\r\n13720\r\n13721\r\n13722\r\n13723\r\n13724\r\n13725\r\n13726\r\n13727\r\n13728\r\n13729\r\n13730\r\n13731\r\n13732\r\n13733\r\n13734\r\n13735\r\n13736\r\n13737\r\n13738\r\n13739\r\n13740\r\n13741\r\n13742\r\n13743\r\n13744\r\n13745\r\n13746\r\n13747\r\n13748\r\n13749\r\n13750\r\n13751\r\n13752\r\n13753\r\n13754\r\n13755\r\n13756\r\n13757\r\n13758\r\n13759\r\n13760\r\n13761\r\n13762\r\n13763\r\n13764\r\n13765\r\n13766\r\n13767\r\n13768\r\n13769\r\n13770\r\n13771\r\n13772\r\n13773\r\n13774\r\n13775\r\n13776\r\n13777\r\n13778\r\n13779\r\n13780\r\n13781\r\n13782\r\n13783\r\n13784\r\n13785\r\n13786\r\n13787\r\n13788\r\n13789\r\n13790\r\n13791\r\n13792\r\n13793\r\n13794\r\n13795\r\n13796\r\n13797\r\n13798\r\n13799\r\n13800\r\n13801\r\n13802\r\n13803\r\n13804\r\n13805\r\n13806\r\n13807\r\n13808\r\n13809\r\n13810\r\n13811\r\n13812\r\n13813\r\n13814\r\n13815\r\n13816\r\n13817\r\n13818\r\n13819\r\n13820\r\n13821\r\n13822\r\n13823\r\n13824\r\n13825\r\n13826\r\n13827\r\n13828\r\n13829\r\n13830\r\n13831\r\n13832\r\n13833\r\n13834\r\n13835\r\n13836\r\n13837\r\n13838\r\n13839\r\n13840\r\n13841\r\n13842\r\n13843\r\n13844\r\n13845\r\n13846\r\n13847\r\n13848\r\n13849\r\n13850\r\n13851\r\n13852\r\n13853\r\n13854\r\n13855\r\n13856\r\n13857\r\n13858\r\n13859\r\n13860\r\n13861\r\n13862\r\n13863\r\n13864\r\n13865\r\n13866\r\n13867\r\n13868\r\n13869\r\n13870\r\n13871\r\n13872\r\n13873\r\n13874\r\n13875\r\n13876\r\n13877\r\n13878\r\n13879\r\n13880\r\n13881\r\n13882\r\n13883\r\n13884\r\n13885\r\n13886\r\n13887\r\n13888\r\n13889\r\n13890\r\n13891\r\n13892\r\n13893\r\n13894\r\n13895\r\n13896\r\n13897\r\n13898\r\n13899\r\n13900\r\n13901\r\n13902\r\n13903\r\n13904\r\n13905\r\n13906\r\n13907\r\n13908\r\n13909\r\n13910\r\n13911\r\n13912\r\n13913\r\n13914\r\n13915\r\n13916\r\n13917\r\n13918\r\n13919\r\n13920\r\n13921\r\n13922\r\n13923\r\n13924\r\n13925\r\n13926\r\n13927\r\n13928\r\n13929\r\n13930\r\n13931\r\n13932\r\n13933\r\n13934\r\n13935\r\n13936\r\n13937\r\n13938\r\n13939\r\n13940\r\n13941\r\n13942\r\n13943\r\n13944\r\n13945\r\n13946\r\n13947\r\n13948\r\n13949\r\n13950\r\n13951\r\n13952\r\n13953\r\n13954\r\n13955\r\n13956\r\n13957\r\n13958\r\n13959\r\n13960\r\n13961\r\n13962\r\n13963\r\n13964\r\n13965\r\n13966\r\n13967\r\n13968\r\n13969\r\n13970\r\n13971\r\n13972\r\n13973\r\n13974\r\n13975\r\n13976\r\n13977\r\n13978\r\n13979\r\n13980\r\n13981\r\n13982\r\n13983\r\n13984\r\n13985\r\n13986\r\n13987\r\n13988\r\n13989\r\n13990\r\n13991\r\n13992\r\n13993\r\n13994\r\n13995\r\n13996\r\n13997\r\n13998\r\n13999\r\n14000\r\n14001\r\n14002\r\n14003\r\n14004\r\n14005\r\n14006\r\n14007\r\n14008\r\n14009\r\n14010\r\n14011\r\n14012\r\n14013\r\n14014\r\n14015\r\n14016\r\n14017\r\n14018\r\n14019\r\n14020\r\n14021\r\n14022\r\n14023\r\n14024\r\n14025\r\n14026\r\n14027\r\n14028\r\n14029\r\n14030\r\n14031\r\n14032\r\n14033\r\n14034\r\n14035\r\n14036\r\n14037\r\n14038\r\n14039\r\n14040\r\n14041\r\n14042\r\n14043\r\n14044\r\n14045\r\n14046\r\n14047\r\n14048\r\n14049\r\n14050\r\n14051\r\n14052\r\n14053\r\n14054\r\n14055\r\n14056\r\n14057\r\n14058\r\n14059\r\n14060\r\n14061\r\n14062\r\n14063\r\n14064\r\n14065\r\n14066\r\n14067\r\n14068\r\n14069\r\n14070\r\n14071\r\n14072\r\n14073\r\n14074\r\n14075\r\n14076\r\n14077\r\n14078\r\n14079\r\n14080\r\n14081\r\n14082\r\n14083\r\n14084\r\n14085\r\n14086\r\n14087\r\n14088\r\n14089\r\n14090\r\n14091\r\n14092\r\n14093\r\n14094\r\n14095\r\n14096\r\n14097\r\n14098\r\n14099\r\n14100\r\n14101\r\n14102\r\n14103\r\n14104\r\n14105\r\n14106\r\n14107\r\n14108\r\n14109\r\n14110\r\n14111\r\n14112\r\n14113\r\n14114\r\n14115\r\n14116\r\n14117\r\n14118\r\n14119\r\n14120\r\n14121\r\n14122\r\n14123\r\n14124\r\n14125\r\n14126\r\n14127\r\n14128\r\n14129\r\n14130\r\n14131\r\n14132\r\n14133\r\n14134\r\n14135\r\n14136\r\n14137\r\n14138\r\n14139\r\n14140\r\n14141\r\n14142\r\n14143\r\n14144\r\n14145\r\n14146\r\n14147\r\n14148\r\n14149\r\n14150\r\n14151\r\n14152\r\n14153\r\n14154\r\n14155\r\n14156\r\n14157\r\n14158\r\n14159\r\n14160\r\n14161\r\n14162\r\n14163\r\n14164\r\n14165\r\n14166\r\n14167\r\n14168\r\n14169\r\n14170\r\n14171\r\n14172\r\n14173\r\n14174\r\n14175\r\n14176\r\n14177\r\n14178\r\n14179\r\n14180\r\n14181\r\n14182\r\n14183\r\n14184\r\n14185\r\n14186\r\n14187\r\n14188\r\n14189\r\n14190\r\n14191\r\n14192\r\n14193\r\n14194\r\n14195\r\n14196\r\n14197\r\n14198\r\n14199\r\n14200\r\n14201\r\n14202\r\n14203\r\n14204\r\n14205\r\n14206\r\n14207\r\n14208\r\n14209\r\n14210\r\n14211\r\n14212\r\n14213\r\n14214\r\n14215\r\n14216\r\n14217\r\n14218\r\n14219\r\n14220\r\n14221\r\n14222\r\n14223\r\n14224\r\n14225\r\n14226\r\n14227\r\n14228\r\n14229\r\n14230\r\n14231\r\n14232\r\n14233\r\n14234\r\n14235\r\n14236\r\n14237\r\n14238\r\n14239\r\n14240\r\n14241\r\n14242\r\n14243\r\n14244\r\n14245\r\n14246\r\n14247\r\n14248\r\n14249\r\n14250\r\n14251\r\n14252\r\n14253\r\n14254\r\n14255\r\n14256\r\n14257\r\n14258\r\n14259\r\n14260\r\n14261\r\n14262\r\n14263\r\n14264\r\n14265\r\n14266\r\n14267\r\n14268\r\n14269\r\n14270\r\n14271\r\n14272\r\n14273\r\n14274\r\n14275\r\n14276\r\n14277\r\n14278\r\n14279\r\n14280\r\n14281\r\n14282\r\n14283\r\n14284\r\n14285\r\n14286\r\n14287\r\n14288\r\n14289\r\n14290\r\n14291\r\n14292\r\n14293\r\n14294\r\n14295\r\n14296\r\n14297\r\n14298\r\n14299\r\n14300\r\n14301\r\n14302\r\n14303\r\n14304\r\n14305\r\n14306\r\n14307\r\n14308\r\n14309\r\n14310\r\n14311\r\n14312\r\n14313\r\n14314\r\n14315\r\n14316\r\n14317\r\n14318\r\n14319\r\n14320\r\n14321\r\n14322\r\n14323\r\n14324\r\n14325\r\n14326\r\n14327\r\n14328\r\n14329\r\n14330\r\n14331\r\n14332\r\n14333\r\n14334\r\n14335\r\n14336\r\n14337\r\n14338\r\n14339\r\n14340\r\n14341\r\n14342\r\n14343\r\n14344\r\n14345\r\n14346\r\n14347\r\n14348\r\n14349\r\n14350\r\n14351\r\n14352\r\n14353\r\n14354\r\n14355\r\n14356\r\n14357\r\n14358\r\n14359\r\n14360\r\n14361\r\n14362\r\n14363\r\n14364\r\n14365\r\n14366\r\n14367\r\n14368\r\n14369\r\n14370\r\n14371\r\n14372\r\n14373\r\n14374\r\n14375\r\n14376\r\n14377\r\n14378\r\n14379\r\n14380\r\n14381\r\n14382\r\n14383\r\n14384\r\n14385\r\n14386\r\n14387\r\n14388\r\n14389\r\n14390\r\n14391\r\n14392\r\n14393\r\n14394\r\n14395\r\n14396\r\n14397\r\n14398\r\n14399\r\n14400\r\n14401\r\n14402\r\n14403\r\n14404\r\n14405\r\n14406\r\n14407\r\n14408\r\n14409\r\n14410\r\n14411\r\n14412\r\n14413\r\n14414\r\n14415\r\n14416\r\n14417\r\n14418\r\n14419\r\n14420\r\n14421\r\n14422\r\n14423\r\n14424\r\n14425\r\n14426\r\n14427\r\n14428\r\n14429\r\n14430\r\n14431\r\n14432\r\n14433\r\n14434\r\n14435\r\n14436\r\n14437\r\n14438\r\n14439\r\n14440\r\n14441\r\n14442\r\n14443\r\n14444\r\n14445\r\n14446\r\n14447\r\n14448\r\n14449\r\n14450\r\n14451\r\n14452\r\n14453\r\n14454\r\n14455\r\n14456\r\n14457\r\n14458\r\n14459\r\n14460\r\n14461\r\n14462\r\n14463\r\n14464\r\n14465\r\n14466\r\n14467\r\n14468\r\n14469\r\n14470\r\n14471\r\n14472\r\n14473\r\n14474\r\n14475\r\n14476\r\n14477\r\n14478\r\n14479\r\n14480\r\n14481\r\n14482\r\n14483\r\n14484\r\n14485\r\n14486\r\n14487\r\n14488\r\n14489\r\n14490\r\n14491\r\n14492\r\n14493\r\n14494\r\n14495\r\n14496\r\n14497\r\n14498\r\n14499\r\n14500\r\n14501\r\n14502\r\n14503\r\n14504\r\n14505\r\n14506\r\n14507\r\n14508\r\n14509\r\n14510\r\n14511\r\n14512\r\n14513\r\n14514\r\n14515\r\n14516\r\n14517\r\n14518\r\n14519\r\n14520\r\n14521\r\n14522\r\n14523\r\n14524\r\n14525\r\n14526\r\n14527\r\n14528\r\n14529\r\n14530\r\n14531\r\n14532\r\n14533\r\n14534\r\n14535\r\n14536\r\n14537\r\n14538\r\n14539\r\n14540\r\n14541\r\n14542\r\n14543\r\n14544\r\n14545\r\n14546\r\n14547\r\n14548\r\n14549\r\n14550\r\n14551\r\n14552\r\n14553\r\n14554\r\n14555\r\n14556\r\n14557\r\n14558\r\n14559\r\n14560\r\n14561\r\n14562\r\n14563\r\n14564\r\n14565\r\n14566\r\n14567\r\n14568\r\n14569\r\n14570\r\n14571\r\n14572\r\n14573\r\n14574\r\n14575\r\n14576\r\n14577\r\n14578\r\n14579\r\n14580\r\n14581\r\n14582\r\n14583\r\n14584\r\n14585\r\n14586\r\n14587\r\n14588\r\n14589\r\n14590\r\n14591\r\n14592\r\n14593\r\n14594\r\n14595\r\n14596\r\n14597\r\n14598\r\n14599\r\n14600\r\n14601\r\n14602\r\n14603\r\n14604\r\n14605\r\n14606\r\n14607\r\n14608\r\n14609\r\n14610\r\n14611\r\n14612\r\n14613\r\n14614\r\n14615\r\n14616\r\n14617\r\n14618\r\n14619\r\n14620\r\n14621\r\n14622\r\n14623\r\n14624\r\n14625\r\n14626\r\n14627\r\n14628\r\n14629\r\n14630\r\n14631\r\n14632\r\n14633\r\n14634\r\n14635\r\n14636\r\n14637\r\n14638\r\n14639\r\n14640\r\n14641\r\n14642\r\n14643\r\n14644\r\n14645\r\n14646\r\n14647\r\n14648\r\n14649\r\n14650\r\n14651\r\n14652\r\n14653\r\n14654\r\n14655\r\n14656\r\n14657\r\n14658\r\n14659\r\n14660\r\n14661\r\n14662\r\n14663\r\n14664\r\n14665\r\n14666\r\n14667\r\n14668\r\n14669\r\n14670\r\n14671\r\n14672\r\n14673\r\n14674\r\n14675\r\n14676\r\n14677\r\n14678\r\n14679\r\n14680\r\n14681\r\n14682\r\n14683\r\n14684\r\n14685\r\n14686\r\n14687\r\n14688\r\n14689\r\n14690\r\n14691\r\n14692\r\n14693\r\n14694\r\n14695\r\n14696\r\n14697\r\n14698\r\n14699\r\n14700\r\n14701\r\n14702\r\n14703\r\n14704\r\n14705\r\n14706\r\n14707\r\n14708\r\n14709\r\n14710\r\n14711\r\n14712\r\n14713\r\n14714\r\n14715\r\n14716\r\n14717\r\n14718\r\n14719\r\n14720\r\n14721\r\n14722\r\n14723\r\n14724\r\n14725\r\n14726\r\n14727\r\n14728\r\n14729\r\n14730\r\n14731\r\n14732\r\n14733\r\n14734\r\n14735\r\n14736\r\n14737\r\n14738\r\n14739\r\n14740\r\n14741\r\n14742\r\n14743\r\n14744\r\n14745\r\n14746\r\n14747\r\n14748\r\n14749\r\n14750\r\n14751\r\n14752\r\n14753\r\n14754\r\n14755\r\n14756\r\n14757\r\n14758\r\n14759\r\n14760\r\n14761\r\n14762\r\n14763\r\n14764\r\n14765\r\n14766\r\n14767\r\n14768\r\n14769\r\n14770\r\n14771\r\n14772\r\n14773\r\n14774\r\n14775\r\n14776\r\n14777\r\n14778\r\n14779\r\n14780\r\n14781\r\n14782\r\n14783\r\n14784\r\n14785\r\n14786\r\n14787\r\n14788\r\n14789\r\n14790\r\n14791\r\n14792\r\n14793\r\n14794\r\n14795\r\n14796\r\n14797\r\n14798\r\n14799\r\n14800\r\n14801\r\n14802\r\n14803\r\n14804\r\n14805\r\n14806\r\n14807\r\n14808\r\n14809\r\n14810\r\n14811\r\n14812\r\n14813\r\n14814\r\n14815\r\n14816\r\n14817\r\n14818\r\n14819\r\n14820\r\n14821\r\n14822\r\n14823\r\n14824\r\n14825\r\n14826\r\n14827\r\n14828\r\n14829\r\n14830\r\n14831\r\n14832\r\n14833\r\n14834\r\n14835\r\n14836\r\n14837\r\n14838\r\n14839\r\n14840\r\n14841\r\n14842\r\n14843\r\n14844\r\n14845\r\n14846\r\n14847\r\n14848\r\n14849\r\n14850\r\n14851\r\n14852\r\n14853\r\n14854\r\n14855\r\n14856\r\n14857\r\n14858\r\n14859\r\n14860\r\n14861\r\n14862\r\n14863\r\n14864\r\n14865\r\n14866\r\n14867\r\n14868\r\n14869\r\n14870\r\n14871\r\n14872\r\n14873\r\n14874\r\n14875\r\n14876\r\n14877\r\n14878\r\n14879\r\n14880\r\n14881\r\n14882\r\n14883\r\n14884\r\n14885\r\n14886\r\n14887\r\n14888\r\n14889\r\n14890\r\n14891\r\n14892\r\n14893\r\n14894\r\n14895\r\n14896\r\n14897\r\n14898\r\n14899\r\n14900\r\n14901\r\n14902\r\n14903\r\n14904\r\n14905\r\n14906\r\n14907\r\n14908\r\n14909\r\n14910\r\n14911\r\n14912\r\n14913\r\n14914\r\n14915\r\n14916\r\n14917\r\n14918\r\n14919\r\n14920\r\n14921\r\n14922\r\n14923\r\n14924\r\n14925\r\n14926\r\n14927\r\n14928\r\n14929\r\n14930\r\n14931\r\n14932\r\n14933\r\n14934\r\n14935\r\n14936\r\n14937\r\n14938\r\n14939\r\n14940\r\n14941\r\n14942\r\n14943\r\n14944\r\n14945\r\n14946\r\n14947\r\n14948\r\n14949\r\n14950\r\n14951\r\n14952\r\n14953\r\n14954\r\n14955\r\n14956\r\n14957\r\n14958\r\n14959\r\n14960\r\n14961\r\n14962\r\n14963\r\n14964\r\n14965\r\n14966\r\n14967\r\n14968\r\n14969\r\n14970\r\n14971\r\n14972\r\n14973\r\n14974\r\n14975\r\n14976\r\n14977\r\n14978\r\n14979\r\n14980\r\n14981\r\n14982\r\n14983\r\n14984\r\n14985\r\n14986\r\n14987\r\n14988\r\n14989\r\n14990\r\n14991\r\n14992\r\n14993\r\n14994\r\n14995\r\n14996\r\n14997\r\n14998\r\n14999\r\n15000\r\n15001\r\n15002\r\n15003\r\n15004\r\n15005\r\n15006\r\n15007\r\n15008\r\n15009\r\n15010\r\n15011\r\n15012\r\n15013\r\n15014\r\n15015\r\n15016\r\n15017\r\n15018\r\n15019\r\n15020\r\n15021\r\n15022\r\n15023\r\n15024\r\n15025\r\n15026\r\n15027\r\n15028\r\n15029\r\n15030\r\n15031\r\n15032\r\n15033\r\n15034\r\n15035\r\n15036\r\n15037\r\n15038\r\n15039\r\n15040\r\n15041\r\n15042\r\n15043\r\n15044\r\n15045\r\n15046\r\n15047\r\n15048\r\n15049\r\n15050\r\n15051\r\n15052\r\n15053\r\n15054\r\n15055\r\n15056\r\n15057\r\n15058\r\n15059\r\n15060\r\n15061\r\n15062\r\n15063\r\n15064\r\n15065\r\n15066\r\n15067\r\n15068\r\n15069\r\n15070\r\n15071\r\n15072\r\n15073\r\n15074\r\n15075\r\n15076\r\n15077\r\n15078\r\n15079\r\n15080\r\n15081\r\n15082\r\n15083\r\n15084\r\n15085\r\n15086\r\n15087\r\n15088\r\n15089\r\n15090\r\n15091\r\n15092\r\n15093\r\n15094\r\n15095\r\n15096\r\n15097\r\n15098\r\n15099\r\n15100\r\n15101\r\n15102\r\n15103\r\n15104\r\n15105\r\n15106\r\n15107\r\n15108\r\n15109\r\n15110\r\n15111\r\n15112\r\n15113\r\n15114\r\n15115\r\n15116\r\n15117\r\n15118\r\n15119\r\n15120\r\n15121\r\n15122\r\n15123\r\n15124\r\n15125\r\n15126\r\n15127\r\n15128\r\n15129\r\n15130\r\n15131\r\n15132\r\n15133\r\n15134\r\n15135\r\n15136\r\n15137\r\n15138\r\n15139\r\n15140\r\n15141\r\n15142\r\n15143\r\n15144\r\n15145\r\n15146\r\n15147\r\n15148\r\n15149\r\n15150\r\n15151\r\n15152\r\n15153\r\n15154\r\n15155\r\n15156\r\n15157\r\n15158\r\n15159\r\n15160\r\n15161\r\n15162\r\n15163\r\n15164\r\n15165\r\n15166\r\n15167\r\n15168\r\n15169\r\n15170\r\n15171\r\n15172\r\n15173\r\n15174\r\n15175\r\n15176\r\n15177\r\n15178\r\n15179\r\n15180\r\n15181\r\n15182\r\n15183\r\n15184\r\n15185\r\n15186\r\n15187\r\n15188\r\n15189\r\n15190\r\n15191\r\n15192\r\n15193\r\n15194\r\n15195\r\n15196\r\n15197\r\n15198\r\n15199\r\n15200\r\n15201\r\n15202\r\n15203\r\n15204\r\n15205\r\n15206\r\n15207\r\n15208\r\n15209\r\n15210\r\n15211\r\n15212\r\n15213\r\n15214\r\n15215\r\n15216\r\n15217\r\n15218\r\n15219\r\n15220\r\n15221\r\n15222\r\n15223\r\n15224\r\n15225\r\n15226\r\n15227\r\n15228\r\n15229\r\n15230\r\n15231\r\n15232\r\n15233\r\n15234\r\n15235\r\n15236\r\n15237\r\n15238\r\n15239\r\n15240\r\n15241\r\n15242\r\n15243\r\n15244\r\n15245\r\n15246\r\n15247\r\n15248\r\n15249\r\n15250\r\n15251\r\n15252\r\n15253\r\n15254\r\n15255\r\n15256\r\n15257\r\n15258\r\n15259\r\n15260\r\n15261\r\n15262\r\n15263\r\n15264\r\n15265\r\n15266\r\n15267\r\n15268\r\n15269\r\n15270\r\n15271\r\n15272\r\n15273\r\n15274\r\n15275\r\n15276\r\n15277\r\n15278\r\n15279\r\n15280\r\n15281\r\n15282\r\n15283\r\n15284\r\n15285\r\n15286\r\n15287\r\n15288\r\n15289\r\n15290\r\n15291\r\n15292\r\n15293\r\n15294\r\n15295\r\n15296\r\n15297\r\n15298\r\n15299\r\n15300\r\n15301\r\n15302\r\n15303\r\n15304\r\n15305\r\n15306\r\n15307\r\n15308\r\n15309\r\n15310\r\n15311\r\n15312\r\n15313\r\n15314\r\n15315\r\n15316\r\n15317\r\n15318\r\n15319\r\n15320\r\n15321\r\n15322\r\n15323\r\n15324\r\n15325\r\n15326\r\n15327\r\n15328\r\n15329\r\n15330\r\n15331\r\n15332\r\n15333\r\n15334\r\n15335\r\n15336\r\n15337\r\n15338\r\n15339\r\n15340\r\n15341\r\n15342\r\n15343\r\n15344\r\n15345\r\n15346\r\n15347\r\n15348\r\n15349\r\n15350\r\n15351\r\n15352\r\n15353\r\n15354\r\n15355\r\n15356\r\n15357\r\n15358\r\n15359\r\n15360\r\n15361\r\n15362\r\n15363\r\n15364\r\n15365\r\n15366\r\n15367\r\n15368\r\n15369\r\n15370\r\n15371\r\n15372\r\n15373\r\n15374\r\n15375\r\n15376\r\n15377\r\n15378\r\n15379\r\n15380\r\n15381\r\n15382\r\n15383\r\n15384\r\n15385\r\n15386\r\n15387\r\n15388\r\n15389\r\n15390\r\n15391\r\n15392\r\n15393\r\n15394\r\n15395\r\n15396\r\n15397\r\n15398\r\n15399\r\n15400\r\n15401\r\n15402\r\n15403\r\n15404\r\n15405\r\n15406\r\n15407\r\n15408\r\n15409\r\n15410\r\n15411\r\n15412\r\n15413\r\n15414\r\n15415\r\n15416\r\n15417\r\n15418\r\n15419\r\n15420\r\n15421\r\n15422\r\n15423\r\n15424\r\n15425\r\n15426\r\n15427\r\n15428\r\n15429\r\n15430\r\n15431\r\n15432\r\n15433\r\n15434\r\n15435\r\n15436\r\n15437\r\n15438\r\n15439\r\n15440\r\n15441\r\n15442\r\n15443\r\n15444\r\n15445\r\n15446\r\n15447\r\n15448\r\n15449\r\n15450\r\n15451\r\n15452\r\n15453\r\n15454\r\n15455\r\n15456\r\n15457\r\n15458\r\n15459\r\n15460\r\n15461\r\n15462\r\n15463\r\n15464\r\n15465\r\n15466\r\n15467\r\n15468\r\n15469\r\n15470\r\n15471\r\n15472\r\n15473\r\n15474\r\n15475\r\n15476\r\n15477\r\n15478\r\n15479\r\n15480\r\n15481\r\n15482\r\n15483\r\n15484\r\n15485\r\n15486\r\n15487\r\n15488\r\n15489\r\n15490\r\n15491\r\n15492\r\n15493\r\n15494\r\n15495\r\n15496\r\n15497\r\n15498\r\n15499\r\n15500\r\n15501\r\n15502\r\n15503\r\n15504\r\n15505\r\n15506\r\n15507\r\n15508\r\n15509\r\n15510\r\n15511\r\n15512\r\n15513\r\n15514\r\n15515\r\n15516\r\n15517\r\n15518\r\n15519\r\n15520\r\n15521\r\n15522\r\n15523\r\n15524\r\n15525\r\n15526\r\n15527\r\n15528\r\n15529\r\n15530\r\n15531\r\n15532\r\n15533\r\n15534\r\n15535\r\n15536\r\n15537\r\n15538\r\n15539\r\n15540\r\n15541\r\n15542\r\n15543\r\n15544\r\n15545\r\n15546\r\n15547\r\n15548\r\n15549\r\n15550\r\n15551\r\n15552\r\n15553\r\n15554\r\n15555\r\n15556\r\n15557\r\n15558\r\n15559\r\n15560\r\n15561\r\n15562\r\n15563\r\n15564\r\n15565\r\n15566\r\n15567\r\n15568\r\n15569\r\n15570\r\n15571\r\n15572\r\n15573\r\n15574\r\n15575\r\n15576\r\n15577\r\n15578\r\n15579\r\n15580\r\n15581\r\n15582\r\n15583\r\n15584\r\n15585\r\n15586\r\n15587\r\n15588\r\n15589\r\n15590\r\n15591\r\n15592\r\n15593\r\n15594\r\n15595\r\n15596\r\n15597\r\n15598\r\n15599\r\n15600\r\n15601\r\n15602\r\n15603\r\n15604\r\n15605\r\n15606\r\n15607\r\n15608\r\n15609\r\n15610\r\n15611\r\n15612\r\n15613\r\n15614\r\n15615\r\n15616\r\n15617\r\n15618\r\n15619\r\n15620\r\n15621\r\n15622\r\n15623\r\n15624\r\n15625\r\n15626\r\n15627\r\n15628\r\n15629\r\n15630\r\n15631\r\n15632\r\n15633\r\n15634\r\n15635\r\n15636\r\n15637\r\n15638\r\n15639\r\n15640\r\n15641\r\n15642\r\n15643\r\n15644\r\n15645\r\n15646\r\n15647\r\n15648\r\n15649\r\n15650\r\n15651\r\n15652\r\n15653\r\n15654\r\n15655\r\n15656\r\n15657\r\n15658\r\n15659\r\n15660\r\n15661\r\n15662\r\n15663\r\n15664\r\n15665\r\n15666\r\n15667\r\n15668\r\n15669\r\n15670\r\n15671\r\n15672\r\n15673\r\n15674\r\n15675\r\n15676\r\n15677\r\n15678\r\n15679\r\n15680\r\n15681\r\n15682\r\n15683\r\n15684\r\n15685\r\n15686\r\n15687\r\n15688\r\n15689\r\n15690\r\n15691\r\n15692\r\n15693\r\n15694\r\n15695\r\n15696\r\n15697\r\n15698\r\n15699\r\n15700\r\n15701\r\n15702\r\n15703\r\n15704\r\n15705\r\n15706\r\n15707\r\n15708\r\n15709\r\n15710\r\n15711\r\n15712\r\n15713\r\n15714\r\n15715\r\n15716\r\n15717\r\n15718\r\n15719\r\n15720\r\n15721\r\n15722\r\n15723\r\n15724\r\n15725\r\n15726\r\n15727\r\n15728\r\n15729\r\n15730\r\n15731\r\n15732\r\n15733\r\n15734\r\n15735\r\n15736\r\n15737\r\n15738\r\n15739\r\n15740\r\n15741\r\n15742\r\n15743\r\n15744\r\n15745\r\n15746\r\n15747\r\n15748\r\n15749\r\n15750\r\n15751\r\n15752\r\n15753\r\n15754\r\n15755\r\n15756\r\n15757\r\n15758\r\n15759\r\n15760\r\n15761\r\n15762\r\n15763\r\n15764\r\n15765\r\n15766\r\n15767\r\n15768\r\n15769\r\n15770\r\n15771\r\n15772\r\n15773\r\n15774\r\n15775\r\n15776\r\n15777\r\n15778\r\n15779\r\n15780\r\n15781\r\n15782\r\n15783\r\n15784\r\n15785\r\n15786\r\n15787\r\n15788\r\n15789\r\n15790\r\n15791\r\n15792\r\n15793\r\n15794\r\n15795\r\n15796\r\n15797\r\n15798\r\n15799\r\n15800\r\n15801\r\n15802\r\n15803\r\n15804\r\n15805\r\n15806\r\n15807\r\n15808\r\n15809\r\n15810\r\n15811\r\n15812\r\n15813\r\n15814\r\n15815\r\n15816\r\n15817\r\n15818\r\n15819\r\n15820\r\n15821\r\n15822\r\n15823\r\n15824\r\n15825\r\n15826\r\n15827\r\n15828\r\n15829\r\n15830\r\n15831\r\n15832\r\n15833\r\n15834\r\n15835\r\n15836\r\n15837\r\n15838\r\n15839\r\n15840\r\n15841\r\n15842\r\n15843\r\n15844\r\n15845\r\n15846\r\n15847\r\n15848\r\n15849\r\n15850\r\n15851\r\n15852\r\n15853\r\n15854\r\n15855\r\n15856\r\n15857\r\n15858\r\n15859\r\n15860\r\n15861\r\n15862\r\n15863\r\n15864\r\n15865\r\n15866\r\n15867\r\n15868\r\n15869\r\n15870\r\n15871\r\n15872\r\n15873\r\n15874\r\n15875\r\n15876\r\n15877\r\n15878\r\n15879\r\n15880\r\n15881\r\n15882\r\n15883\r\n15884\r\n15885\r\n15886\r\n15887\r\n15888\r\n15889\r\n15890\r\n15891\r\n15892\r\n15893\r\n15894\r\n15895\r\n15896\r\n15897\r\n15898\r\n15899\r\n15900\r\n15901\r\n15902\r\n15903\r\n15904\r\n15905\r\n15906\r\n15907\r\n15908\r\n15909\r\n15910\r\n15911\r\n15912\r\n15913\r\n15914\r\n15915\r\n15916\r\n15917\r\n15918\r\n15919\r\n15920\r\n15921\r\n15922\r\n15923\r\n15924\r\n15925\r\n15926\r\n15927\r\n15928\r\n15929\r\n15930\r\n15931\r\n15932\r\n15933\r\n15934\r\n15935\r\n15936\r\n15937\r\n15938\r\n15939\r\n15940\r\n15941\r\n15942\r\n15943\r\n15944\r\n15945\r\n15946\r\n15947\r\n15948\r\n15949\r\n15950\r\n15951\r\n15952\r\n15953\r\n15954\r\n15955\r\n15956\r\n15957\r\n15958\r\n15959\r\n15960\r\n15961\r\n15962\r\n15963\r\n15964\r\n15965\r\n15966\r\n15967\r\n15968\r\n15969\r\n15970\r\n15971\r\n15972\r\n15973\r\n15974\r\n15975\r\n15976\r\n15977\r\n15978\r\n15979\r\n15980\r\n15981\r\n15982\r\n15983\r\n15984\r\n15985\r\n15986\r\n15987\r\n15988\r\n15989\r\n15990\r\n15991\r\n15992\r\n15993\r\n15994\r\n15995\r\n15996\r\n15997\r\n15998\r\n15999\r\n16000\r\n16001\r\n16002\r\n16003\r\n16004\r\n16005\r\n16006\r\n16007\r\n16008\r\n16009\r\n16010\r\n16011\r\n16012\r\n16013\r\n16014\r\n16015\r\n16016\r\n16017\r\n16018\r\n16019\r\n16020\r\n16021\r\n16022\r\n16023\r\n16024\r\n16025\r\n16026\r\n16027\r\n16028\r\n16029\r\n16030\r\n16031\r\n16032\r\n16033\r\n16034\r\n16035\r\n16036\r\n16037\r\n16038\r\n16039\r\n16040\r\n16041\r\n16042\r\n16043\r\n16044\r\n16045\r\n16046\r\n16047\r\n16048\r\n16049\r\n16050\r\n16051\r\n16052\r\n16053\r\n16054\r\n16055\r\n16056\r\n16057\r\n16058\r\n16059\r\n16060\r\n16061\r\n16062\r\n16063\r\n16064\r\n16065\r\n16066\r\n16067\r\n16068\r\n16069\r\n16070\r\n16071\r\n16072\r\n16073\r\n16074\r\n16075\r\n16076\r\n16077\r\n16078\r\n16079\r\n16080\r\n16081\r\n16082\r\n16083\r\n16084\r\n16085\r\n16086\r\n16087\r\n16088\r\n16089\r\n16090\r\n16091\r\n16092\r\n16093\r\n16094\r\n16095\r\n16096\r\n16097\r\n16098\r\n16099\r\n16100\r\n16101\r\n16102\r\n16103\r\n16104\r\n16105\r\n16106\r\n16107\r\n16108\r\n16109\r\n16110\r\n16111\r\n16112\r\n16113\r\n16114\r\n16115\r\n16116\r\n16117\r\n16118\r\n16119\r\n16120\r\n16121\r\n16122\r\n16123\r\n16124\r\n16125\r\n16126\r\n16127\r\n16128\r\n16129\r\n16130\r\n16131\r\n16132\r\n16133\r\n16134\r\n16135\r\n16136\r\n16137\r\n16138\r\n16139\r\n16140\r\n16141\r\n16142\r\n16143\r\n16144\r\n16145\r\n16146\r\n16147\r\n16148\r\n16149\r\n16150\r\n16151\r\n16152\r\n16153\r\n16154\r\n16155\r\n16156\r\n16157\r\n16158\r\n16159\r\n16160\r\n16161\r\n16162\r\n16163\r\n16164\r\n16165\r\n16166\r\n16167\r\n16168\r\n16169\r\n16170\r\n16171\r\n16172\r\n16173\r\n16174\r\n16175\r\n16176\r\n16177\r\n16178\r\n16179\r\n16180\r\n16181\r\n16182\r\n16183\r\n16184\r\n16185\r\n16186\r\n16187\r\n16188\r\n16189\r\n16190\r\n16191\r\n16192\r\n16193\r\n16194\r\n16195\r\n16196\r\n16197\r\n16198\r\n16199\r\n16200\r\n16201\r\n16202\r\n16203\r\n16204\r\n16205\r\n16206\r\n16207\r\n16208\r\n16209\r\n16210\r\n16211\r\n16212\r\n16213\r\n16214\r\n16215\r\n16216\r\n16217\r\n16218\r\n16219\r\n16220\r\n16221\r\n16222\r\n16223\r\n16224\r\n16225\r\n16226\r\n16227\r\n16228\r\n16229\r\n16230\r\n16231\r\n16232\r\n16233\r\n16234\r\n16235\r\n16236\r\n16237\r\n16238\r\n16239\r\n16240\r\n16241\r\n16242\r\n16243\r\n16244\r\n16245\r\n16246\r\n16247\r\n16248\r\n16249\r\n16250\r\n16251\r\n16252\r\n16253\r\n16254\r\n16255\r\n16256\r\n16257\r\n16258\r\n16259\r\n16260\r\n16261\r\n16262\r\n16263\r\n16264\r\n16265\r\n16266\r\n16267\r\n16268\r\n16269\r\n16270\r\n16271\r\n16272\r\n16273\r\n16274\r\n16275\r\n16276\r\n16277\r\n16278\r\n16279\r\n16280\r\n16281\r\n16282\r\n16283\r\n16284\r\n16285\r\n16286\r\n16287\r\n16288\r\n16289\r\n16290\r\n16291\r\n16292\r\n16293\r\n16294\r\n16295\r\n16296\r\n16297\r\n16298\r\n16299\r\n16300\r\n16301\r\n16302\r\n16303\r\n16304\r\n16305\r\n16306\r\n16307\r\n16308\r\n16309\r\n16310\r\n16311\r\n16312\r\n16313\r\n16314\r\n16315\r\n16316\r\n16317\r\n16318\r\n16319\r\n16320\r\n16321\r\n16322\r\n16323\r\n16324\r\n16325\r\n16326\r\n16327\r\n16328\r\n16329\r\n16330\r\n16331\r\n16332\r\n16333\r\n16334\r\n16335\r\n16336\r\n16337\r\n16338\r\n16339\r\n16340\r\n16341\r\n16342\r\n16343\r\n16344\r\n16345\r\n16346\r\n16347\r\n16348\r\n16349\r\n16350\r\n16351\r\n16352\r\n16353\r\n16354\r\n16355\r\n16356\r\n16357\r\n16358\r\n16359\r\n16360\r\n16361\r\n16362\r\n16363\r\n16364\r\n16365\r\n16366\r\n16367\r\n16368\r\n16369\r\n16370\r\n16371\r\n16372\r\n16373\r\n16374\r\n16375\r\n16376\r\n16377\r\n16378\r\n16379\r\n16380\r\n16381\r\n16382\r\n16383\r\n16384\r\n16385\r\n16386\r\n16387\r\n16388\r\n16389\r\n16390\r\n16391\r\n16392\r\n16393\r\n16394\r\n16395\r\n16396\r\n16397\r\n16398\r\n16399\r\n16400\r\n16401\r\n16402\r\n16403\r\n16404\r\n16405\r\n16406\r\n16407\r\n16408\r\n16409\r\n16410\r\n16411\r\n16412\r\n16413\r\n16414\r\n16415\r\n16416\r\n16417\r\n16418\r\n16419\r\n16420\r\n16421\r\n16422\r\n16423\r\n16424\r\n16425\r\n16426\r\n16427\r\n16428\r\n16429\r\n16430\r\n16431\r\n16432\r\n16433\r\n16434\r\n16435\r\n16436\r\n16437\r\n16438\r\n16439\r\n16440\r\n16441\r\n16442\r\n16443\r\n16444\r\n16445\r\n16446\r\n16447\r\n16448\r\n16449\r\n16450\r\n16451\r\n16452\r\n16453\r\n16454\r\n16455\r\n16456\r\n16457\r\n16458\r\n16459\r\n16460\r\n16461\r\n16462\r\n16463\r\n16464\r\n16465\r\n16466\r\n16467\r\n16468\r\n16469\r\n16470\r\n16471\r\n16472\r\n16473\r\n16474\r\n16475\r\n16476\r\n16477\r\n16478\r\n16479\r\n16480\r\n16481\r\n16482\r\n16483\r\n16484\r\n16485\r\n16486\r\n16487\r\n16488\r\n16489\r\n16490\r\n16491\r\n16492\r\n16493\r\n16494\r\n16495\r\n16496\r\n16497\r\n16498\r\n16499\r\n16500\r\n16501\r\n16502\r\n16503\r\n16504\r\n16505\r\n16506\r\n16507\r\n16508\r\n16509\r\n16510\r\n16511\r\n16512\r\n16513\r\n16514\r\n16515\r\n16516\r\n16517\r\n16518\r\n16519\r\n16520\r\n16521\r\n16522\r\n16523\r\n16524\r\n16525\r\n16526\r\n16527\r\n16528\r\n16529\r\n16530\r\n16531\r\n16532\r\n16533\r\n16534\r\n16535\r\n16536\r\n16537\r\n16538\r\n16539\r\n16540\r\n16541\r\n16542\r\n16543\r\n16544\r\n16545\r\n16546\r\n16547\r\n16548\r\n16549\r\n16550\r\n16551\r\n16552\r\n16553\r\n16554\r\n16555\r\n16556\r\n16557\r\n16558\r\n16559\r\n16560\r\n16561\r\n16562\r\n16563\r\n16564\r\n16565\r\n16566\r\n16567\r\n16568\r\n16569\r\n16570\r\n16571\r\n16572\r\n16573\r\n16574\r\n16575\r\n16576\r\n16577\r\n16578\r\n16579\r\n16580\r\n16581\r\n16582\r\n16583\r\n16584\r\n16585\r\n16586\r\n16587\r\n16588\r\n16589\r\n16590\r\n16591\r\n16592\r\n16593\r\n16594\r\n16595\r\n16596\r\n16597\r\n16598\r\n16599\r\n16600\r\n16601\r\n16602\r\n16603\r\n16604\r\n16605\r\n16606\r\n16607\r\n16608\r\n16609\r\n16610\r\n16611\r\n16612\r\n16613\r\n16614\r\n16615\r\n16616\r\n16617\r\n16618\r\n16619\r\n16620\r\n16621\r\n16622\r\n16623\r\n16624\r\n16625\r\n16626\r\n16627\r\n16628\r\n16629\r\n16630\r\n16631\r\n16632\r\n16633\r\n16634\r\n16635\r\n16636\r\n16637\r\n16638\r\n16639\r\n16640\r\n16641\r\n16642\r\n16643\r\n16644\r\n16645\r\n16646\r\n16647\r\n16648\r\n16649\r\n16650\r\n16651\r\n16652\r\n16653\r\n16654\r\n16655\r\n16656\r\n16657\r\n16658\r\n16659\r\n16660\r\n16661\r\n16662\r\n16663\r\n16664\r\n16665\r\n16666\r\n16667\r\n16668\r\n16669\r\n16670\r\n16671\r\n16672\r\n16673\r\n16674\r\n16675\r\n16676\r\n16677\r\n16678\r\n16679\r\n16680\r\n16681\r\n16682\r\n16683\r\n16684\r\n16685\r\n16686\r\n16687\r\n16688\r\n16689\r\n16690\r\n16691\r\n16692\r\n16693\r\n16694\r\n16695\r\n16696\r\n16697\r\n16698\r\n16699\r\n16700\r\n16701\r\n16702\r\n16703\r\n16704\r\n16705\r\n16706\r\n16707\r\n16708\r\n16709\r\n16710\r\n16711\r\n16712\r\n16713\r\n16714\r\n16715\r\n16716\r\n16717\r\n16718\r\n16719\r\n16720\r\n16721\r\n16722\r\n16723\r\n16724\r\n16725\r\n16726\r\n16727\r\n16728\r\n16729\r\n16730\r\n16731\r\n16732\r\n16733\r\n16734\r\n16735\r\n16736\r\n16737\r\n16738\r\n16739\r\n16740\r\n16741\r\n16742\r\n16743\r\n16744\r\n16745\r\n16746\r\n16747\r\n16748\r\n16749\r\n16750\r\n16751\r\n16752\r\n16753\r\n16754\r\n16755\r\n16756\r\n16757\r\n16758\r\n16759\r\n16760\r\n16761\r\n16762\r\n16763\r\n16764\r\n16765\r\n16766\r\n16767\r\n16768\r\n16769\r\n16770\r\n16771\r\n16772\r\n16773\r\n16774\r\n16775\r\n16776\r\n16777\r\n16778\r\n16779\r\n16780\r\n16781\r\n16782\r\n16783\r\n16784\r\n16785\r\n16786\r\n16787\r\n16788\r\n16789\r\n16790\r\n16791\r\n16792\r\n16793\r\n16794\r\n16795\r\n16796\r\n16797\r\n16798\r\n16799\r\n16800\r\n16801\r\n16802\r\n16803\r\n16804\r\n16805\r\n16806\r\n16807\r\n16808\r\n16809\r\n16810\r\n16811\r\n16812\r\n16813\r\n16814\r\n16815\r\n16816\r\n16817\r\n16818\r\n16819\r\n16820\r\n16821\r\n16822\r\n16823\r\n16824\r\n16825\r\n16826\r\n16827\r\n16828\r\n16829\r\n16830\r\n16831\r\n16832\r\n16833\r\n16834\r\n16835\r\n16836\r\n16837\r\n16838\r\n16839\r\n16840\r\n16841\r\n16842\r\n16843\r\n16844\r\n16845\r\n16846\r\n16847\r\n16848\r\n16849\r\n16850\r\n16851\r\n16852\r\n16853\r\n16854\r\n16855\r\n16856\r\n16857\r\n16858\r\n16859\r\n16860\r\n16861\r\n16862\r\n16863\r\n16864\r\n16865\r\n16866\r\n16867\r\n16868\r\n16869\r\n16870\r\n16871\r\n16872\r\n16873\r\n16874\r\n16875\r\n16876\r\n16877\r\n16878\r\n16879\r\n16880\r\n16881\r\n16882\r\n16883\r\n16884\r\n16885\r\n16886\r\n16887\r\n16888\r\n16889\r\n16890\r\n16891\r\n16892\r\n16893\r\n16894\r\n16895\r\n16896\r\n16897\r\n16898\r\n16899\r\n16900\r\n16901\r\n16902\r\n16903\r\n16904\r\n16905\r\n16906\r\n16907\r\n16908\r\n16909\r\n16910\r\n16911\r\n16912\r\n16913\r\n16914\r\n16915\r\n16916\r\n16917\r\n16918\r\n16919\r\n16920\r\n16921\r\n16922\r\n16923\r\n16924\r\n16925\r\n16926\r\n16927\r\n16928\r\n16929\r\n16930\r\n16931\r\n16932\r\n16933\r\n16934\r\n16935\r\n16936\r\n16937\r\n16938\r\n16939\r\n16940\r\n16941\r\n16942\r\n16943\r\n16944\r\n16945\r\n16946\r\n16947\r\n16948\r\n16949\r\n16950\r\n16951\r\n16952\r\n16953\r\n16954\r\n16955\r\n16956\r\n16957\r\n16958\r\n16959\r\n16960\r\n16961\r\n16962\r\n16963\r\n16964\r\n16965\r\n16966\r\n16967\r\n16968\r\n16969\r\n16970\r\n16971\r\n16972\r\n16973\r\n16974\r\n16975\r\n16976\r\n16977\r\n16978\r\n16979\r\n16980\r\n16981\r\n16982\r\n16983\r\n16984\r\n16985\r\n16986\r\n16987\r\n16988\r\n16989\r\n16990\r\n16991\r\n16992\r\n16993\r\n16994\r\n16995\r\n16996\r\n16997\r\n16998\r\n16999\r\n17000\r\n17001\r\n17002\r\n17003\r\n17004\r\n17005\r\n17006\r\n17007\r\n17008\r\n17009\r\n17010\r\n17011\r\n17012\r\n17013\r\n17014\r\n17015\r\n17016\r\n17017\r\n17018\r\n17019\r\n17020\r\n17021\r\n17022\r\n17023\r\n17024\r\n17025\r\n17026\r\n17027\r\n17028\r\n17029\r\n17030\r\n17031\r\n17032\r\n17033\r\n17034\r\n17035\r\n17036\r\n17037\r\n17038\r\n17039\r\n17040\r\n17041\r\n17042\r\n17043\r\n17044\r\n17045\r\n17046\r\n17047\r\n17048\r\n17049\r\n17050\r\n17051\r\n17052\r\n17053\r\n17054\r\n17055\r\n17056\r\n17057\r\n17058\r\n17059\r\n17060\r\n17061\r\n17062\r\n17063\r\n17064\r\n17065\r\n17066\r\n17067\r\n17068\r\n17069\r\n17070\r\n17071\r\n17072\r\n17073\r\n17074\r\n17075\r\n17076\r\n17077\r\n17078\r\n17079\r\n17080\r\n17081\r\n17082\r\n17083\r\n17084\r\n17085\r\n17086\r\n17087\r\n17088\r\n17089\r\n17090\r\n17091\r\n17092\r\n17093\r\n17094\r\n17095\r\n17096\r\n17097\r\n17098\r\n17099\r\n17100\r\n17101\r\n17102\r\n17103\r\n17104\r\n17105\r\n17106\r\n17107\r\n17108\r\n17109\r\n17110\r\n17111\r\n17112\r\n17113\r\n17114\r\n17115\r\n17116\r\n17117\r\n17118\r\n17119\r\n17120\r\n17121\r\n17122\r\n17123\r\n17124\r\n17125\r\n17126\r\n17127\r\n17128\r\n17129\r\n17130\r\n17131\r\n17132\r\n17133\r\n17134\r\n17135\r\n17136\r\n17137\r\n17138\r\n17139\r\n17140\r\n17141\r\n17142\r\n17143\r\n17144\r\n17145\r\n17146\r\n17147\r\n17148\r\n17149\r\n17150\r\n17151\r\n17152\r\n17153\r\n17154\r\n17155\r\n17156\r\n17157\r\n17158\r\n17159\r\n17160\r\n17161\r\n17162\r\n17163\r\n17164\r\n17165\r\n17166\r\n17167\r\n17168\r\n17169\r\n17170\r\n17171\r\n17172\r\n17173\r\n17174\r\n17175\r\n17176\r\n17177\r\n17178\r\n17179\r\n17180\r\n17181\r\n17182\r\n17183\r\n17184\r\n17185\r\n17186\r\n17187\r\n17188\r\n17189\r\n17190\r\n17191\r\n17192\r\n17193\r\n17194\r\n17195\r\n17196\r\n17197\r\n17198\r\n17199\r\n17200\r\n17201\r\n17202\r\n17203\r\n17204\r\n17205\r\n17206\r\n17207\r\n17208\r\n17209\r\n17210\r\n17211\r\n17212\r\n17213\r\n17214\r\n17215\r\n17216\r\n17217\r\n17218\r\n17219\r\n17220\r\n17221\r\n17222\r\n17223\r\n17224\r\n17225\r\n17226\r\n17227\r\n17228\r\n17229\r\n17230\r\n17231\r\n17232\r\n17233\r\n17234\r\n17235\r\n17236\r\n17237\r\n17238\r\n17239\r\n17240\r\n17241\r\n17242\r\n17243\r\n17244\r\n17245\r\n17246\r\n17247\r\n17248\r\n17249\r\n17250\r\n17251\r\n17252\r\n17253\r\n17254\r\n17255\r\n17256\r\n17257\r\n17258\r\n17259\r\n17260\r\n17261\r\n17262\r\n17263\r\n17264\r\n17265\r\n17266\r\n17267\r\n17268\r\n17269\r\n17270\r\n17271\r\n17272\r\n17273\r\n17274\r\n17275\r\n17276\r\n17277\r\n17278\r\n17279\r\n17280\r\n17281\r\n17282\r\n17283\r\n17284\r\n17285\r\n17286\r\n17287\r\n17288\r\n17289\r\n17290\r\n17291\r\n17292\r\n17293\r\n17294\r\n17295\r\n17296\r\n17297\r\n17298\r\n17299\r\n17300\r\n17301\r\n17302\r\n17303\r\n17304\r\n17305\r\n17306\r\n17307\r\n17308\r\n17309\r\n17310\r\n17311\r\n17312\r\n17313\r\n17314\r\n17315\r\n17316\r\n17317\r\n17318\r\n17319\r\n17320\r\n17321\r\n17322\r\n17323\r\n17324\r\n17325\r\n17326\r\n17327\r\n17328\r\n17329\r\n17330\r\n17331\r\n17332\r\n17333\r\n17334\r\n17335\r\n17336\r\n17337\r\n17338\r\n17339\r\n17340\r\n17341\r\n17342\r\n17343\r\n17344\r\n17345\r\n17346\r\n17347\r\n17348\r\n17349\r\n17350\r\n17351\r\n17352\r\n17353\r\n17354\r\n17355\r\n17356\r\n17357\r\n17358\r\n17359\r\n17360\r\n17361\r\n17362\r\n17363\r\n17364\r\n17365\r\n17366\r\n17367\r\n17368\r\n17369\r\n17370\r\n17371\r\n17372\r\n17373\r\n17374\r\n17375\r\n17376\r\n17377\r\n17378\r\n17379\r\n17380\r\n17381\r\n17382\r\n17383\r\n17384\r\n17385\r\n17386\r\n17387\r\n17388\r\n17389\r\n17390\r\n17391\r\n17392\r\n17393\r\n17394\r\n17395\r\n17396\r\n17397\r\n17398\r\n17399\r\n17400\r\n17401\r\n17402\r\n17403\r\n17404\r\n17405\r\n17406\r\n17407\r\n17408\r\n17409\r\n17410\r\n17411\r\n17412\r\n17413\r\n17414\r\n17415\r\n17416\r\n17417\r\n17418\r\n17419\r\n17420\r\n17421\r\n17422\r\n17423\r\n17424\r\n17425\r\n17426\r\n17427\r\n17428\r\n17429\r\n17430\r\n17431\r\n17432\r\n17433\r\n17434\r\n17435\r\n17436\r\n17437\r\n17438\r\n17439\r\n17440\r\n17441\r\n17442\r\n17443\r\n17444\r\n17445\r\n17446\r\n17447\r\n17448\r\n17449\r\n17450\r\n17451\r\n17452\r\n17453\r\n17454\r\n17455\r\n17456\r\n17457\r\n17458\r\n17459\r\n17460\r\n17461\r\n17462\r\n17463\r\n17464\r\n17465\r\n17466\r\n17467\r\n17468\r\n17469\r\n17470\r\n17471\r\n17472\r\n17473\r\n17474\r\n17475\r\n17476\r\n17477\r\n17478\r\n17479\r\n17480\r\n17481\r\n17482\r\n17483\r\n17484\r\n17485\r\n17486\r\n17487\r\n17488\r\n17489\r\n17490\r\n17491\r\n17492\r\n17493\r\n17494\r\n17495\r\n17496\r\n17497\r\n17498\r\n17499\r\n17500\r\n17501\r\n17502\r\n17503\r\n17504\r\n17505\r\n17506\r\n17507\r\n17508\r\n17509\r\n17510\r\n17511\r\n17512\r\n17513\r\n17514\r\n17515\r\n17516\r\n17517\r\n17518\r\n17519\r\n17520\r\n17521\r\n17522\r\n17523\r\n17524\r\n17525\r\n17526\r\n17527\r\n17528\r\n17529\r\n17530\r\n17531\r\n17532\r\n17533\r\n17534\r\n17535\r\n17536\r\n17537\r\n17538\r\n17539\r\n17540\r\n17541\r\n17542\r\n17543\r\n17544\r\n17545\r\n17546\r\n17547\r\n17548\r\n17549\r\n17550\r\n17551\r\n17552\r\n17553\r\n17554\r\n17555\r\n17556\r\n17557\r\n17558\r\n17559\r\n17560\r\n17561\r\n17562\r\n17563\r\n17564\r\n17565\r\n17566\r\n17567\r\n17568\r\n17569\r\n17570\r\n17571\r\n17572\r\n17573\r\n17574\r\n17575\r\n17576\r\n17577\r\n17578\r\n17579\r\n17580\r\n17581\r\n17582\r\n17583\r\n17584\r\n17585\r\n17586\r\n17587\r\n17588\r\n17589\r\n17590\r\n17591\r\n17592\r\n17593\r\n17594\r\n17595\r\n17596\r\n17597\r\n17598\r\n17599\r\n17600\r\n17601\r\n17602\r\n17603\r\n17604\r\n17605\r\n17606\r\n17607\r\n17608\r\n17609\r\n17610\r\n17611\r\n17612\r\n17613\r\n17614\r\n17615\r\n17616\r\n17617\r\n17618\r\n17619\r\n17620\r\n17621\r\n17622\r\n17623\r\n17624\r\n17625\r\n17626\r\n17627\r\n17628\r\n17629\r\n17630\r\n17631\r\n17632\r\n17633\r\n17634\r\n17635\r\n17636\r\n17637\r\n17638\r\n17639\r\n17640\r\n17641\r\n17642\r\n17643\r\n17644\r\n17645\r\n17646\r\n17647\r\n17648\r\n17649\r\n17650\r\n17651\r\n17652\r\n17653\r\n17654\r\n17655\r\n17656\r\n17657\r\n17658\r\n17659\r\n17660\r\n17661\r\n17662\r\n17663\r\n17664\r\n17665\r\n17666\r\n17667\r\n17668\r\n17669\r\n17670\r\n17671\r\n17672\r\n17673\r\n17674\r\n17675\r\n17676\r\n17677\r\n17678\r\n17679\r\n17680\r\n17681\r\n17682\r\n17683\r\n17684\r\n17685\r\n17686\r\n17687\r\n17688\r\n17689\r\n17690\r\n17691\r\n17692\r\n17693\r\n17694\r\n17695\r\n17696\r\n17697\r\n17698\r\n17699\r\n17700\r\n17701\r\n17702\r\n17703\r\n17704\r\n17705\r\n17706\r\n17707\r\n17708\r\n17709\r\n17710\r\n17711\r\n17712\r\n17713\r\n17714\r\n17715\r\n17716\r\n17717\r\n17718\r\n17719\r\n17720\r\n17721\r\n17722\r\n17723\r\n17724\r\n17725\r\n17726\r\n17727\r\n17728\r\n17729\r\n17730\r\n17731\r\n17732\r\n17733\r\n17734\r\n17735\r\n17736\r\n17737\r\n17738\r\n17739\r\n17740\r\n17741\r\n17742\r\n17743\r\n17744\r\n17745\r\n17746\r\n17747\r\n17748\r\n17749\r\n17750\r\n17751\r\n17752\r\n17753\r\n17754\r\n17755\r\n17756\r\n17757\r\n17758\r\n17759\r\n17760\r\n17761\r\n17762\r\n17763\r\n17764\r\n17765\r\n17766\r\n17767\r\n17768\r\n17769\r\n17770\r\n17771\r\n17772\r\n17773\r\n17774\r\n17775\r\n17776\r\n17777\r\n17778\r\n17779\r\n17780\r\n17781\r\n17782\r\n17783\r\n17784\r\n17785\r\n17786\r\n17787\r\n17788\r\n17789\r\n17790\r\n17791\r\n17792\r\n17793\r\n17794\r\n17795\r\n17796\r\n17797\r\n17798\r\n17799\r\n17800\r\n17801\r\n17802\r\n17803\r\n17804\r\n17805\r\n17806\r\n17807\r\n17808\r\n17809\r\n17810\r\n17811\r\n17812\r\n17813\r\n17814\r\n17815\r\n17816\r\n17817\r\n17818\r\n17819\r\n17820\r\n17821\r\n17822\r\n17823\r\n17824\r\n17825\r\n17826\r\n17827\r\n17828\r\n17829\r\n17830\r\n17831\r\n17832\r\n17833\r\n17834\r\n17835\r\n17836\r\n17837\r\n17838\r\n17839\r\n17840\r\n17841\r\n17842\r\n17843\r\n17844\r\n17845\r\n17846\r\n17847\r\n17848\r\n17849\r\n17850\r\n17851\r\n17852\r\n17853\r\n17854\r\n17855\r\n17856\r\n17857\r\n17858\r\n17859\r\n17860\r\n17861\r\n17862\r\n17863\r\n17864\r\n17865\r\n17866\r\n17867\r\n17868\r\n17869\r\n17870\r\n17871\r\n17872\r\n17873\r\n17874\r\n17875\r\n17876\r\n17877\r\n17878\r\n17879\r\n17880\r\n17881\r\n17882\r\n17883\r\n17884\r\n17885\r\n17886\r\n17887\r\n17888\r\n17889\r\n17890\r\n17891\r\n17892\r\n17893\r\n17894\r\n17895\r\n17896\r\n17897\r\n17898\r\n17899\r\n17900\r\n17901\r\n17902\r\n17903\r\n17904\r\n17905\r\n17906\r\n17907\r\n17908\r\n17909\r\n17910\r\n17911\r\n17912\r\n17913\r\n17914\r\n17915\r\n17916\r\n17917\r\n17918\r\n17919\r\n17920\r\n17921\r\n17922\r\n17923\r\n17924\r\n17925\r\n17926\r\n17927\r\n17928\r\n17929\r\n17930\r\n17931\r\n17932\r\n17933\r\n17934\r\n17935\r\n17936\r\n17937\r\n17938\r\n17939\r\n17940\r\n17941\r\n17942\r\n17943\r\n17944\r\n17945\r\n17946\r\n17947\r\n17948\r\n17949\r\n17950\r\n17951\r\n17952\r\n17953\r\n17954\r\n17955\r\n17956\r\n17957\r\n17958\r\n17959\r\n17960\r\n17961\r\n17962\r\n17963\r\n17964\r\n17965\r\n17966\r\n17967\r\n17968\r\n17969\r\n17970\r\n17971\r\n17972\r\n17973\r\n17974\r\n17975\r\n17976\r\n17977\r\n17978\r\n17979\r\n17980\r\n17981\r\n17982\r\n17983\r\n17984\r\n17985\r\n17986\r\n17987\r\n17988\r\n17989\r\n17990\r\n17991\r\n17992\r\n17993\r\n17994\r\n17995\r\n17996\r\n17997\r\n17998\r\n17999\r\n18000\r\n18001\r\n18002\r\n18003\r\n18004\r\n18005\r\n18006\r\n18007\r\n18008\r\n18009\r\n18010\r\n18011\r\n18012\r\n18013\r\n18014\r\n18015\r\n18016\r\n18017\r\n18018\r\n18019\r\n18020\r\n18021\r\n18022\r\n18023\r\n18024\r\n18025\r\n18026\r\n18027\r\n18028\r\n18029\r\n18030\r\n18031\r\n18032\r\n18033\r\n18034\r\n18035\r\n18036\r\n18037\r\n18038\r\n18039\r\n18040\r\n18041\r\n18042\r\n18043\r\n18044\r\n18045\r\n18046\r\n18047\r\n18048\r\n18049\r\n18050\r\n18051\r\n18052\r\n18053\r\n18054\r\n18055\r\n18056\r\n18057\r\n18058\r\n18059\r\n18060\r\n18061\r\n18062\r\n18063\r\n18064\r\n18065\r\n18066\r\n18067\r\n18068\r\n18069\r\n18070\r\n18071\r\n18072\r\n18073\r\n18074\r\n18075\r\n18076\r\n18077\r\n18078\r\n18079\r\n18080\r\n18081\r\n18082\r\n18083\r\n18084\r\n18085\r\n18086\r\n18087\r\n18088\r\n18089\r\n18090\r\n18091\r\n18092\r\n18093\r\n18094\r\n18095\r\n18096\r\n18097\r\n18098\r\n18099\r\n18100\r\n18101\r\n18102\r\n18103\r\n18104\r\n18105\r\n18106\r\n18107\r\n18108\r\n18109\r\n18110\r\n18111\r\n18112\r\n18113\r\n18114\r\n18115\r\n18116\r\n18117\r\n18118\r\n18119\r\n18120\r\n18121\r\n18122\r\n18123\r\n18124\r\n18125\r\n18126\r\n18127\r\n18128\r\n18129\r\n18130\r\n18131\r\n18132\r\n18133\r\n18134\r\n18135\r\n18136\r\n18137\r\n18138\r\n18139\r\n18140\r\n18141\r\n18142\r\n18143\r\n18144\r\n18145\r\n18146\r\n18147\r\n18148\r\n18149\r\n18150\r\n18151\r\n18152\r\n18153\r\n18154\r\n18155\r\n18156\r\n18157\r\n18158\r\n18159\r\n18160\r\n18161\r\n18162\r\n18163\r\n18164\r\n18165\r\n18166\r\n18167\r\n18168\r\n18169\r\n18170\r\n18171\r\n18172\r\n18173\r\n18174\r\n18175\r\n18176\r\n18177\r\n18178\r\n18179\r\n18180\r\n18181\r\n18182\r\n18183\r\n18184\r\n18185\r\n18186\r\n18187\r\n18188\r\n18189\r\n18190\r\n18191\r\n18192\r\n18193\r\n18194\r\n18195\r\n18196\r\n18197\r\n18198\r\n18199\r\n18200\r\n18201\r\n18202\r\n18203\r\n18204\r\n18205\r\n18206\r\n18207\r\n18208\r\n18209\r\n18210\r\n18211\r\n18212\r\n18213\r\n18214\r\n18215\r\n18216\r\n18217\r\n18218\r\n18219\r\n18220\r\n18221\r\n18222\r\n18223\r\n18224\r\n18225\r\n18226\r\n18227\r\n18228\r\n18229\r\n18230\r\n18231\r\n18232\r\n18233\r\n18234\r\n18235\r\n18236\r\n18237\r\n18238\r\n18239\r\n18240\r\n18241\r\n18242\r\n18243\r\n18244\r\n18245\r\n18246\r\n18247\r\n18248\r\n18249\r\n18250\r\n18251\r\n18252\r\n18253\r\n18254\r\n18255\r\n18256\r\n18257\r\n18258\r\n18259\r\n18260\r\n18261\r\n18262\r\n18263\r\n18264\r\n18265\r\n18266\r\n18267\r\n18268\r\n18269\r\n18270\r\n18271\r\n18272\r\n18273\r\n18274\r\n18275\r\n18276\r\n18277\r\n18278\r\n18279\r\n18280\r\n18281\r\n18282\r\n18283\r\n18284\r\n18285\r\n18286\r\n18287\r\n18288\r\n18289\r\n18290\r\n18291\r\n18292\r\n18293\r\n18294\r\n18295\r\n18296\r\n18297\r\n18298\r\n18299\r\n18300\r\n18301\r\n18302\r\n18303\r\n18304\r\n18305\r\n18306\r\n18307\r\n18308\r\n18309\r\n18310\r\n18311\r\n18312\r\n18313\r\n18314\r\n18315\r\n18316\r\n18317\r\n18318\r\n18319\r\n18320\r\n18321\r\n18322\r\n18323\r\n18324\r\n18325\r\n18326\r\n18327\r\n18328\r\n18329\r\n18330\r\n18331\r\n18332\r\n18333\r\n18334\r\n18335\r\n18336\r\n18337\r\n18338\r\n18339\r\n18340\r\n18341\r\n18342\r\n18343\r\n18344\r\n18345\r\n18346\r\n18347\r\n18348\r\n18349\r\n18350\r\n18351\r\n18352\r\n18353\r\n18354\r\n18355\r\n18356\r\n18357\r\n18358\r\n18359\r\n18360\r\n18361\r\n18362\r\n18363\r\n18364\r\n18365\r\n18366\r\n18367\r\n18368\r\n18369\r\n18370\r\n18371\r\n18372\r\n18373\r\n18374\r\n18375\r\n18376\r\n18377\r\n18378\r\n18379\r\n18380\r\n18381\r\n18382\r\n18383\r\n18384\r\n18385\r\n18386\r\n18387\r\n18388\r\n18389\r\n18390\r\n18391\r\n18392\r\n18393\r\n18394\r\n18395\r\n18396\r\n18397\r\n18398\r\n18399\r\n18400\r\n18401\r\n18402\r\n18403\r\n18404\r\n18405\r\n18406\r\n18407\r\n18408\r\n18409\r\n18410\r\n18411\r\n18412\r\n18413\r\n18414\r\n18415\r\n18416\r\n18417\r\n18418\r\n18419\r\n18420\r\n18421\r\n18422\r\n18423\r\n18424\r\n18425\r\n18426\r\n18427\r\n18428\r\n18429\r\n18430\r\n18431\r\n18432\r\n18433\r\n18434\r\n18435\r\n18436\r\n18437\r\n18438\r\n18439\r\n18440\r\n18441\r\n18442\r\n18443\r\n18444\r\n18445\r\n18446\r\n18447\r\n18448\r\n18449\r\n18450\r\n18451\r\n18452\r\n18453\r\n18454\r\n18455\r\n18456\r\n18457\r\n18458\r\n18459\r\n18460\r\n18461\r\n18462\r\n18463\r\n18464\r\n18465\r\n18466\r\n18467\r\n18468\r\n18469\r\n18470\r\n18471\r\n18472\r\n18473\r\n18474\r\n18475\r\n18476\r\n18477\r\n18478\r\n18479\r\n18480\r\n18481\r\n18482\r\n18483\r\n18484\r\n18485\r\n18486\r\n18487\r\n18488\r\n18489\r\n18490\r\n18491\r\n18492\r\n18493\r\n18494\r\n18495\r\n18496\r\n18497\r\n18498\r\n18499\r\n18500\r\n18501\r\n18502\r\n18503\r\n18504\r\n18505\r\n18506\r\n18507\r\n18508\r\n18509\r\n18510\r\n18511\r\n18512\r\n18513\r\n18514\r\n18515\r\n18516\r\n18517\r\n18518\r\n18519\r\n18520\r\n18521\r\n18522\r\n18523\r\n18524\r\n18525\r\n18526\r\n18527\r\n18528\r\n18529\r\n18530\r\n18531\r\n18532\r\n18533\r\n18534\r\n18535\r\n18536\r\n18537\r\n18538\r\n18539\r\n18540\r\n18541\r\n18542\r\n18543\r\n18544\r\n18545\r\n18546\r\n18547\r\n18548\r\n18549\r\n18550\r\n18551\r\n18552\r\n18553\r\n18554\r\n18555\r\n18556\r\n18557\r\n18558\r\n18559\r\n18560\r\n18561\r\n18562\r\n18563\r\n18564\r\n18565\r\n18566\r\n18567\r\n18568\r\n18569\r\n18570\r\n18571\r\n18572\r\n18573\r\n18574\r\n18575\r\n18576\r\n18577\r\n18578\r\n18579\r\n18580\r\n18581\r\n18582\r\n18583\r\n18584\r\n18585\r\n18586\r\n18587\r\n18588\r\n18589\r\n18590\r\n18591\r\n18592\r\n18593\r\n18594\r\n18595\r\n18596\r\n18597\r\n18598\r\n18599\r\n18600\r\n18601\r\n18602\r\n18603\r\n18604\r\n18605\r\n18606\r\n18607\r\n18608\r\n18609\r\n18610\r\n18611\r\n18612\r\n18613\r\n18614\r\n18615\r\n18616\r\n18617\r\n18618\r\n18619\r\n18620\r\n18621\r\n18622\r\n18623\r\n18624\r\n18625\r\n18626\r\n18627\r\n18628\r\n18629\r\n18630\r\n18631\r\n18632\r\n18633\r\n18634\r\n18635\r\n18636\r\n18637\r\n18638\r\n18639\r\n18640\r\n18641\r\n18642\r\n18643\r\n18644\r\n18645\r\n18646\r\n18647\r\n18648\r\n18649\r\n18650\r\n18651\r\n18652\r\n18653\r\n18654\r\n18655\r\n18656\r\n18657\r\n18658\r\n18659\r\n18660\r\n18661\r\n18662\r\n18663\r\n18664\r\n18665\r\n18666\r\n18667\r\n18668\r\n18669\r\n18670\r\n18671\r\n18672\r\n18673\r\n18674\r\n18675\r\n18676\r\n18677\r\n18678\r\n18679\r\n18680\r\n18681\r\n18682\r\n18683\r\n18684\r\n18685\r\n18686\r\n18687\r\n18688\r\n18689\r\n18690\r\n18691\r\n18692\r\n18693\r\n18694\r\n18695\r\n18696\r\n18697\r\n18698\r\n18699\r\n18700\r\n18701\r\n18702\r\n18703\r\n18704\r\n18705\r\n18706\r\n18707\r\n18708\r\n18709\r\n18710\r\n18711\r\n18712\r\n18713\r\n18714\r\n18715\r\n18716\r\n18717\r\n18718\r\n18719\r\n18720\r\n18721\r\n18722\r\n18723\r\n18724\r\n18725\r\n18726\r\n18727\r\n18728\r\n18729\r\n18730\r\n18731\r\n18732\r\n18733\r\n18734\r\n18735\r\n18736\r\n18737\r\n18738\r\n18739\r\n18740\r\n18741\r\n18742\r\n18743\r\n18744\r\n18745\r\n18746\r\n18747\r\n18748\r\n18749\r\n18750\r\n18751\r\n18752\r\n18753\r\n18754\r\n18755\r\n18756\r\n18757\r\n18758\r\n18759\r\n18760\r\n18761\r\n18762\r\n18763\r\n18764\r\n18765\r\n18766\r\n18767\r\n18768\r\n18769\r\n18770\r\n18771\r\n18772\r\n18773\r\n18774\r\n18775\r\n18776\r\n18777\r\n18778\r\n18779\r\n18780\r\n18781\r\n18782\r\n18783\r\n18784\r\n18785\r\n18786\r\n18787\r\n18788\r\n18789\r\n18790\r\n18791\r\n18792\r\n18793\r\n18794\r\n18795\r\n18796\r\n18797\r\n18798\r\n18799\r\n18800\r\n18801\r\n18802\r\n18803\r\n18804\r\n18805\r\n18806\r\n18807\r\n18808\r\n18809\r\n18810\r\n18811\r\n18812\r\n18813\r\n18814\r\n18815\r\n18816\r\n18817\r\n18818\r\n18819\r\n18820\r\n18821\r\n18822\r\n18823\r\n18824\r\n18825\r\n18826\r\n18827\r\n18828\r\n18829\r\n18830\r\n18831\r\n18832\r\n18833\r\n18834\r\n18835\r\n18836\r\n18837\r\n18838\r\n18839\r\n18840\r\n18841\r\n18842\r\n18843\r\n18844\r\n18845\r\n18846\r\n18847\r\n18848\r\n18849\r\n18850\r\n18851\r\n18852\r\n18853\r\n18854\r\n18855\r\n18856\r\n18857\r\n18858\r\n18859\r\n18860\r\n18861\r\n18862\r\n18863\r\n18864\r\n18865\r\n18866\r\n18867\r\n18868\r\n18869\r\n18870\r\n18871\r\n18872\r\n18873\r\n18874\r\n18875\r\n18876\r\n18877\r\n18878\r\n18879\r\n18880\r\n18881\r\n18882\r\n18883\r\n18884\r\n18885\r\n18886\r\n18887\r\n18888\r\n18889\r\n18890\r\n18891\r\n18892\r\n18893\r\n18894\r\n18895\r\n18896\r\n18897\r\n18898\r\n18899\r\n18900\r\n18901\r\n18902\r\n18903\r\n18904\r\n18905\r\n18906\r\n18907\r\n18908\r\n18909\r\n18910\r\n18911\r\n18912\r\n18913\r\n18914\r\n18915\r\n18916\r\n18917\r\n18918\r\n18919\r\n18920\r\n18921\r\n18922\r\n18923\r\n18924\r\n18925\r\n18926\r\n18927\r\n18928\r\n18929\r\n18930\r\n18931\r\n18932\r\n18933\r\n18934\r\n18935\r\n18936\r\n18937\r\n18938\r\n18939\r\n18940\r\n18941\r\n18942\r\n18943\r\n18944\r\n18945\r\n18946\r\n18947\r\n18948\r\n18949\r\n18950\r\n18951\r\n18952\r\n18953\r\n18954\r\n18955\r\n18956\r\n18957\r\n18958\r\n18959\r\n18960\r\n18961\r\n18962\r\n18963\r\n18964\r\n18965\r\n18966\r\n18967\r\n18968\r\n18969\r\n18970\r\n18971\r\n18972\r\n18973\r\n18974\r\n18975\r\n18976\r\n18977\r\n18978\r\n18979\r\n18980\r\n18981\r\n18982\r\n18983\r\n18984\r\n18985\r\n18986\r\n18987\r\n18988\r\n18989\r\n18990\r\n18991\r\n18992\r\n18993\r\n18994\r\n18995\r\n18996\r\n18997\r\n18998\r\n18999\r\n19000\r\n19001\r\n19002\r\n19003\r\n19004\r\n19005\r\n19006\r\n19007\r\n19008\r\n19009\r\n19010\r\n19011\r\n19012\r\n19013\r\n19014\r\n19015\r\n19016\r\n19017\r\n19018\r\n19019\r\n19020\r\n19021\r\n19022\r\n19023\r\n19024\r\n19025\r\n19026\r\n19027\r\n19028\r\n19029\r\n19030\r\n19031\r\n19032\r\n19033\r\n19034\r\n19035\r\n19036\r\n19037\r\n19038\r\n19039\r\n19040\r\n19041\r\n19042\r\n19043\r\n19044\r\n19045\r\n19046\r\n19047\r\n19048\r\n19049\r\n19050\r\n19051\r\n19052\r\n19053\r\n19054\r\n19055\r\n19056\r\n19057\r\n19058\r\n19059\r\n19060\r\n19061\r\n19062\r\n19063\r\n19064\r\n19065\r\n19066\r\n19067\r\n19068\r\n19069\r\n19070\r\n19071\r\n19072\r\n19073\r\n19074\r\n19075\r\n19076\r\n19077\r\n19078\r\n19079\r\n19080\r\n19081\r\n19082\r\n19083\r\n19084\r\n19085\r\n19086\r\n19087\r\n19088\r\n19089\r\n19090\r\n19091\r\n19092\r\n19093\r\n19094\r\n19095\r\n19096\r\n19097\r\n19098\r\n19099\r\n19100\r\n19101\r\n19102\r\n19103\r\n19104\r\n19105\r\n19106\r\n19107\r\n19108\r\n19109\r\n19110\r\n19111\r\n19112\r\n19113\r\n19114\r\n19115\r\n19116\r\n19117\r\n19118\r\n19119\r\n19120\r\n19121\r\n19122\r\n19123\r\n19124\r\n19125\r\n19126\r\n19127\r\n19128\r\n19129\r\n19130\r\n19131\r\n19132\r\n19133\r\n19134\r\n19135\r\n19136\r\n19137\r\n19138\r\n19139\r\n19140\r\n19141\r\n19142\r\n19143\r\n19144\r\n19145\r\n19146\r\n19147\r\n19148\r\n19149\r\n19150\r\n19151\r\n19152\r\n19153\r\n19154\r\n19155\r\n19156\r\n19157\r\n19158\r\n19159\r\n19160\r\n19161\r\n19162\r\n19163\r\n19164\r\n19165\r\n19166\r\n19167\r\n19168\r\n19169\r\n19170\r\n19171\r\n19172\r\n19173\r\n19174\r\n19175\r\n19176\r\n19177\r\n19178\r\n19179\r\n19180\r\n19181\r\n19182\r\n19183\r\n19184\r\n19185\r\n19186\r\n19187\r\n19188\r\n19189\r\n19190\r\n19191\r\n19192\r\n19193\r\n19194\r\n19195\r\n19196\r\n19197\r\n19198\r\n19199\r\n19200\r\n19201\r\n19202\r\n19203\r\n19204\r\n19205\r\n19206\r\n19207\r\n19208\r\n19209\r\n19210\r\n19211\r\n19212\r\n19213\r\n19214\r\n19215\r\n19216\r\n19217\r\n19218\r\n19219\r\n19220\r\n19221\r\n19222\r\n19223\r\n19224\r\n19225\r\n19226\r\n19227\r\n19228\r\n19229\r\n19230\r\n19231\r\n19232\r\n19233\r\n19234\r\n19235\r\n19236\r\n19237\r\n19238\r\n19239\r\n19240\r\n19241\r\n19242\r\n19243\r\n19244\r\n19245\r\n19246\r\n19247\r\n19248\r\n19249\r\n19250\r\n19251\r\n19252\r\n19253\r\n19254\r\n19255\r\n19256\r\n19257\r\n19258\r\n19259\r\n19260\r\n19261\r\n19262\r\n19263\r\n19264\r\n19265\r\n19266\r\n19267\r\n19268\r\n19269\r\n19270\r\n19271\r\n19272\r\n19273\r\n19274\r\n19275\r\n19276\r\n19277\r\n19278\r\n19279\r\n19280\r\n19281\r\n19282\r\n19283\r\n19284\r\n19285\r\n19286\r\n19287\r\n19288\r\n19289\r\n19290\r\n19291\r\n19292\r\n19293\r\n19294\r\n19295\r\n19296\r\n19297\r\n19298\r\n19299\r\n19300\r\n19301\r\n19302\r\n19303\r\n19304\r\n19305\r\n19306\r\n19307\r\n19308\r\n19309\r\n19310\r\n19311\r\n19312\r\n19313\r\n19314\r\n19315\r\n19316\r\n19317\r\n19318\r\n19319\r\n19320\r\n19321\r\n19322\r\n19323\r\n19324\r\n19325\r\n19326\r\n19327\r\n19328\r\n19329\r\n19330\r\n19331\r\n19332\r\n19333\r\n19334\r\n19335\r\n19336\r\n19337\r\n19338\r\n19339\r\n19340\r\n19341\r\n19342\r\n19343\r\n19344\r\n19345\r\n19346\r\n19347\r\n19348\r\n19349\r\n19350\r\n19351\r\n19352\r\n19353\r\n19354\r\n19355\r\n19356\r\n19357\r\n19358\r\n19359\r\n19360\r\n19361\r\n19362\r\n19363\r\n19364\r\n19365\r\n19366\r\n19367\r\n19368\r\n19369\r\n19370\r\n19371\r\n19372\r\n19373\r\n19374\r\n19375\r\n19376\r\n19377\r\n19378\r\n19379\r\n19380\r\n19381\r\n19382\r\n19383\r\n19384\r\n19385\r\n19386\r\n19387\r\n19388\r\n19389\r\n19390\r\n19391\r\n19392\r\n19393\r\n19394\r\n19395\r\n19396\r\n19397\r\n19398\r\n19399\r\n19400\r\n19401\r\n19402\r\n19403\r\n19404\r\n19405\r\n19406\r\n19407\r\n19408\r\n19409\r\n19410\r\n19411\r\n19412\r\n19413\r\n19414\r\n19415\r\n19416\r\n19417\r\n19418\r\n19419\r\n19420\r\n19421\r\n19422\r\n19423\r\n19424\r\n19425\r\n19426\r\n19427\r\n19428\r\n19429\r\n19430\r\n19431\r\n19432\r\n19433\r\n19434\r\n19435\r\n19436\r\n19437\r\n19438\r\n19439\r\n19440\r\n19441\r\n19442\r\n19443\r\n19444\r\n19445\r\n19446\r\n19447\r\n19448\r\n19449\r\n19450\r\n19451\r\n19452\r\n19453\r\n19454\r\n19455\r\n19456\r\n19457\r\n19458\r\n19459\r\n19460\r\n19461\r\n19462\r\n19463\r\n19464\r\n19465\r\n19466\r\n19467\r\n19468\r\n19469\r\n19470\r\n19471\r\n19472\r\n19473\r\n19474\r\n19475\r\n19476\r\n19477\r\n19478\r\n19479\r\n19480\r\n19481\r\n19482\r\n19483\r\n19484\r\n19485\r\n19486\r\n19487\r\n19488\r\n19489\r\n19490\r\n19491\r\n19492\r\n19493\r\n19494\r\n19495\r\n19496\r\n19497\r\n19498\r\n19499\r\n19500\r\n19501\r\n19502\r\n19503\r\n19504\r\n19505\r\n19506\r\n19507\r\n19508\r\n19509\r\n19510\r\n19511\r\n19512\r\n19513\r\n19514\r\n19515\r\n19516\r\n19517\r\n19518\r\n19519\r\n19520\r\n19521\r\n19522\r\n19523\r\n19524\r\n19525\r\n19526\r\n19527\r\n19528\r\n19529\r\n19530\r\n19531\r\n19532\r\n19533\r\n19534\r\n19535\r\n19536\r\n19537\r\n19538\r\n19539\r\n19540\r\n19541\r\n19542\r\n19543\r\n19544\r\n19545\r\n19546\r\n19547\r\n19548\r\n19549\r\n19550\r\n19551\r\n19552\r\n19553\r\n19554\r\n19555\r\n19556\r\n19557\r\n19558\r\n19559\r\n19560\r\n19561\r\n19562\r\n19563\r\n19564\r\n19565\r\n19566\r\n19567\r\n19568\r\n19569\r\n19570\r\n19571\r\n19572\r\n19573\r\n19574\r\n19575\r\n19576\r\n19577\r\n19578\r\n19579\r\n19580\r\n19581\r\n19582\r\n19583\r\n19584\r\n19585\r\n19586\r\n19587\r\n19588\r\n19589\r\n19590\r\n19591\r\n19592\r\n19593\r\n19594\r\n19595\r\n19596\r\n19597\r\n19598\r\n19599\r\n19600\r\n19601\r\n19602\r\n19603\r\n19604\r\n19605\r\n19606\r\n19607\r\n19608\r\n19609\r\n19610\r\n19611\r\n19612\r\n19613\r\n19614\r\n19615\r\n19616\r\n19617\r\n19618\r\n19619\r\n19620\r\n19621\r\n19622\r\n19623\r\n19624\r\n19625\r\n19626\r\n19627\r\n19628\r\n19629\r\n19630\r\n19631\r\n19632\r\n19633\r\n19634\r\n19635\r\n19636\r\n19637\r\n19638\r\n19639\r\n19640\r\n19641\r\n19642\r\n19643\r\n19644\r\n19645\r\n19646\r\n19647\r\n19648\r\n19649\r\n19650\r\n19651\r\n19652\r\n19653\r\n19654\r\n19655\r\n19656\r\n19657\r\n19658\r\n19659\r\n19660\r\n19661\r\n19662\r\n19663\r\n19664\r\n19665\r\n19666\r\n19667\r\n19668\r\n19669\r\n19670\r\n19671\r\n19672\r\n19673\r\n19674\r\n19675\r\n19676\r\n19677\r\n19678\r\n19679\r\n19680\r\n19681\r\n19682\r\n19683\r\n19684\r\n19685\r\n19686\r\n19687\r\n19688\r\n19689\r\n19690\r\n19691\r\n19692\r\n19693\r\n19694\r\n19695\r\n19696\r\n19697\r\n19698\r\n19699\r\n19700\r\n19701\r\n19702\r\n19703\r\n19704\r\n19705\r\n19706\r\n19707\r\n19708\r\n19709\r\n19710\r\n19711\r\n19712\r\n19713\r\n19714\r\n19715\r\n19716\r\n19717\r\n19718\r\n19719\r\n19720\r\n19721\r\n19722\r\n19723\r\n19724\r\n19725\r\n19726\r\n19727\r\n19728\r\n19729\r\n19730\r\n19731\r\n19732\r\n19733\r\n19734\r\n19735\r\n19736\r\n19737\r\n19738\r\n19739\r\n19740\r\n19741\r\n19742\r\n19743\r\n19744\r\n19745\r\n19746\r\n19747\r\n19748\r\n19749\r\n19750\r\n19751\r\n19752\r\n19753\r\n19754\r\n19755\r\n19756\r\n19757\r\n19758\r\n19759\r\n19760\r\n19761\r\n19762\r\n19763\r\n19764\r\n19765\r\n19766\r\n19767\r\n19768\r\n19769\r\n19770\r\n19771\r\n19772\r\n19773\r\n19774\r\n19775\r\n19776\r\n19777\r\n19778\r\n19779\r\n19780\r\n19781\r\n19782\r\n19783\r\n19784\r\n19785\r\n19786\r\n19787\r\n19788\r\n19789\r\n19790\r\n19791\r\n19792\r\n19793\r\n19794\r\n19795\r\n19796\r\n19797\r\n19798\r\n19799\r\n19800\r\n19801\r\n19802\r\n19803\r\n19804\r\n19805\r\n19806\r\n19807\r\n19808\r\n19809\r\n19810\r\n19811\r\n19812\r\n19813\r\n19814\r\n19815\r\n19816\r\n19817\r\n19818\r\n19819\r\n19820\r\n19821\r\n19822\r\n19823\r\n19824\r\n19825\r\n19826\r\n19827\r\n19828\r\n19829\r\n19830\r\n19831\r\n19832\r\n19833\r\n19834\r\n19835\r\n19836\r\n19837\r\n19838\r\n19839\r\n19840\r\n19841\r\n19842\r\n19843\r\n19844\r\n19845\r\n19846\r\n19847\r\n19848\r\n19849\r\n19850\r\n19851\r\n19852\r\n19853\r\n19854\r\n19855\r\n19856\r\n19857\r\n19858\r\n19859\r\n19860\r\n19861\r\n19862\r\n19863\r\n19864\r\n19865\r\n19866\r\n19867\r\n19868\r\n19869\r\n19870\r\n19871\r\n19872\r\n19873\r\n19874\r\n19875\r\n19876\r\n19877\r\n19878\r\n19879\r\n19880\r\n19881\r\n19882\r\n19883\r\n19884\r\n19885\r\n19886\r\n19887\r\n19888\r\n19889\r\n19890\r\n19891\r\n19892\r\n19893\r\n19894\r\n19895\r\n19896\r\n19897\r\n19898\r\n19899\r\n19900\r\n19901\r\n19902\r\n19903\r\n19904\r\n19905\r\n19906\r\n19907\r\n19908\r\n19909\r\n19910\r\n19911\r\n19912\r\n19913\r\n19914\r\n19915\r\n19916\r\n19917\r\n19918\r\n19919\r\n19920\r\n19921\r\n19922\r\n19923\r\n19924\r\n19925\r\n19926\r\n19927\r\n19928\r\n19929\r\n19930\r\n19931\r\n19932\r\n19933\r\n19934\r\n19935\r\n19936\r\n19937\r\n19938\r\n19939\r\n19940\r\n19941\r\n19942\r\n19943\r\n19944\r\n19945\r\n19946\r\n19947\r\n19948\r\n19949\r\n19950\r\n19951\r\n19952\r\n19953\r\n19954\r\n19955\r\n19956\r\n19957\r\n19958\r\n19959\r\n19960\r\n19961\r\n19962\r\n19963\r\n19964\r\n19965\r\n19966\r\n19967\r\n19968\r\n19969\r\n19970\r\n19971\r\n19972\r\n19973\r\n19974\r\n19975\r\n19976\r\n19977\r\n19978\r\n19979\r\n19980\r\n19981\r\n19982\r\n19983\r\n19984\r\n19985\r\n19986\r\n19987\r\n19988\r\n19989\r\n19990\r\n19991\r\n19992\r\n19993\r\n19994\r\n19995\r\n19996\r\n19997\r\n19998\r\n19999\r\n20000\r\nrouter1:~$ "} +{"t": 14.237, "dir": "send", "data": "ls /nonexistent\r", "phase": "command"} +{"t": 14.239, "dir": "recv", "data": "ls /nonexistent\r\nls: /nonexistent: No such file or directory\r\nrouter1:~$ "} +{"t": 16.24, "dir": "send", "data": "exit\r", "phase": "command"} +{"t": 16.242, "dir": "recv", "data": "exit\r\nlogout\r\n"} +{"t": 16.242, "dir": "close", "data": ""} diff --git a/tests/recordings/linux/busybox-1.37-ash-telnet.golden.json b/tests/recordings/linux/busybox-1.37-ash-telnet.golden.json new file mode 100644 index 00000000..7307e489 --- /dev/null +++ b/tests/recordings/linux/busybox-1.37-ash-telnet.golden.json @@ -0,0 +1,6 @@ +{ + "login": "TimeoutException", + "driver": "junos", + "os": "junos", + "commands": [] +} diff --git a/tests/recordings/linux/busybox-1.37-ash-telnet.jsonl b/tests/recordings/linux/busybox-1.37-ash-telnet.jsonl new file mode 100644 index 00000000..af1e952e --- /dev/null +++ b/tests/recordings/linux/busybox-1.37-ash-telnet.jsonl @@ -0,0 +1,25 @@ +{"format": "exscript-recording", "version": 1, "protocol": "telnet", "date": "2026-10-31", "sanitized": true, "device": {"os": "shell", "version": "BusyBox 1.37", "platform": "docker busybox:1.37, telnetd"}, "terminal": {"type": "dumb", "width": 80, "height": 24}} +{"t": 0.014, "dir": "recv", "data": "\u00ff\u00fd\u0001\u00ff\u00fd\u001f\u00ff\u00fb\u0001\u00ff\u00fb\u0003"} +{"t": 0.014, "dir": "send", "data": "\u00ff\u00fc\u0001", "phase": "negotiation"} +{"t": 0.014, "dir": "send", "data": "\u00ff\u00fb\u001f", "phase": "negotiation"} +{"t": 0.014, "dir": "send", "data": "\u00ff\u00fa\u001f\u0000P\u0000\u0018\u00ff\u00f0", "phase": "negotiation"} +{"t": 0.014, "dir": "send", "data": "\u00ff\u00fd\u0001", "phase": "negotiation"} +{"t": 0.014, "dir": "send", "data": "\u00ff\u00fd\u0003", "phase": "negotiation"} +{"t": 0.018, "dir": "recv", "data": "\r\r\nrouter3 login: "} +{"t": 2.061, "dir": "send", "data": "admin\r", "phase": "login"} +{"t": 2.062, "dir": "recv", "data": "admin\r\nPassword: "} +{"t": 4.073, "dir": "send", "data": "", "secret": true, "phase": "login"} +{"t": 4.074, "dir": "recv", "data": "\r\nWelcome to router3, an Exscript test host.\r\nrouter3:~$ "} +{"t": 6.088, "dir": "send", "data": "uname -sr\r", "phase": "command"} +{"t": 6.091, "dir": "recv", "data": "uname -sr\r\nLinux 6.18.54.1-microsoft-standard-WSL2\r\nrouter3:~$ "} +{"t": 8.101, "dir": "send", "data": "cat /etc/passwd\r", "phase": "command"} +{"t": 8.105, "dir": "recv", "data": "cat /etc/passwd\r\nroot:x:0:0:root:/root:/bin/sh\r\ndaemon:x:1:1:daemon:/usr/sbin:/bin/false\r\nbin:x:2:2:bin:/bin:/bin/false\r\nsys:x:3:3:sys:/dev:/bin/false\r\nsync:x:4:100:sync:/bin:/bin/sync\r\nmail:x:8:8:mail:/var/spool/mail:/bin/false\r\nwww-data:x:33:33:www-data:/var/www:/bin/false\r\noperator:x:37:37:Operator:/var:/bin/false\r\nnobody:x:65534:65534:nobody:/home:/bin/false\r\nadmin:x:1000:1000:Linux User,,,:/home/admin:/bin/sh\r\nrouter3:~$ "} +{"t": 10.107, "dir": "send", "data": "ls -l /\r", "phase": "command"} +{"t": 10.109, "dir": "recv", "data": "ls -l /\r\ntotal 44\r\ndrwxr-xr-x 2 root root 12288 Sep 26 2024 \u001b[1;34mbin\u001b[m\r\ndrwxr-xr-x 5 root root 340 Oct 31 08:38 \u001b[1;34mdev\u001b[m\r\ndrwxr-xr-x 1 root root 4096 Oct 31 08:38 \u001b[1;34metc\u001b[m\r\ndrwxr-xr-x 1 nobody nobody 4096 Oct 31 08:38 \u001b[1;34mhome\u001b[m\r\ndrwxr-xr-x 2 root root 4096 Sep 26 2024 \u001b[1;34mlib\u001b[m\r\nlrwxrwxrwx 1 root root 3 Sep 26 2024 \u001b[1;36mlib64\u001b[m -> \u001b[1;34mlib\u001b[m\r\ndr-xr-xr-x 397 root root 0 Oct 31 08:38 \u001b[1;34mproc\u001b[m\r\ndrwx------ 2 root root 4096 Sep 26 2024 \u001b[1;34mroot\u001b[m\r\ndr-xr-xr-x 12 root root 0 Oct 30 12:05 \u001b[1;34msys\u001b[m\r\ndrwxrwxrwt 2 root root 4096 Sep 26 2024 \u001b[1;34mtmp\u001b[m\r\ndrwxr-xr-x 4 root root 4096 Sep 26 2024 \u001b[1;34musr\u001b[m\r\ndrwxr-xr-x 4 root root 4096 Sep 26 2024 \u001b[1;34mvar\u001b[m\r\nrouter3:~$ "} +{"t": 12.12, "dir": "send", "data": "seq 1 5000\r", "phase": "command"} +{"t": 12.123, "dir": "recv", "data": "seq 1 5000\r\n1\r\n2\r\n3\r\n4\r\n5\r\n6\r\n7\r\n8\r\n9\r\n10\r\n11\r\n12\r\n13\r\n14\r\n15\r\n16\r\n17\r\n18\r\n19\r\n20\r\n21\r\n22\r\n23\r\n24\r\n25\r\n26\r\n27\r\n28\r\n29\r\n30\r\n31\r\n32\r\n33\r\n34\r\n35\r\n36\r\n37\r\n38\r\n39\r\n40\r\n41\r\n42\r\n43\r\n44\r\n45\r\n46\r\n47\r\n48\r\n49\r\n50\r\n51\r\n52\r\n53\r\n54\r\n55\r\n56\r\n57\r\n58\r\n59\r\n60\r\n61\r\n62\r\n63\r\n64\r\n65\r\n66\r\n67\r\n68\r\n69\r\n70\r\n71\r\n72\r\n73\r\n74\r\n75\r\n76\r\n77\r\n78\r\n79\r\n80\r\n81\r\n82\r\n83\r\n84\r\n85\r\n86\r\n87\r\n88\r\n89\r\n90\r\n91\r\n92\r\n93\r\n94\r\n95\r\n96\r\n97\r\n98\r\n99\r\n100\r\n101\r\n102\r\n103\r\n104\r\n105\r\n106\r\n107\r\n108\r\n109\r\n110\r\n111\r\n112\r\n113\r\n114\r\n115\r\n116\r\n117\r\n118\r\n119\r\n120\r\n121\r\n122\r\n123\r\n124\r\n125\r\n126\r\n127\r\n128\r\n129\r\n130\r\n131\r\n132\r\n133\r\n134\r\n135\r\n136\r\n137\r\n138\r\n139\r\n140\r\n141\r\n142\r\n143\r\n144\r\n145\r\n146\r\n147\r\n148\r\n149\r\n150\r\n151\r\n152\r\n153\r\n154\r\n155\r\n156\r\n157\r\n158\r\n159\r\n160\r\n161\r\n162\r\n163\r\n164\r\n165\r\n166\r\n167\r\n168\r\n169\r\n170\r\n171\r\n172\r\n173\r\n174\r\n175\r\n176\r\n177\r\n178\r\n179\r\n180\r\n181\r\n182\r\n183\r\n184\r\n185\r\n186\r\n187\r\n188\r\n189\r\n190\r\n191\r\n192\r\n193\r\n194\r\n195\r\n196\r\n197\r\n198\r\n199\r\n200\r\n201\r\n202\r\n203\r\n204\r\n205\r\n206\r\n207\r\n208\r\n209\r\n210\r\n211\r\n212\r\n213\r\n214\r\n215\r\n216\r\n217\r\n218\r\n219\r\n220\r\n221\r\n222\r\n223\r\n224\r\n225\r\n226\r\n227\r\n228\r\n229\r\n230\r\n231\r\n232\r\n233\r\n234\r\n235\r\n236\r\n237\r\n238\r\n239\r\n240\r\n241\r\n242\r\n243\r\n244\r\n245\r\n246\r\n247\r\n248\r\n249\r\n250\r\n251\r\n252\r\n253\r\n254\r\n255\r\n256\r\n257\r\n258\r\n259\r\n260\r\n261\r\n262\r\n263\r\n264\r\n265\r\n266\r\n267\r\n268\r\n269\r\n270\r\n271\r\n272\r\n273\r\n274\r\n275\r\n276\r\n277\r\n278\r\n279\r\n280\r\n281\r\n282\r\n283\r\n284\r\n285\r\n286\r\n287\r\n288\r\n289\r\n290\r\n291\r\n292\r\n293\r\n294\r\n295\r\n296\r\n297\r\n298\r\n299\r\n300\r\n301\r\n302\r\n303\r\n304\r\n305\r\n306\r\n307\r\n308\r\n309\r\n310\r\n311\r\n312\r\n313\r\n314\r\n315\r\n316\r\n317\r\n318\r\n319\r\n320\r\n321\r\n322\r\n323\r\n324\r\n325\r\n326\r\n327\r\n328\r\n329\r\n330\r\n331\r\n332\r\n333\r\n334\r\n335\r\n336\r\n337\r\n338\r\n339\r\n340\r\n341\r\n342\r\n343\r\n344\r\n345\r\n346\r\n347\r\n348\r\n349\r\n350\r\n351\r\n352\r\n353\r\n354\r\n355\r\n356\r\n357\r\n358\r\n359\r\n360\r\n361\r\n362\r\n363\r\n364\r\n365\r\n366\r\n367\r\n368\r\n369\r\n370\r\n371\r\n372\r\n373\r\n374\r\n375\r\n376\r\n377\r\n378\r\n379\r\n380\r\n381\r\n382\r\n383\r\n384\r\n385\r\n386\r\n387\r\n388\r\n389\r\n390\r\n391\r\n392\r\n393\r\n394\r\n395\r\n396\r\n397\r\n398\r\n399\r\n400\r\n401\r\n402\r\n403\r\n404\r\n405\r\n406\r\n407\r\n408\r\n409\r\n410\r\n411\r\n412\r\n413\r\n414\r\n415\r\n416\r\n417\r\n418\r\n419\r\n420\r\n421\r\n422\r\n423\r\n424\r\n425\r\n426\r\n427\r\n428\r\n429\r\n430\r\n431\r\n432\r\n433\r\n434\r\n435\r\n436\r\n437\r\n438\r\n439\r\n440\r\n441\r\n442\r\n443\r\n444\r\n445\r\n446\r\n447\r\n448\r\n449\r\n450\r\n451\r\n452\r\n453\r\n454\r\n455\r\n456\r\n457\r\n458\r\n459\r\n460\r\n461\r\n462\r\n463\r\n464\r\n465\r\n466\r\n467\r\n468\r\n469\r\n470\r\n471\r\n472\r\n473\r\n474\r\n475\r\n476\r\n477\r\n478\r\n479\r\n480\r\n481\r\n482\r\n483\r\n484\r\n485\r\n486\r\n487\r\n488\r\n489\r\n490\r\n491\r\n492\r\n493\r\n494\r\n495\r\n496\r\n497\r\n498\r\n499\r\n500\r\n501\r\n502\r\n503\r\n504\r\n505\r\n506\r\n507\r\n508\r\n509\r\n510\r\n511\r\n512\r\n513\r\n514\r\n515\r\n516\r\n517\r\n518\r\n519\r\n520\r\n521\r\n522\r\n523\r\n524\r\n525\r\n526\r\n527\r\n528\r\n529\r\n530\r\n531\r\n532\r\n533\r\n534\r\n535\r\n536\r\n537\r\n538\r\n539\r\n540\r\n541\r\n542\r\n543\r\n544\r\n545\r\n546\r\n547\r\n548\r\n549\r\n550\r\n551\r\n552\r\n553\r\n554\r\n555\r\n556\r\n557\r\n558\r\n559\r\n560\r\n561\r\n562\r\n563\r\n564\r\n565\r\n566\r\n567\r\n568\r\n569\r\n570\r\n571\r\n572\r\n573\r\n574\r\n575\r\n576\r\n577\r\n578\r\n579\r\n580\r\n581\r\n582\r\n583\r\n584\r\n585\r\n586\r\n587\r\n588\r\n589\r\n590\r\n591\r\n592\r\n593\r\n594\r\n595\r\n596\r\n597\r\n598\r\n599\r\n600\r\n601\r\n602\r\n603\r\n604\r\n605\r\n606\r\n607\r\n608\r\n609\r\n610\r\n611\r\n612\r\n613\r\n614\r\n615\r\n616\r\n617\r\n618\r\n619\r\n620\r\n621\r\n622\r\n623\r\n624\r\n625\r\n626\r\n627\r\n628\r\n629\r\n630\r\n631\r\n632\r\n633\r\n634\r\n635\r\n636\r\n637\r\n638\r\n639\r\n640\r\n641\r\n642\r\n643\r\n644\r\n645\r\n646\r\n647\r\n648\r\n649\r\n650\r\n651\r\n652\r\n653\r\n654\r\n655\r\n656\r\n657\r\n658\r\n659\r\n660\r\n661\r\n662\r\n663\r\n664\r\n665\r\n666\r\n667\r\n668\r\n669\r\n670\r\n671\r\n672\r\n673\r\n674\r\n675\r\n676\r\n677\r\n678\r\n679\r\n680\r\n681\r\n682\r\n683\r\n684\r\n685\r\n686\r\n687\r\n688\r\n689\r\n690\r\n691\r\n692\r\n693\r\n694\r\n695\r\n696\r\n697\r\n698\r\n699\r\n700\r\n701\r\n702\r\n703\r\n704\r\n705\r\n706\r\n707\r\n708\r\n709\r\n710\r\n711\r\n712\r\n713\r\n714\r\n715\r\n716\r\n717\r\n718\r\n719\r\n720\r\n721\r\n722\r\n723\r\n724\r\n725\r\n726\r\n727\r\n728\r\n729\r\n730\r\n731\r\n732\r\n733\r\n734\r\n735\r\n736\r\n737\r\n738\r\n739\r\n740\r\n741\r\n742\r\n743\r\n744\r\n745\r\n746\r\n747\r\n748\r\n749\r\n750\r\n751\r\n752\r\n753\r\n754\r\n755\r\n756\r\n757\r\n758\r\n759\r\n760\r\n761\r\n762\r\n763\r\n764\r\n765\r\n766\r\n767\r\n768\r\n769\r\n770\r\n771\r\n772\r\n773\r\n774\r\n775\r\n776\r\n777\r\n778\r\n779\r\n780\r\n781\r\n782\r\n783\r\n784\r\n785\r\n786\r\n787\r\n788\r\n789\r\n790\r\n791\r\n792\r\n793\r\n794\r\n795\r\n796\r\n797\r\n798\r\n799\r\n800\r\n801\r\n802\r\n803\r\n804\r\n805\r\n806\r\n807\r\n808\r\n809\r\n810\r\n811\r\n812\r\n813\r\n814\r\n815\r\n816\r\n817\r\n818\r\n819\r\n820\r\n821\r\n822\r\n823\r\n824\r\n825\r\n826\r\n827\r\n828\r\n829\r\n830\r\n831\r\n832\r\n833\r\n834\r\n835\r\n836\r\n837\r\n838\r\n839\r\n840\r\n841\r\n842\r\n843\r\n844\r\n845\r\n846\r\n847\r\n848\r\n849\r\n850\r\n851\r\n852\r\n853\r\n854\r\n855\r\n856\r\n857\r\n858\r\n859\r\n860\r\n861\r\n862\r\n863\r\n864\r\n865\r\n866\r\n867\r\n868\r\n869\r\n870\r\n871\r\n872\r\n873\r\n874\r\n875\r\n876\r\n877\r\n878\r\n879\r\n880\r\n881\r\n882\r\n883\r\n884\r\n885\r\n886\r\n887\r\n888\r\n889\r\n890\r\n891\r\n892\r\n893\r\n894\r\n895\r\n896\r\n897\r\n898\r\n899\r\n900\r\n901\r\n902\r\n903\r\n904\r\n905\r\n906\r\n907\r\n908\r\n909\r\n910\r\n911\r\n912\r\n913\r\n914\r\n915\r\n916\r\n917\r\n918\r\n919\r\n920\r\n921\r\n922\r\n923\r\n924\r\n925\r\n926\r\n927\r\n928\r\n929\r\n930\r\n931\r\n932\r\n933\r\n934\r\n935\r\n936\r\n937\r\n938\r\n939\r\n940\r\n941\r\n942\r\n943\r\n944\r\n945\r\n946\r\n947\r\n948\r\n949\r\n950\r\n951\r\n952\r\n953\r\n954\r\n955\r\n956\r\n957\r\n958\r\n959\r\n960\r\n961\r\n962\r\n963\r\n964\r\n965\r\n966\r\n967\r\n968\r\n969\r\n970\r\n971\r\n972\r\n973\r\n974\r\n975\r\n976\r\n977\r\n978\r\n979\r\n980\r\n981\r\n982\r\n983\r\n984\r\n985\r\n986\r\n987\r\n988\r\n989\r\n990\r\n991\r\n992\r\n993\r\n994\r\n995\r\n996\r\n997\r\n998\r\n999\r\n1000\r\n1001\r\n1002\r\n1003\r\n1004\r\n1005\r\n1006\r\n1007\r\n1008\r\n1009\r\n1010\r\n1011\r\n1012\r\n1013\r\n1014\r\n1015\r\n1016\r\n1017\r\n1018\r\n1019\r\n1020\r\n1021\r\n1022\r\n1023\r\n1024\r\n1025\r\n1026\r\n1027\r\n1028\r\n1029\r\n1030\r\n1031\r\n1032\r\n1033\r\n1034\r\n1035\r\n1036\r\n1037\r\n1038\r\n1039\r\n1040\r\n1041\r\n1042\r\n1043\r\n1044\r\n1045\r\n1046\r\n1047\r\n1048\r\n1049\r\n1050\r\n1051\r\n1052\r\n1053\r\n1054\r\n1055\r\n1056\r\n1057\r\n1058\r\n1059\r\n1060\r\n1061\r\n1062\r\n1063\r\n1064\r\n1065\r\n1066\r\n1067\r\n1068\r\n1069\r\n1070\r\n1071\r\n1072\r\n1073\r\n1074\r\n1075\r\n1076\r\n1077\r\n1078\r\n1079\r\n1080\r\n1081\r\n1082\r\n1083\r\n1084\r\n1085\r\n1086\r\n1087\r\n1088\r\n1089\r\n1090\r\n1091\r\n1092\r\n1093\r\n1094\r\n1095\r\n1096\r\n1097\r\n1098\r\n1099\r\n1100\r\n1101\r\n1102\r\n1103\r\n1104\r\n1105\r\n1106\r\n1107\r\n1108\r\n1109\r\n1110\r\n1111\r\n1112\r\n1113\r\n1114\r\n1115\r\n1116\r\n1117\r\n1118\r\n1119\r\n1120\r\n1121\r\n1122\r\n1123\r\n1124\r\n1125\r\n1126\r\n1127\r\n1128\r\n1129\r\n1130\r\n1131\r\n1132\r\n1133\r\n1134\r\n1135\r\n1136\r\n1137\r\n1138\r\n1139\r\n1140\r\n1141\r\n1142\r\n1143\r\n1144\r\n1145\r\n1146\r\n1147\r\n1148\r\n1149\r\n1150\r\n1151\r\n1152\r\n1153\r\n1154\r\n1155\r\n1156\r\n1157\r\n1158\r\n1159\r\n1160\r\n1161\r\n1162\r\n1163\r\n1164\r\n1165\r\n1166\r\n1167\r\n1168\r\n1169\r\n1170\r\n1171\r\n1172\r\n1173\r\n1174\r\n1175\r\n1176\r\n1177\r\n1178\r\n1179\r\n1180\r\n1181\r\n1182\r\n1183\r\n1184\r\n1185\r\n1186\r\n1187\r\n1188\r\n1189\r\n1190\r\n1191\r\n1192\r\n1193\r\n1194\r\n1195\r\n1196\r\n1197\r\n1198\r\n1199\r\n1200\r\n1201\r\n1202\r\n1203\r\n1204\r\n1205\r\n1206\r\n1207\r\n1208\r\n1209\r\n1210\r\n1211\r\n1212\r\n1213\r\n1214\r\n1215\r\n1216\r\n1217\r\n1218\r\n1219\r\n1220\r\n1221\r\n1222\r\n1223\r\n1224\r\n1225\r\n1226\r\n1227\r\n1228\r\n1229\r\n1230\r\n1231\r\n1232\r\n1233\r\n1234\r\n1235\r\n1236\r\n1237\r\n1238\r\n1239\r\n1240\r\n1241\r\n1242\r\n1243\r\n1244\r\n1245\r\n1246\r\n1247\r\n1248\r\n1249\r\n1250\r\n1251\r\n1252\r\n1253\r\n1254\r\n1255\r\n1256\r\n1257\r\n1258\r\n1259\r\n1260\r\n1261\r\n1262\r\n1263\r\n1264\r\n1265\r\n1266\r\n1267\r\n1268\r\n1269\r\n1270\r\n1271\r\n1272\r\n1273\r\n1274\r\n1275\r\n1276\r\n1277\r\n1278\r\n1279\r\n1280\r\n1281\r\n1282\r\n1283\r\n1284\r\n1285\r\n1286\r\n1287\r\n1288\r\n1289\r\n1290\r\n1291\r\n1292\r\n1293\r\n1294\r\n1295\r\n1296\r\n1297\r\n1298\r\n1299\r\n1300\r\n1301\r\n1302\r\n1303\r\n1304\r\n1305\r\n1306\r\n1307\r\n1308\r\n1309\r\n1310\r\n1311\r\n1312\r\n1313\r\n1314\r\n1315\r\n1316\r\n1317\r\n1318\r\n1319\r\n1320\r\n1321\r\n1322\r\n1323\r\n1324\r\n1325\r\n1326\r\n1327\r\n1328\r\n1329\r\n1330\r\n1331\r\n1332\r\n1333\r\n1334\r\n1335\r\n1336\r\n1337\r\n1338\r\n1339\r\n1340\r\n1341\r\n1342\r\n1343\r\n1344\r\n1345\r\n1346\r\n1347\r\n1348\r\n1349\r\n1350\r\n1351\r\n1352\r\n1353\r\n1354\r\n1355\r\n1356\r\n1357\r\n1358\r\n1359\r\n1360\r\n1361\r\n1362\r\n1363\r\n1364\r\n1365\r\n1366\r\n1367\r\n1368\r\n1369\r\n1370\r\n1371\r\n1372\r\n1373\r\n1374\r\n1375\r\n1376\r\n1377\r\n1378\r\n1379\r\n1380\r\n1381\r\n1382\r\n1383\r\n1384\r\n1385\r\n1386\r\n1387\r\n1388\r\n1389\r\n1390\r\n1391\r\n1392\r\n1393\r\n1394\r\n1395\r\n1396\r\n1397\r\n1398\r\n1399\r\n1400\r\n1401\r\n1402\r\n1403\r\n1404\r\n1405\r\n1406\r\n1407\r\n1408\r\n1409\r\n1410\r\n1411\r\n1412\r\n1413\r\n1414\r\n1415\r\n1416\r\n1417\r\n1418\r\n1419\r\n1420\r\n1421\r\n1422\r\n1423\r\n1424\r\n1425\r\n1426\r\n1427\r\n1428\r\n1429\r\n1430\r\n1431\r\n1432\r\n1433\r\n1434\r\n1435\r\n1436\r\n1437\r\n1438\r\n1439\r\n1440\r\n1441\r\n1442\r\n1443\r\n1444\r\n1445\r\n1446\r\n1447\r\n1448\r\n1449\r\n1450\r\n1451\r\n1452\r\n1453\r\n1454\r\n1455\r\n1456\r\n1457\r\n1458\r\n1459\r\n1460\r\n1461\r\n1462\r\n1463\r\n1464\r\n1465\r\n1466\r\n1467\r\n1468\r\n1469\r\n1470\r\n1471\r\n1472\r\n1473\r\n1474\r\n1475\r\n1476\r\n1477\r\n1478\r\n1479\r\n1480\r\n1481\r\n1482\r\n1483\r\n1484\r\n1485\r\n1486\r\n1487\r\n1488\r\n1489\r\n1490\r\n1491\r\n1492\r\n1493\r\n1494\r\n1495\r\n1496\r\n1497\r\n1498\r\n1499\r\n1500\r\n1501\r\n1502\r\n1503\r\n1504\r\n1505\r\n1506\r\n1507\r\n1508\r\n1509\r\n1510\r\n1511\r\n1512\r\n1513\r\n1514\r\n1515\r\n1516\r\n1517\r\n1518\r\n1519\r\n1520\r\n1521\r\n1522\r\n1523\r\n1524\r\n1525\r\n1526\r\n1527\r\n1528\r\n1529\r\n1530\r\n1531\r\n1532\r\n1533\r\n1534\r\n1535\r\n1536\r\n1537\r\n1538\r\n1539\r\n1540\r\n1541\r\n1542\r\n1543\r\n1544\r\n1545\r\n1546\r\n1547\r\n1548\r\n1549\r\n1550\r\n1551\r\n1552\r\n1553\r\n1554\r\n1555\r\n1556\r\n1557\r\n1558\r\n1559\r\n1560\r\n1561\r\n1562\r\n1563\r\n1564\r\n1565\r\n1566\r\n1567\r\n1568\r\n1569\r\n1570\r\n1571\r\n1572\r\n1573\r\n1574\r\n1575\r\n1576\r\n1577\r\n1578\r\n1579\r\n1580\r\n1581\r\n1582\r\n1583\r\n1584\r\n1585\r\n1586\r\n1587\r\n1588\r\n1589\r\n1590\r\n1591\r\n1592\r\n1593\r\n1594\r\n1595\r\n1596\r\n1597\r\n1598\r\n1599\r\n1600\r\n1601\r\n1602\r\n1603\r\n1604\r\n1605\r\n1606\r\n1607\r\n1608\r\n1609\r\n1610\r\n1611\r\n1612\r\n1613\r\n1614\r\n1615\r\n1616\r\n1617\r\n1618\r\n1619\r\n1620\r\n1621\r\n1622\r\n1623\r\n1624\r\n1625\r\n1626\r\n1627\r\n1628\r\n1629\r\n1630\r\n1631\r\n1632\r\n1633\r\n1634\r\n1635\r\n1636\r\n1637\r\n1638\r\n1639\r\n1640\r\n1641\r\n1642\r\n1643\r\n1644\r\n1645\r\n1646\r\n1647\r\n1648\r\n1649\r\n1650\r\n1651\r\n1652\r\n1653\r\n1654\r\n1655\r\n1656\r\n1657\r\n1658\r\n1659\r\n1660\r\n1661\r\n1662\r\n1663\r\n1664\r\n1665\r\n1666\r\n1667\r\n1668\r\n1669\r\n1670\r\n1671\r\n1672\r\n1673\r\n1674\r\n1675\r\n1676\r\n1677\r\n1678\r\n1679\r\n1680\r\n1681\r\n1682\r\n1683\r\n1684\r\n1685\r\n1686\r\n1687\r\n1688\r\n1689\r\n1690\r\n1691\r\n1692\r\n1693\r\n1694\r\n1695\r\n1696\r\n1697\r\n1698\r\n1699\r\n1700\r\n1701\r\n1702\r\n1703\r\n1704\r\n1705\r\n1706\r\n1707\r\n1708\r\n1709\r\n1710\r\n1711\r\n1712\r\n1713\r\n1714\r\n1715\r\n1716\r\n1717\r\n1718\r\n1719\r\n1720\r\n1721\r\n1722\r\n1723\r\n1724\r\n1725\r\n1726\r\n1727\r\n1728\r\n1729\r\n1730\r\n1731\r\n1732\r\n1733\r\n1734\r\n1735\r\n1736\r\n1737\r\n1738\r\n1739\r\n1740\r\n1741\r\n1742\r\n1743\r\n1744\r\n1745\r\n1746\r\n1747\r\n1748\r\n1749\r\n1750\r\n1751\r\n1752\r\n1753\r\n1754\r\n1755\r\n1756\r\n1757\r\n1758\r\n1759\r\n1760\r\n1761\r\n1762\r\n1763\r\n1764\r\n1765\r\n1766\r\n1767\r\n1768\r\n1769\r\n1770\r\n1771\r\n1772\r\n1773\r\n1774\r\n1775\r\n1776\r\n1777\r\n1778\r\n1779\r\n1780\r\n1781\r\n1782\r\n1783\r\n1784\r\n1785\r\n1786\r\n1787\r\n1788\r\n1789\r\n1790\r\n1791\r\n1792\r\n1793\r\n1794\r\n1795\r\n1796\r\n1797\r\n1798\r\n1799\r\n1800\r\n1801\r\n1802\r\n1803\r\n1804\r\n1805\r\n1806\r\n1807\r\n1808\r\n1809\r\n1810\r\n1811\r\n1812\r\n1813\r\n1814\r\n1815\r\n1816\r\n1817\r\n1818\r\n1819\r\n1820\r\n1821\r\n1822\r\n1823\r\n1824\r\n1825\r\n1826\r\n1827\r\n1828\r\n1829\r\n1830\r\n1831\r\n1832\r\n1833\r\n1834\r\n1835\r\n1836\r\n1837\r\n1838\r\n1839\r\n1840\r\n1841\r\n1842\r\n1843\r\n1844\r\n1845\r\n1846\r\n1847\r\n1848\r\n1849\r\n1850\r\n1851\r\n1852\r\n1853\r\n1854\r\n1855\r\n1856\r\n1857\r\n1858\r\n1859\r\n1860\r\n1861\r\n1862\r\n1863\r\n1864\r\n1865\r\n1866\r\n1867\r\n1868\r\n1869\r\n1870\r\n1871\r\n1872\r\n1873\r\n1874\r\n1875\r\n1876\r\n1877\r\n1878\r\n1879\r\n1880\r\n1881\r\n1882\r\n1883\r\n1884\r\n1885\r\n1886\r\n1887\r\n1888\r\n1889\r\n1890\r\n1891\r\n1892\r\n1893\r\n1894\r\n1895\r\n1896\r\n1897\r\n1898\r\n1899\r\n1900\r\n1901\r\n1902\r\n1903\r\n1904\r\n1905\r\n1906\r\n1907\r\n1908\r\n1909\r\n1910\r\n1911\r\n1912\r\n1913\r\n1914\r\n1915\r\n1916\r\n1917\r\n1918\r\n1919\r\n1920\r\n1921\r\n1922\r\n1923\r\n1924\r\n1925\r\n1926\r\n1927\r\n1928\r\n1929\r\n1930\r\n1931\r\n1932\r\n1933\r\n1934\r\n1935\r\n1936\r\n1937\r\n1938\r\n1939\r\n1940\r\n1941\r\n1942\r\n1943\r\n1944\r\n1945\r\n1946\r\n1947\r\n1948\r\n1949\r\n1950\r\n1951\r\n1952\r\n1953\r\n1954\r\n1955\r\n1956\r\n1957\r\n1958\r\n1959\r\n1960\r\n1961\r\n1962\r\n1963\r\n1964\r\n1965\r\n1966\r\n1967\r\n1968\r\n1969\r\n1970\r\n1971\r\n1972\r\n1973\r\n1974\r\n1975\r\n1976\r\n1977\r\n1978\r\n1979\r\n1980\r\n1981\r\n1982\r\n1983\r\n1984\r\n1985\r\n1986\r\n1987\r\n1988\r\n1989\r\n1990\r\n1991\r\n1992\r\n1993\r\n1994\r\n1995\r\n1996\r\n1997\r\n1998\r\n1999\r\n2000\r\n2001\r\n2002\r\n2003\r\n2004\r\n2005\r\n2006\r\n2007\r\n2008\r\n2009\r\n2010\r\n2011\r\n2012\r\n2013\r\n2014\r\n2015\r\n2016\r\n2017\r\n2018\r\n2019\r\n2020\r\n2021\r\n2022\r\n2023\r\n2024\r\n2025\r\n2026\r\n2027\r\n2028\r\n2029\r\n2030\r\n2031\r\n2032\r\n2033\r\n2034\r\n2035\r\n2036\r\n2037\r\n2038\r\n2039\r\n2040\r\n2041\r\n2042\r\n2043\r\n2044\r\n2045\r\n2046\r\n2047\r\n2048\r\n2049\r\n2050\r\n2051\r\n2052\r\n2053\r\n2054\r\n2055\r\n2056\r\n2057\r\n2058\r\n2059\r\n2060\r\n2061\r\n2062\r\n2063\r\n2064\r\n2065\r\n2066\r\n2067\r\n2068\r\n2069\r\n2070\r\n2071\r\n2072\r\n2073\r\n2074\r\n2075\r\n2076\r\n2077\r\n2078\r\n2079\r\n2080\r\n2081\r\n2082\r\n2083\r\n2084\r\n2085\r\n2086\r\n2087\r\n2088\r\n2089\r\n2090\r\n2091\r\n2092\r\n2093\r\n2094\r\n2095\r\n2096\r\n2097\r\n2098\r\n2099\r\n2100\r\n2101\r\n2102\r\n2103\r\n2104\r\n2105\r\n2106\r\n2107\r\n2108\r\n2109\r\n2110\r\n2111\r\n2112\r\n2113\r\n2114\r\n2115\r\n2116\r\n2117\r\n2118\r\n2119\r\n2120\r\n2121\r\n2122\r\n2123\r\n2124\r\n2125\r\n2126\r\n2127\r\n2128\r\n2129\r\n2130\r\n2131\r\n2132\r\n2133\r\n2134\r\n2135\r\n2136\r\n2137\r\n2138\r\n2139\r\n2140\r\n2141\r\n2142\r\n2143\r\n2144\r\n2145\r\n2146\r\n2147\r\n2148\r\n2149\r\n2150\r\n2151\r\n2152\r\n2153\r\n2154\r\n2155\r\n2156\r\n2157\r\n2158\r\n2159\r\n2160\r\n2161\r\n2162\r\n2163\r\n2164\r\n2165\r\n2166\r\n2167\r\n2168\r\n2169\r\n2170\r\n2171\r\n2172\r\n2173\r\n2174\r\n2175\r\n2176\r\n2177\r\n2178\r\n2179\r\n2180\r\n2181\r\n2182\r\n2183\r\n2184\r\n2185\r\n2186\r\n2187\r\n2188\r\n2189\r\n2190\r\n2191\r\n2192\r\n2193\r\n2194\r\n2195\r\n2196\r\n2197\r\n2198\r\n2199\r\n2200\r\n2201\r\n2202\r\n2203\r\n2204\r\n2205\r\n2206\r\n2207\r\n2208\r\n2209\r\n2210\r\n2211\r\n2212\r\n2213\r\n2214\r\n2215\r\n2216\r\n2217\r\n2218\r\n2219\r\n2220\r\n2221\r\n2222\r\n2223\r\n2224\r\n2225\r\n2226\r\n2227\r\n2228\r\n2229\r\n2230\r\n2231\r\n2232\r\n2233\r\n2234\r\n2235\r\n2236\r\n2237\r\n2238\r\n2239\r\n2240\r\n2241\r\n2242\r\n2243\r\n2244\r\n2245\r\n2246\r\n2247\r\n2248\r\n2249\r\n2250\r\n2251\r\n2252\r\n2253\r\n2254\r\n2255\r\n2256\r\n2257\r\n2258\r\n2259\r\n2260\r\n2261\r\n2262\r\n2263\r\n2264\r\n2265\r\n2266\r\n2267\r\n2268\r\n2269\r\n2270\r\n2271\r\n2272\r\n2273\r\n2274\r\n2275\r\n2276\r\n2277\r\n2278\r\n2279\r\n2280\r\n2281\r\n2282\r\n2283\r\n2284\r\n2285\r\n2286\r\n2287\r\n2288\r\n2289\r\n2290\r\n2291\r\n2292\r\n2293\r\n2294\r\n2295\r\n2296\r\n2297\r\n2298\r\n2299\r\n2300\r\n2301\r\n2302\r\n2303\r\n2304\r\n2305\r\n2306\r\n2307\r\n2308\r\n2309\r\n2310\r\n2311\r\n2312\r\n2313\r\n2314\r\n2315\r\n2316\r\n2317\r\n2318\r\n2319\r\n2320\r\n2321\r\n2322\r\n2323\r\n2324\r\n2325\r\n2326\r\n2327\r\n2328\r\n2329\r\n2330\r\n2331\r\n2332\r\n2333\r\n2334\r\n2335\r\n2336\r\n2337\r\n2338\r\n2339\r\n2340\r\n2341\r\n2342\r\n2343\r\n2344\r\n2345\r\n2346\r\n2347\r\n2348\r\n2349\r\n2350\r\n2351\r\n2352\r\n2353\r\n2354\r\n2355\r\n2356\r\n2357\r\n2358\r\n2359\r\n2360\r\n2361\r\n2362\r\n2363\r\n2364\r\n2365\r\n2366\r\n2367\r\n2368\r\n2369\r\n2370\r\n2371\r\n2372\r\n2373\r\n2374\r\n2375\r\n2376\r\n2377\r\n2378\r\n2379\r\n2380\r\n2381\r\n2382\r\n2383\r\n2384\r\n2385\r\n2386\r\n2387\r\n2388\r\n2389\r\n2390\r\n2391\r\n2392\r\n2393\r\n2394\r\n2395\r\n2396\r\n2397\r\n2398\r\n2399\r\n2400\r\n2401\r\n2402\r\n2403\r\n2404\r\n2405\r\n2406\r\n2407\r\n2408\r\n2409\r\n2410\r\n2411\r\n2412\r\n2413\r\n2414\r\n2415\r\n2416\r\n2417\r\n2418\r\n2419\r\n2420\r\n2421\r\n2422\r\n2423\r\n2424\r\n2425\r\n2426\r\n2427\r\n2428\r\n2429\r\n2430\r\n2431\r\n2432\r\n2433\r\n2434\r\n2435\r\n2436\r\n2437\r\n2438\r\n2439\r\n2440\r\n2441\r\n2442\r\n2443\r\n2444\r\n2445\r\n2446\r\n2447\r\n2448\r\n2449\r\n2450\r\n2451\r\n2452\r\n2453\r\n2454\r\n2455\r\n2456\r\n2457\r\n2458\r\n2459\r\n2460\r\n2461\r\n2462\r\n2463\r\n2464\r\n2465\r\n2466\r\n2467\r\n2468\r\n2469\r\n2470\r\n2471\r\n2472\r\n2473\r\n2474\r\n2475\r\n2476\r\n2477\r\n2478\r\n2479\r\n2480\r\n2481\r\n2482\r\n2483\r\n2484\r\n2485\r\n2486\r\n2487\r\n2488\r\n2489\r\n2490\r\n2491\r\n2492\r\n2493\r\n2494\r\n2495\r\n2496\r\n2497\r\n2498\r\n2499\r\n2500\r\n2501\r\n2502\r\n2503\r\n2504\r\n2505\r\n2506\r\n2507\r\n2508\r\n2509\r\n2510\r\n2511\r\n2512\r\n2513\r\n2514\r\n2515\r\n2516\r\n2517\r\n2518\r\n2519\r\n2520\r\n2521\r\n2522\r\n2523\r\n2524\r\n2525\r\n2526\r\n2527\r\n2528\r\n2529\r\n2530\r\n2531\r\n2532\r\n2533\r\n2534\r\n2535\r\n2536\r\n2537\r\n2538\r\n2539\r\n2540\r\n2541\r\n2542\r\n2543\r\n2544\r\n2545\r\n2546\r\n2547\r\n2548\r\n2549\r\n2550\r\n2551\r\n2552\r\n2553\r\n2554\r\n2555\r\n2556\r\n2557\r\n2558\r\n2559\r\n2560\r\n2561\r\n2562\r\n2563\r\n2564\r\n2565\r\n2566\r\n2567\r\n2568\r\n2569\r\n2570\r\n2571\r\n2572\r\n2573\r\n2574\r\n2575\r\n2576\r\n2577\r\n2578\r\n2579\r\n2580\r\n2581\r\n2582\r\n2583\r\n2584\r\n2585\r\n2586\r\n2587\r\n2588\r\n2589\r\n2590\r\n2591\r\n2592\r\n2593\r\n2594\r\n2595\r\n2596\r\n2597\r\n2598\r\n2599\r\n2600\r\n2601\r\n2602\r\n2603\r\n2604\r\n2605\r\n2606\r\n2607\r\n2608\r\n2609\r\n2610\r\n2611\r\n2612\r\n2613\r\n2614\r\n2615\r\n2616\r\n2617\r\n2618\r\n2619\r\n2620\r\n2621\r\n2622\r\n2623\r\n2624\r\n2625\r\n2626\r\n2627\r\n2628\r\n2629\r\n2630\r\n2631\r\n2632\r\n2633\r\n2634\r\n2635\r\n2636\r\n2637\r\n2638\r\n2639\r\n2640\r\n2641\r\n2642\r\n2643\r\n2644\r\n2645\r\n2646\r\n2647\r\n2648\r\n2649\r\n2650\r\n2651\r\n2652\r\n2653\r\n2654\r\n2655\r\n2656\r\n2657\r\n2658\r\n2659\r\n2660\r\n2661\r\n2662\r\n2663\r\n2664\r\n2665\r\n2666\r\n2667\r\n2668\r\n2669\r\n2670\r\n2671\r\n2672\r\n2673\r\n2674\r\n2675\r\n2676\r\n2677\r\n2678\r\n2679\r\n2680\r\n2681\r\n2682\r\n2683\r\n2684\r\n2685\r\n2686\r\n2687\r\n2688\r\n2689\r\n2690\r\n2691\r\n2692\r\n2693\r\n2694\r\n2695\r\n2696\r\n2697\r\n2698\r\n2699\r\n2700\r\n2701\r\n2702\r\n2703\r\n2704\r\n2705\r\n2706\r\n2707\r\n2708\r\n2709\r\n2710\r\n2711\r\n2712\r\n2713\r\n2714\r\n2715\r\n2716\r\n2717\r\n2718\r\n2719\r\n2720\r\n2721\r\n2722\r\n2723\r\n2724\r\n2725\r\n2726\r\n2727\r\n2728\r\n2729\r\n2730\r\n2731\r\n2732\r\n2733\r\n2734\r\n2735\r\n2736\r\n2737\r\n2738\r\n2739\r\n2740\r\n2741\r\n2742\r\n2743\r\n2744\r\n2745\r\n2746\r\n2747\r\n2748\r\n2749\r\n2750\r\n2751\r\n2752\r\n2753\r\n2754\r\n2755\r\n2756\r\n2757\r\n2758\r\n2759\r\n2760\r\n2761\r\n2762\r\n2763\r\n2764\r\n2765\r\n2766\r\n2767\r\n2768\r\n2769\r\n2770\r\n2771\r\n2772\r\n2773\r\n2774\r\n2775\r\n2776\r\n2777\r\n2778\r\n2779\r\n2780\r\n2781\r\n2782\r\n2783\r\n2784\r\n2785\r\n2786\r\n2787\r\n2788\r\n2789\r\n2790\r\n2791\r\n2792\r\n2793\r\n2794\r\n2795\r\n2796\r\n2797\r\n2798\r\n2799\r\n2800\r\n2801\r\n2802\r\n2803\r\n2804\r\n2805\r\n2806\r\n2807\r\n2808\r\n2809\r\n2810\r\n2811\r\n2812\r\n2813\r\n2814\r\n2815\r\n2816\r\n2817\r\n2818\r\n2819\r\n2820\r\n2821\r\n2822\r\n2823\r\n2824\r\n2825\r\n2826\r\n2827\r\n2828\r\n2829\r\n2830\r\n2831\r\n2832\r\n2833\r\n2834\r\n2835\r\n2836\r\n2837\r\n2838\r\n2839\r\n2840\r\n2841\r\n2842\r\n2843\r\n2844\r\n2845\r\n2846\r\n2847\r\n2848\r\n2849\r\n2850\r\n2851\r\n2852\r\n2853\r\n2854\r\n2855\r\n2856\r\n2857\r\n2858\r\n2859\r\n2860\r\n2861\r\n2862\r\n2863\r\n2864\r\n2865\r\n2866\r\n2867\r\n2868\r\n2869\r\n2870\r\n2871\r\n2872\r\n2873\r\n2874\r\n2875\r\n2876\r\n2877\r\n2878\r\n2879\r\n2880\r\n2881\r\n2882\r\n2883\r\n2884\r\n2885\r\n2886\r\n2887\r\n2888\r\n2889\r\n2890\r\n2891\r\n2892\r\n2893\r\n2894\r\n2895\r\n2896\r\n2897\r\n2898\r\n2899\r\n2900\r\n2901\r\n2902\r\n2903\r\n2904\r\n2905\r\n2906\r\n2907\r\n2908\r\n2909\r\n2910\r\n2911\r\n2912\r\n2913\r\n2914\r\n2915\r\n2916\r\n2917\r\n2918\r\n2919\r\n2920\r\n2921\r\n2922\r\n2923\r\n2924\r\n2925\r\n2926\r\n2927\r\n2928\r\n2929\r\n2930\r\n2931\r\n2932\r\n2933\r\n2934\r\n2935\r\n2936\r\n2937\r\n2938\r\n2939\r\n2940\r\n2941\r\n2942\r\n2943\r\n2944\r\n2945\r\n2946\r\n2947\r\n2948\r\n2949\r\n2950\r\n2951\r\n2952\r\n2953\r\n2954\r\n2955\r\n2956\r\n2957\r\n2958\r\n2959\r\n2960\r\n2961\r\n2962\r\n2963\r\n2964\r\n2965\r\n2966\r\n2967\r\n2968\r\n2969\r\n2970\r\n2971\r\n2972\r\n2973\r\n2974\r\n2975\r\n2976\r\n2977\r\n2978\r\n2979\r\n2980\r\n2981\r\n2982\r\n2983\r\n2984\r\n2985\r\n2986\r\n2987\r\n2988\r\n2989\r\n2990\r\n2991\r\n2992\r\n2993\r\n2994\r\n2995\r\n2996\r\n2997\r\n2998\r\n2999\r\n3000\r\n3001\r\n3002\r\n3003\r\n3004\r\n3005\r\n3006\r\n3007\r\n3008\r\n3009\r\n3010\r\n3011\r\n3012\r\n3013\r\n3014\r\n3015\r\n3016\r\n3017\r\n3018\r\n3019\r\n3020\r\n3021\r\n3022\r\n3023\r\n3024\r\n3025\r\n3026\r\n3027\r\n3028\r\n3029\r\n3030\r\n3031\r\n3032\r\n3033\r\n3034\r\n3035\r\n3036\r\n3037\r\n3038\r\n3039\r\n3040\r\n3041\r\n3042\r\n3043\r\n3044\r\n3045\r\n3046\r\n3047\r\n3048\r\n3049\r\n3050\r\n3051\r\n3052\r\n3053\r\n3054\r\n3055\r\n3056\r\n3057\r\n3058\r\n3059\r\n3060\r\n3061\r\n3062\r\n3063\r\n3064\r\n3065\r\n3066\r\n3067\r\n3068\r\n3069\r\n3070\r\n3071\r\n3072\r\n3073\r\n3074\r\n3075\r\n3076\r\n3077\r\n3078\r\n3079\r\n3080\r\n3081\r\n3082\r\n3083\r\n3084\r\n3085\r\n3086\r\n3087\r\n3088\r\n3089\r\n3090\r\n3091\r\n3092\r\n3093\r\n3094\r\n3095\r\n3096\r\n3097\r\n3098\r\n3099\r\n3100\r\n3101\r\n3102\r\n3103\r\n3104\r\n3105\r\n3106\r\n3107\r\n3108\r\n3109\r\n3110\r\n3111\r\n3112\r\n3113\r\n3114\r\n3115\r\n3116\r\n3117\r\n3118\r\n3119\r\n3120\r\n3121\r\n3122\r\n3123\r\n3124\r\n3125\r\n3126\r\n3127\r\n3128\r\n3129\r\n3130\r\n3131\r\n3132\r\n3133\r\n3134\r\n3135\r\n3136\r\n3137\r\n3138\r\n3139\r\n3140\r\n3141\r\n3142\r\n3143\r\n3144\r\n3145\r\n3146\r\n3147\r\n3148\r\n3149\r\n3150\r\n3151\r\n3152\r\n3153\r\n3154\r\n3155\r\n3156\r\n3157\r\n3158\r\n3159\r\n3160\r\n3161\r\n3162\r\n3163\r\n3164\r\n3165\r\n3166\r\n3167\r\n3168\r\n3169\r\n3170\r\n3171\r\n3172\r\n3173\r\n3174\r\n3175\r\n3176\r\n3177\r\n3178\r\n3179\r\n3180\r\n3181\r\n3182\r\n3183\r\n3184\r\n3185\r\n3186\r\n3187\r\n3188\r\n3189\r\n3190\r\n3191\r\n3192\r\n3193\r\n3194\r\n3195\r\n3196\r\n3197\r\n3198\r\n3199\r\n3200\r\n3201\r\n3202\r\n3203\r\n3204\r\n3205\r\n3206\r\n3207\r\n3208\r\n3209\r\n3210\r\n3211\r\n3212\r\n3213\r\n3214\r\n3215\r\n3216\r\n3217\r\n3218\r\n3219\r\n3220\r\n3221\r\n3222\r\n3223\r\n3224\r\n3225\r\n3226\r\n3227\r\n3228\r\n3229\r\n3230\r\n3231\r\n3232\r\n3233\r\n3234\r\n3235\r\n3236\r\n3237\r\n3238\r\n3239\r\n3240\r\n3241\r\n3242\r\n3243\r\n3244\r\n3245\r\n3246\r\n3247\r\n3248\r\n3249\r\n3250\r\n3251\r\n3252\r\n3253\r\n3254\r\n3255\r\n3256\r\n3257\r\n3258\r\n3259\r\n3260\r\n3261\r\n3262\r\n3263\r\n3264\r\n3265\r\n3266\r\n3267\r\n3268\r\n3269\r\n3270\r\n3271\r\n3272\r\n3273\r\n3274\r\n3275\r\n3276\r\n3277\r\n3278\r\n3279\r\n3280\r\n3281\r\n3282\r\n3283\r\n3284\r\n3285\r\n3286\r\n3287\r\n3288\r\n3289\r\n3290\r\n3291\r\n3292\r\n3293\r\n3294\r\n3295\r\n3296\r\n3297\r\n3298\r\n3299\r\n3300\r\n3301\r\n3302\r\n3303\r\n3304\r\n3305\r\n3306\r\n3307\r\n3308\r\n3309\r\n3310\r\n3311\r\n3312\r\n3313\r\n3314\r\n3315\r\n3316\r\n3317\r\n3318\r\n3319\r\n3320\r\n3321\r\n3322\r\n3323\r\n3324\r\n3325\r\n3326\r\n3327\r\n3328\r\n3329\r\n3330\r\n3331\r\n3332\r\n3333\r\n3334\r\n3335\r\n3336\r\n3337\r\n3338\r\n3339\r\n3340\r\n3341\r\n3342\r\n3343\r\n3344\r\n3345\r\n3346\r\n3347\r\n3348\r\n3349\r\n3350\r\n3351\r\n3352\r\n3353\r\n3354\r\n3355\r\n3356\r\n3357\r\n3358\r\n3359\r\n3360\r\n3361\r\n3362\r\n3363\r\n3364\r\n3365\r\n3366\r\n3367\r\n3368\r\n3369\r\n3370\r\n3371\r\n3372\r\n3373\r\n3374\r\n3375\r\n3376\r\n3377\r\n3378\r\n3379\r\n3380\r\n3381\r\n3382\r\n3383\r\n3384\r\n3385\r\n3386\r\n3387\r\n3388\r\n3389\r\n3390\r\n3391\r\n3392\r\n3393\r\n3394\r\n3395\r\n3396\r\n3397\r\n3398\r\n3399\r\n3400\r\n3401\r\n3402\r\n3403\r\n3404\r\n3405\r\n3406\r\n3407\r\n3408\r\n3409\r\n3410\r\n3411\r\n3412\r\n3413\r\n3414\r\n3415\r\n3416\r\n3417\r\n3418\r\n3419\r\n3420\r\n3421\r\n3422\r\n3423\r\n3424\r\n3425\r\n3426\r\n3427\r\n3428\r\n3429\r\n3430\r\n3431\r\n3432\r\n3433\r\n3434\r\n3435\r\n3436\r\n3437\r\n3438\r\n3439\r\n3440\r\n3441\r\n3442\r\n3443\r\n3444\r\n3445\r\n3446\r\n3447\r\n3448\r\n3449\r\n3450\r\n3451\r\n3452\r\n3453\r\n3454\r\n3455\r\n3456\r\n3457\r\n3458\r\n3459\r\n3460\r\n3461\r\n3462\r\n3463\r\n3464\r\n3465\r\n3466\r\n3467\r\n3468\r\n3469\r\n3470\r\n3471\r\n3472\r\n3473\r\n3474\r\n3475\r\n3476\r\n3477\r\n3478\r\n3479\r\n3480\r\n3481\r\n3482\r\n3483\r\n3484\r\n3485\r\n3486\r\n3487\r\n3488\r\n3489\r\n3490\r\n3491\r\n3492\r\n3493\r\n3494\r\n3495\r\n3496\r\n3497\r\n3498\r\n3499\r\n3500\r\n3501\r\n3502\r\n3503\r\n3504\r\n3505\r\n3506\r\n3507\r\n3508\r\n3509\r\n3510\r\n3511\r\n3512\r\n3513\r\n3514\r\n3515\r\n3516\r\n3517\r\n3518\r\n3519\r\n3520\r\n3521\r\n3522\r\n3523\r\n3524\r\n3525\r\n3526\r\n3527\r\n3528\r\n3529\r\n3530\r\n3531\r\n3532\r\n3533\r\n3534\r\n3535\r\n3536\r\n3537\r\n3538\r\n3539\r\n3540\r\n3541\r\n3542\r\n3543\r\n3544\r\n3545\r\n3546\r\n3547\r\n3548\r\n3549\r\n3550\r\n3551\r\n3552\r\n3553\r\n3554\r\n3555\r\n3556\r\n3557\r\n3558\r\n3559\r\n3560\r\n3561\r\n3562\r\n3563\r\n3564\r\n3565\r\n3566\r\n3567\r\n3568\r\n3569\r\n3570\r\n3571\r\n3572\r\n3573\r\n3574\r\n3575\r\n3576\r\n3577\r\n3578\r\n3579\r\n3580\r\n3581\r\n3582\r\n3583\r\n3584\r\n3585\r\n3586\r\n3587\r\n3588\r\n3589\r\n3590\r\n3591\r\n3592\r\n3593\r\n3594\r\n3595\r\n3596\r\n3597\r\n3598\r\n3599\r\n3600\r\n3601\r\n3602\r\n3603\r\n3604\r\n3605\r\n3606\r\n3607\r\n3608\r\n3609\r\n3610\r\n3611\r\n3612\r\n3613\r\n3614\r\n3615\r\n3616\r\n3617\r\n3618\r\n3619\r\n3620\r\n3621\r\n3622\r\n3623\r\n3624\r\n3625\r\n3626\r\n3627\r\n3628\r\n3629\r\n3630\r\n3631\r\n3632\r\n3633\r\n3634\r\n3635\r\n3636\r\n3637\r\n3638\r\n3639\r\n3640\r\n3641\r\n3642\r\n3643\r\n3644\r\n3645\r\n3646\r\n3647\r\n3648\r\n3649\r\n3650\r\n3651\r\n3652\r\n3653\r\n3654\r\n3655\r\n3656\r\n3657\r\n3658\r\n3659\r\n3660\r\n3661\r\n3662\r\n3663\r\n3664\r\n3665\r\n3666\r\n3667\r\n3668\r\n3669\r\n3670\r\n3671\r\n3672\r\n3673\r\n3674\r\n3675\r\n3676\r\n3677\r\n3678\r\n3679\r\n3680\r\n3681\r\n3682\r\n3683\r\n3684\r\n3685\r\n3686\r\n3687\r\n3688\r\n3689\r\n3690\r\n3691\r\n3692\r\n3693\r\n3694\r\n3695\r\n3696\r\n3697\r\n3698\r\n3699\r\n3700\r\n3701\r\n3702\r\n3703\r\n3704\r\n3705\r\n3706\r\n3707\r\n3708\r\n3709\r\n3710\r\n3711\r\n3712\r\n3713\r\n3714\r\n3715\r\n3716\r\n3717\r\n3718\r\n3719\r\n3720\r\n3721\r\n3722\r\n3723\r\n3724\r\n3725\r\n3726\r\n3727\r\n3728\r\n3729\r\n3730\r\n3731\r\n3732\r\n3733\r\n3734\r\n3735\r\n3736\r\n3737\r\n3738\r\n3739\r\n3740\r\n3741\r\n3742\r\n3743\r\n3744\r\n3745\r\n3746\r\n3747\r\n3748\r\n3749\r\n3750\r\n3751\r\n3752\r\n3753\r\n3754\r\n3755\r\n3756\r\n3757\r\n3758\r\n3759\r\n3760\r\n3761\r\n3762\r\n3763\r\n3764\r\n3765\r\n3766\r\n3767\r\n3768\r\n3769\r\n3770\r\n3771\r\n3772\r\n3773\r\n3774\r\n3775\r\n3776\r\n3777\r\n3778\r\n3779\r\n3780\r\n3781\r\n3782\r\n3783\r\n3784\r\n3785\r\n3786\r\n3787\r\n3788\r\n3789\r\n3790\r\n3791\r\n3792\r\n3793\r\n3794\r\n3795\r\n3796\r\n3797\r\n3798\r\n3799\r\n3800\r\n3801\r\n3802\r\n3803\r\n3804\r\n3805\r\n3806\r\n3807\r\n3808\r\n3809\r\n3810\r\n3811\r\n3812\r\n3813\r\n3814\r\n3815\r\n3816\r\n3817\r\n3818\r\n3819\r\n3820\r\n3821\r\n3822\r\n3823\r\n3824\r\n3825\r\n3826\r\n3827\r\n3828\r\n3829\r\n3830\r\n3831\r\n3832\r\n3833\r\n3834\r\n3835\r\n3836\r\n3837\r\n3838\r\n3839\r\n3840\r\n3841\r\n3842\r\n3843\r\n3844\r\n3845\r\n3846\r\n3847\r\n3848\r\n3849\r\n3850\r\n3851\r\n3852\r\n3853\r\n3854\r\n3855\r\n3856\r\n3857\r\n3858\r\n3859\r\n3860\r\n3861\r\n3862\r\n3863\r\n3864\r\n3865\r\n3866\r\n3867\r\n3868\r\n3869\r\n3870\r\n3871\r\n3872\r\n3873\r\n3874\r\n3875\r\n3876\r\n3877\r\n3878\r\n3879\r\n3880\r\n3881\r\n3882\r\n3883\r\n3884\r\n3885\r\n3886\r\n3887\r\n3888\r\n3889\r\n3890\r\n3891\r\n3892\r\n3893\r\n3894\r\n3895\r\n3896\r\n3897\r\n3898\r\n3899\r\n3900\r\n3901\r\n3902\r\n3903\r\n3904\r\n3905\r\n3906\r\n3907\r\n3908\r\n3909\r\n3910\r\n3911\r\n3912\r\n3913\r\n3914\r\n3915\r\n3916\r\n3917\r\n3918\r\n3919\r\n3920\r\n3921\r\n3922\r\n3923\r\n3924\r\n3925\r\n3926\r\n3927\r\n3928\r\n3929\r\n3930\r\n3931\r\n3932\r\n3933\r\n3934\r\n3935\r\n3936\r\n3937\r\n3938\r\n3939\r\n3940\r\n3941\r\n3942\r\n3943\r\n3944\r\n3945\r\n3946\r\n3947\r\n3948\r\n3949\r\n3950\r\n3951\r\n3952\r\n3953\r\n3954\r\n3955\r\n3956\r\n3957\r\n3958\r\n3959\r\n3960\r\n3961\r\n3962\r\n3963\r\n3964\r\n3965\r\n3966\r\n3967\r\n3968\r\n3969\r\n3970\r\n3971\r\n3972\r\n3973\r\n3974\r\n3975\r\n3976\r\n3977\r\n3978\r\n3979\r\n3980\r\n3981\r\n3982\r\n3983\r\n3984\r\n3985\r\n3986\r\n3987\r\n3988\r\n3989\r\n3990\r\n3991\r\n3992\r\n3993\r\n3994\r\n3995\r\n3996\r\n3997\r\n3998\r\n3999\r\n4000\r\n4001\r\n4002\r\n4003\r\n4004\r\n4005\r\n4006\r\n4007\r\n4008\r\n4009\r\n4010\r\n4011\r\n4012\r\n4013\r\n4014\r\n4015\r\n4016\r\n4017\r\n4018\r\n4019\r\n4020\r\n4021\r\n4022\r\n4023\r\n4024\r\n4025\r\n4026\r\n4027\r\n4028\r\n4029\r\n4030\r\n4031\r\n4032\r\n4033\r\n4034\r\n4035\r\n4036\r\n4037\r\n4038\r\n4039\r\n4040\r\n4041\r\n4042\r\n4043\r\n4044\r\n4045\r\n4046\r\n4047\r\n4048\r\n4049\r\n4050\r\n4051\r\n4052\r\n4053\r\n4054\r\n4055\r\n4056\r\n4057\r\n4058\r\n4059\r\n4060\r\n4061\r\n4062\r\n4063\r\n4064\r\n4065\r\n4066\r\n4067\r\n4068\r\n4069\r\n4070\r\n4071\r\n4072\r\n4073\r\n4074\r\n4075\r\n4076\r\n4077\r\n4078\r\n4079\r\n4080\r\n4081\r\n4082\r\n4083\r\n4084\r\n4085\r\n4086\r\n4087\r\n4088\r\n4089\r\n4090\r\n4091\r\n4092\r\n4093\r\n4094\r\n4095\r\n4096\r\n4097\r\n4098\r\n4099\r\n4100\r\n4101\r\n4102\r\n4103\r\n4104\r\n4105\r\n4106\r\n4107\r\n4108\r\n4109\r\n4110\r\n4111\r\n4112\r\n4113\r\n4114\r\n4115\r\n4116\r\n4117\r\n4118\r\n4119\r\n4120\r\n4121\r\n4122\r\n4123\r\n4124\r\n4125\r\n4126\r\n4127\r\n4128\r\n4129\r\n4130\r\n4131\r\n4132\r\n4133\r\n4134\r\n4135\r\n4136\r\n4137\r\n4138\r\n4139\r\n4140\r\n4141\r\n4142\r\n4143\r\n4144\r\n4145\r\n4146\r\n4147\r\n4148\r\n4149\r\n4150\r\n4151\r\n4152\r\n4153\r\n4154\r\n4155\r\n4156\r\n4157\r\n4158\r\n4159\r\n4160\r\n4161\r\n4162\r\n4163\r\n4164\r\n4165\r\n4166\r\n4167\r\n4168\r\n4169\r\n4170\r\n4171\r\n4172\r\n4173\r\n4174\r\n4175\r\n4176\r\n4177\r\n4178\r\n4179\r\n4180\r\n4181\r\n4182\r\n4183\r\n4184\r\n4185\r\n4186\r\n4187\r\n4188\r\n4189\r\n4190\r\n4191\r\n4192\r\n4193\r\n4194\r\n4195\r\n4196\r\n4197\r\n4198\r\n4199\r\n4200\r\n4201\r\n4202\r\n4203\r\n4204\r\n4205\r\n4206\r\n4207\r\n4208\r\n4209\r\n4210\r\n4211\r\n4212\r\n4213\r\n4214\r\n4215\r\n4216\r\n4217\r\n4218\r\n4219\r\n4220\r\n4221\r\n4222\r\n4223\r\n4224\r\n4225\r\n4226\r\n4227\r\n4228\r\n4229\r\n4230\r\n4231\r\n4232\r\n4233\r\n4234\r\n4235\r\n4236\r\n4237\r\n4238\r\n4239\r\n4240\r\n4241\r\n4242\r\n4243\r\n4244\r\n4245\r\n4246\r\n4247\r\n4248\r\n4249\r\n4250\r\n4251\r\n4252\r\n4253\r\n4254\r\n4255\r\n4256\r\n4257\r\n4258\r\n4259\r\n4260\r\n4261\r\n4262\r\n4263\r\n4264\r\n4265\r\n4266\r\n4267\r\n4268\r\n4269\r\n4270\r\n4271\r\n4272\r\n4273\r\n4274\r\n4275\r\n4276\r\n4277\r\n4278\r\n4279\r\n4280\r\n4281\r\n4282\r\n4283\r\n4284\r\n4285\r\n4286\r\n4287\r\n4288\r\n4289\r\n4290\r\n4291\r\n4292\r\n4293\r\n4294\r\n4295\r\n4296\r\n4297\r\n4298\r\n4299\r\n4300\r\n4301\r\n4302\r\n4303\r\n4304\r\n4305\r\n4306\r\n4307\r\n4308\r\n4309\r\n4310\r\n4311\r\n4312\r\n4313\r\n4314\r\n4315\r\n4316\r\n4317\r\n4318\r\n4319\r\n4320\r\n4321\r\n4322\r\n4323\r\n4324\r\n4325\r\n4326\r\n4327\r\n4328\r\n4329\r\n4330\r\n4331\r\n4332\r\n4333\r\n4334\r\n4335\r\n4336\r\n4337\r\n4338\r\n4339\r\n4340\r\n4341\r\n4342\r\n4343\r\n4344\r\n4345\r\n4346\r\n4347\r\n4348\r\n4349\r\n4350\r\n4351\r\n4352\r\n4353\r\n4354\r\n4355\r\n4356\r\n4357\r\n4358\r\n4359\r\n4360\r\n4361\r\n4362\r\n4363\r\n4364\r\n4365\r\n4366\r\n4367\r\n4368\r\n4369\r\n4370\r\n4371\r\n4372\r\n4373\r\n4374\r\n4375\r\n4376\r\n4377\r\n4378\r\n4379\r\n4380\r\n4381\r\n4382\r\n4383\r\n4384\r\n4385\r\n4386\r\n4387\r\n4388\r\n4389\r\n4390\r\n4391\r\n4392\r\n4393\r\n4394\r\n4395\r\n4396\r\n4397\r\n4398\r\n4399\r\n4400\r\n4401\r\n4402\r\n4403\r\n4404\r\n4405\r\n4406\r\n4407\r\n4408\r\n4409\r\n4410\r\n4411\r\n4412\r\n4413\r\n4414\r\n4415\r\n4416\r\n4417\r\n4418\r\n4419\r\n4420\r\n4421\r\n4422\r\n4423\r\n4424\r\n4425\r\n4426\r\n4427\r\n4428\r\n4429\r\n4430\r\n4431\r\n4432\r\n4433\r\n4434\r\n4435\r\n4436\r\n4437\r\n4438\r\n4439\r\n4440\r\n4441\r\n4442\r\n4443\r\n4444\r\n4445\r\n4446\r\n4447\r\n4448\r\n4449\r\n4450\r\n4451\r\n4452\r\n4453\r\n4454\r\n4455\r\n4456\r\n4457\r\n4458\r\n4459\r\n4460\r\n4461\r\n4462\r\n4463\r\n4464\r\n4465\r\n4466\r\n4467\r\n4468\r\n4469\r\n4470\r\n4471\r\n4472\r\n4473\r\n4474\r\n4475\r\n4476\r\n4477\r\n4478\r\n4479\r\n4480\r\n4481\r\n4482\r\n4483\r\n4484\r\n4485\r\n4486\r\n4487\r\n4488\r\n4489\r\n4490\r\n4491\r\n4492\r\n4493\r\n4494\r\n4495\r\n4496\r\n4497\r\n4498\r\n4499\r\n4500\r\n4501\r\n4502\r\n4503\r\n4504\r\n4505\r\n4506\r\n4507\r\n4508\r\n4509\r\n4510\r\n4511\r\n4512\r\n4513\r\n4514\r\n4515\r\n4516\r\n4517\r\n4518\r\n4519\r\n4520\r\n4521\r\n4522\r\n4523\r\n4524\r\n4525\r\n4526\r\n4527\r\n4528\r\n4529\r\n4530\r\n4531\r\n4532\r\n4533\r\n4534\r\n4535\r\n4536\r\n4537\r\n4538\r\n4539\r\n4540\r\n4541\r\n4542\r\n4543\r\n4544\r\n4545\r\n4546\r\n4547\r\n4548\r\n4549\r\n4550\r\n4551\r\n4552\r\n4553\r\n4554\r\n4555\r\n4556\r\n4557\r\n4558\r\n4559\r\n4560\r\n4561\r\n4562\r\n4563\r\n4564\r\n4565\r\n4566\r\n4567\r\n4568\r\n4569\r\n4570\r\n4571\r\n4572\r\n4573\r\n4574\r\n4575\r\n4576\r\n4577\r\n4578\r\n4579\r\n4580\r\n4581\r\n4582\r\n4583\r\n4584\r\n4585\r\n4586\r\n4587\r\n4588\r\n4589\r\n4590\r\n4591\r\n4592\r\n4593\r\n4594\r\n4595\r\n4596\r\n4597\r\n4598\r\n4599\r\n4600\r\n4601\r\n4602\r\n4603\r\n4604\r\n4605\r\n4606\r\n4607\r\n4608\r\n4609\r\n4610\r\n4611\r\n4612\r\n4613\r\n4614\r\n4615\r\n4616\r\n4617\r\n4618\r\n4619\r\n4620\r\n4621\r\n4622\r\n4623\r\n4624\r\n4625\r\n4626\r\n4627\r\n4628\r\n4629\r\n4630\r\n4631\r\n4632\r\n4633\r\n4634\r\n4635\r\n4636\r\n4637\r\n4638\r\n4639\r\n4640\r\n4641\r\n4642\r\n4643\r\n4644\r\n4645\r\n4646\r\n4647\r\n4648\r\n4649\r\n4650\r\n4651\r\n4652\r\n4653\r\n4654\r\n4655\r\n4656\r\n4657\r\n4658\r\n4659\r\n4660\r\n4661\r\n4662\r\n4663\r\n4664\r\n4665\r\n4666\r\n4667\r\n4668\r\n4669\r\n4670\r\n4671\r\n4672\r\n4673\r\n4674\r\n4675\r\n4676\r\n4677\r\n4678\r\n4679\r\n4680\r\n4681\r\n4682\r\n4683\r\n4684\r\n4685\r\n4686\r\n4687\r\n4688\r\n4689\r\n4690\r\n4691\r\n4692\r\n4693\r\n4694\r\n4695\r\n4696\r\n4697\r\n4698\r\n4699\r\n4700\r\n4701\r\n4702\r\n4703\r\n4704\r\n4705\r\n4706\r\n4707\r\n4708\r\n4709\r\n4710\r\n4711\r\n4712\r\n4713\r\n4714\r\n4715\r\n4716\r\n4717\r\n4718\r\n4719\r\n4720\r\n4721\r\n4722\r\n4723\r\n4724\r\n4725\r\n4726\r\n4727\r\n4728\r\n4729\r\n4730\r\n4731\r\n4732\r\n4733\r\n4734\r\n4735\r\n4736\r\n4737\r\n4738\r\n4739\r\n4740\r\n4741\r\n4742\r\n4743\r\n4744\r\n4745\r\n4746\r\n4747\r\n4748\r\n4749\r\n4750\r\n4751\r\n4752\r\n4753\r\n4754\r\n4755\r\n4756\r\n4757\r\n4758\r\n4759\r\n4760\r\n4761\r\n4762\r\n4763\r\n4764\r\n4765\r\n4766\r\n4767\r\n4768\r\n4769\r\n4770\r\n4771\r\n4772\r\n4773\r\n4774\r\n4775\r\n4776\r\n4777\r\n4778\r\n4779\r\n4780\r\n4781\r\n4782\r\n4783\r\n4784\r\n4785\r\n4786\r\n4787\r\n4788\r\n4789\r\n4790\r\n4791\r\n4792\r\n4793\r\n4794\r\n4795\r\n4796\r\n4797\r\n4798\r\n4799\r\n4800\r\n4801\r\n4802\r\n4803\r\n4804\r\n4805\r\n4806\r\n4807\r\n4808\r\n4809\r\n4810\r\n4811\r\n4812\r\n4813\r\n4814\r\n4815\r\n4816\r\n4817\r\n4818\r\n4819\r\n4820\r\n4821\r\n4822\r\n4823\r\n4824\r\n4825\r\n4826\r\n4827\r\n4828\r\n4829\r\n4830\r\n4831\r\n4832\r\n4833\r\n4834\r\n4835\r\n4836\r\n4837\r\n4838\r\n4839\r\n4840\r\n4841\r\n4842\r\n4843\r\n4844\r\n4845\r\n4846\r\n4847\r\n4848\r\n4849\r\n4850\r\n4851\r\n4852\r\n4853\r\n4854\r\n4855\r\n4856\r\n4857\r\n4858\r\n4859\r\n4860\r\n4861\r\n4862\r\n4863\r\n4864\r\n4865\r\n4866\r\n4867\r\n4868\r\n4869\r\n4870\r\n4871\r\n4872\r\n4873\r\n4874\r\n4875\r\n4876\r\n4877\r\n4878\r\n4879\r\n4880\r\n4881\r\n4882\r\n4883\r\n4884\r\n4885\r\n4886\r\n4887\r\n4888\r\n4889\r\n4890\r\n4891\r\n4892\r\n4893\r\n4894\r\n4895\r\n4896\r\n4897\r\n4898\r\n4899\r\n4900\r\n4901\r\n4902\r\n4903\r\n4904\r\n4905\r\n4906\r\n4907\r\n4908\r\n4909\r\n4910\r\n4911\r\n4912\r\n4913\r\n4914\r\n4915\r\n4916\r\n4917\r\n4918\r\n4919\r\n4920\r\n4921\r\n4922\r\n4923\r\n4924\r\n4925\r\n4926\r\n4927\r\n4928\r\n4929\r\n4930\r\n4931\r\n4932\r\n4933\r\n4934\r\n4935\r\n4936\r\n4937\r\n4938\r\n4939\r\n4940\r\n4941\r\n4942\r\n4943\r\n4944\r\n4945\r\n4946\r\n4947\r\n4948\r\n4949\r\n4950\r\n4951\r\n4952\r\n4953\r\n4954\r\n4955\r\n4956\r\n4957\r\n4958\r\n4959\r\n4960\r\n4961\r\n4962\r\n4963\r\n4964\r\n4965\r\n4966\r\n4967\r\n4968\r\n4969\r\n4970\r\n4971\r\n4972\r\n4973\r\n4974\r\n4975\r\n4976\r\n4977\r\n4978\r\n4979\r\n4980\r\n4981\r\n4982\r\n4983\r\n4984\r\n4985\r\n4986\r\n4987\r\n4988\r\n4989\r\n4990\r\n4991\r\n4992\r\n4993\r\n4994\r\n4995\r\n4996\r\n4997\r\n4998\r\n4999\r\n5000\r\nrouter3:~$ "} +{"t": 14.136, "dir": "send", "data": "ls /nonexistent\r", "phase": "command"} +{"t": 14.142, "dir": "recv", "data": "ls /nonexistent\r\nls: /nonexistent: No such file or directory\r\nrouter3:~$ "} +{"t": 16.144, "dir": "send", "data": "exit\r", "phase": "command"} +{"t": 16.146, "dir": "recv", "data": "exit\r\n"} +{"t": 16.157, "dir": "close", "data": ""} diff --git a/tests/recordings/netopeer2/netopeer2-netconf.golden.json b/tests/recordings/netopeer2/netopeer2-netconf.golden.json new file mode 100644 index 00000000..fa2ac6f6 --- /dev/null +++ b/tests/recordings/netopeer2/netopeer2-netconf.golden.json @@ -0,0 +1,55 @@ +{ + "login": "ok", + "driver": "isam", + "os": "isam", + "commands": [ + { + "command": "", + "error": null, + "lines": 1, + "sha256": "da0a6eef846da19d7d140707c48dd66d602120196e48fe922fd5fa296a750188", + "first": "", + "error": null, + "lines": 1, + "sha256": "3c933c4e9edc9d377bef21fe7e0a1a8549ea0d71e6d0e47e41482c4753afb79e", + "first": "", + "last": "" + }, + { + "command": "", + "error": "RPCError", + "lines": 1, + "sha256": "cecfbe1d8fc5f8ee829f80d9021c356e5bbb8ace38ef70850aa01c0113252481", + "first": "applicationapplication", + "error": null, + "lines": 1, + "sha256": "3c933c4e9edc9d377bef21fe7e0a1a8549ea0d71e6d0e47e41482c4753afb79e", + "first": "", + "last": "" + }, + { + "command": "", + "error": null, + "lines": 1, + "sha256": "85d97aba2989dcb93758b14c2754b63126602f64b633dc6f83e7e84129cb6bd3", + "first": "urn:ietf:params:netconf:base:1.0urn:ietf:params:netconf:base:1.1urn:ietf:params:netconf:capability:writable-running:1.0urn:ietf:params:netconf:capability:candidate:1.0urn:ietf:params:netconf:capability:confirmed-commit:1.1urn:ietf:params:netconf:capability:rollback-on-error:1.0urn:ietf:params:netconf:capability:validate:1.1urn:ietf:params:netconf:capability:startup:1.0urn:ietf:params:netconf:capability:xpath:1.0urn:ietf:params:netconf:capability:with-defaults:1.0?basic-mode=explicit&also-supported=report-all,report-all-tagged,trim,expliciturn:ietf:params:netconf:capability:notification:1.0urn:ietf:params:netconf:capability:interleave:1.0urn:ietf:params:xml:ns:yang:ietf-yang-metadata?module=ietf-yang-metadata&revision=2016-08-05urn:ietf:params:xml:ns:yang:1?module=yang&revision=2022-06-16urn:ietf:params:xml:ns:yang:ietf-inet-types?module=ietf-inet-types&revision=2013-07-15urn:ietf:params:xml:ns:yang:ietf-yang-types?module=ietf-yang-types&revision=2013-07-15urn:ietf:params:netconf:capability:yang-library:1.1?revision=2019-01-04&content-id=1urn:ietf:params:xml:ns:yang:ietf-netconf-acm?module=ietf-netconf-acm&revision=2018-02-14urn:sysrepo:plugind?module=sysrepo-plugind&revision=2022-08-26urn:ietf:params:xml:ns:netconf:base:1.0?module=ietf-netconf&revision=2013-09-29&features=writable-running,candidate,confirmed-commit,rollback-on-error,validate,startup,url,xpathurn:ietf:params:xml:ns:yang:ietf-netconf-with-defaults?module=ietf-netconf-with-defaults&revision=2011-06-01urn:ietf:params:xml:ns:yang:ietf-netconf-notifications?module=ietf-netconf-notifications&revision=2012-02-06urn:ietf:params:xml:ns:yang:ietf-netconf-monitoring?module=ietf-netconf-monitoring&revision=2010-10-04urn:ietf:params:xml:ns:netmod:notification?module=nc-notifications&revision=2008-07-14urn:ietf:params:xml:ns:netconf:notification:1.0?module=notifications&revision=2008-07-14urn:ietf:params:xml:ns:yang:ietf-x509-cert-to-name?module=ietf-x509-cert-to-name&revision=2014-12-10urn:ietf:params:xml:ns:yang:iana-crypt-hash?module=iana-crypt-hash&revision=2014-08-061]]>]]>"} +{"t": 0.191, "dir": "send", "data": "urn:ietf:params:netconf:base:1.0urn:ietf:params:netconf:base:1.1]]>]]>", "phase": "login"} +{"t": 0.192, "dir": "send", "data": "\n#126\n\n##\n", "phase": "command"} +{"t": 0.195, "dir": "recv", "data": "\n#1016\ngenkeyrsa2048REMOVEDdefault-ssh0.0.0.01105default-keyREMOVED\n##\n"} +{"t": 0.195, "dir": "send", "data": "\n#223\n\n##\n", "phase": "command"} +{"t": 0.198, "dir": "recv", "data": "\n#1024\nurn:ietf:params:netconf:base:1.0urn:ietf:params:netconf:base:1.1urn:ietf:params:netconf:capability:writable-running:1.0urn:ietf:params:netconf:capability:candidate:1.0urn:ietf:params:netconf:capability:confirmed-commit:1.1urn:ietf:params:netconf:capability:rollback-on-error:1.0urn:ietf:params:netconf:capability:validate:1.1urn:ietf:params:netconf:capability:startup:1.0urn:ietf:params:netconf:capability:xpath:1.0urn:ietf:params:netconf:capability:with-defaults:1.0?basic-mode=explicit&also-supported=report-all,report-all-tagged,trim,expliciturn:ietf:params:netconf:capability:notificat\n#1024\nion:1.0urn:ietf:params:netconf:capability:interleave:1.0urn:ietf:params:xml:ns:yang:ietf-yang-metadata?module=ietf-yang-metadata&revision=2016-08-05urn:ietf:params:xml:ns:yang:1?module=yang&revision=2022-06-16urn:ietf:params:xml:ns:yang:ietf-inet-types?module=ietf-inet-types&revision=2013-07-15urn:ietf:params:xml:ns:yang:ietf-yang-types?module=ietf-yang-types&revision=2013-07-15urn:ietf:params:netconf:capability:yang-library:1.1?revision=2019-01-04&content-id=1urn:ietf:params:xml:ns:yang:ietf-netconf-acm?module=ietf-netconf-acm&revision=2018-02-14urn:sysrepo:plugind?module=sysrepo-plugind&revision=2022-08-26urn:ietf:params:xml:ns:netconf:base:1.0?module=ietf-netconf&revision=2013-09-29&features=writable-running,candidate,confirmed-commit,rollback-on-err\n#973\nor,validate,startup,url,xpathurn:ietf:params:xml:ns:yang:ietf-netconf-with-defaults?module=ietf-netconf-with-defaults&revision=2011-06-01urn:ietf:params:xml:ns:yang:ietf-netconf-notifications?module=ietf-netconf-notifications&revision=2012-02-06urn:ietf:params:xml:ns:yang:ietf-netconf-monitoring?module=ietf-netconf-monitoring&revision=2010-10-04urn:ietf:params:xml:ns:netmod:notification?module=nc-notifications&revision=2008-07-14urn:ietf:params:xml:ns:netconf:notification:1.0?module=notifications&revision=2008-07-14urn:ietf:params:xml:ns:yang:ietf-x509-cert-to-name?module=ietf-x509-cert-to-name&revision=2014-12-10urn:ietf:params:xml:ns:yang:iana-crypt-hash?module=iana-crypt-hash&revision=2014-08-06\n##\n"} +{"t": 0.198, "dir": "send", "data": "\n#114\n\n##\n", "phase": "command"} +{"t": 0.2, "dir": "recv", "data": "\n#91\n\n##\n"} +{"t": 0.2, "dir": "send", "data": "\n#114\n\n##\n", "phase": "command"} +{"t": 0.202, "dir": "recv", "data": "\n#502\napplicationoperation-failederrorRequested resource already lockedapplicationoperation-failederrorUser callback failed.\n##\n"} +{"t": 0.202, "dir": "send", "data": "\n#118\n\n##\n", "phase": "command"} +{"t": 0.203, "dir": "recv", "data": "\n#91\n\n##\n"} +{"t": 0.203, "dir": "send", "data": "\n#186\n\n##\n", "phase": "command"} +{"t": 0.207, "dir": "recv", "data": "\n#1024\ncompleteyang2022-06-16urn:ietf:params:xml:ns:yang:1ietf-yang-schema-mount2019-01-14urn:ietf:params:xml:ns:yang:ietf-yang-schema-mountsysrepo2021-10-08http://www.sysrepo.org/yang/sysrepoietf-datastores2018-02-14urn:ietf:params:xml:ns:yang:ietf-datastoresfile:///etc/sysrepo/yang/ietf-datastores@2018-02-14.yangietf-yang-library2019-01-04urn:ietf:params:xml:ns:yang:ietf-yang-libraryfile:///etc/sysrepo/yang/ietf-yang-library@2019-01-04.yang\n#1022\nietf-netconf-acm2018-02-14urn:ietf:params:xml:ns:yang:ietf-netconf-acmfile:///etc/sysrepo/yang/ietf-netconf-acm@2018-02-14.yangsysrepo-monitoring2022-08-19http://www.sysrepo.org/yang/sysrepo-monitoringfile:///etc/sysrepo/yang/sysrepo-monitoring@2022-08-19.yangsysrepo-plugind2022-08-26urn:sysrepo:plugindfile:///etc/sysrepo/yang/sysrepo-plugind@2022-08-26.yangietf-netconf2013-09-29urn:ietf:params:xml:ns:netconf:base:1.0file:///etc/sysrepo/yang/ietf-netconf@2013-09-29.yangwritable-runningcandidateconfirmed-commitrollback-on-errorvalidatestartup\n#1019\nurlxpathietf-netconf-with-defaults2011-06-01urn:ietf:params:xml:ns:yang:ietf-netconf-with-defaultsfile:///etc/sysrepo/yang/ietf-netconf-with-defaults@2011-06-01.yangietf-netconf-notifications2012-02-06urn:ietf:params:xml:ns:yang:ietf-netconf-notificationsfile:///etc/sysrepo/yang/ietf-netconf-notifications@2012-02-06.yangietf-origin2018-02-14urn:ietf:params:xml:ns:yang:ietf-originfile:///etc/sysrepo/yang/ietf-origin@2018-02-14.yangietf-netconf-monitoring2010-10-04urn:ietf:params:xml:ns:yang:ietf-netconf-monitoringfile:///etc/sysrepo/yang/ietf-netconf-monitoring@2010-10-04.yang\n#1015\nietf-netconf-nmda2019-01-07urn:ietf:params:xml:ns:yang:ietf-netconf-nmdafile:///etc/sysrepo/yang/ietf-netconf-nmda@2019-01-07.yangoriginwith-defaultsnc-notifications2008-07-14urn:ietf:params:xml:ns:netmod:notificationfile:///etc/sysrepo/yang/nc-notifications@2008-07-14.yangnotifications2008-07-14urn:ietf:params:xml:ns:netconf:notification:1.0file:///etc/sysrepo/yang/notifications@2008-07-14.yangietf-x509-cert-to-name2014-12-10urn:ietf:params:xml:ns:yang:ietf-x509-cert-to-namefile:///etc/sysrepo/yang/ietf-x509-cert-to-name@2014-12-10.yangietf-keystore2019-07-02\n#1021\nurn:ietf:params:xml:ns:yang:ietf-keystorefile:///etc/sysrepo/yang/ietf-keystore@2019-07-02.yangkeystore-supportedietf-truststore2019-07-02urn:ietf:params:xml:ns:yang:ietf-truststorefile:///etc/sysrepo/yang/ietf-truststore@2019-07-02.yangtruststore-supportedx509-certificatesietf-tcp-common2019-07-02urn:ietf:params:xml:ns:yang:ietf-tcp-commonfile:///etc/sysrepo/yang/ietf-tcp-common@2019-07-02.yangkeepalives-supportedietf-ssh-server2019-07-02urn:ietf:params:xml:ns:yang:ietf-ssh-serverfile:///etc/sysrepo/yang/ietf-ssh-server@2019-07-02.yanglocal-client-auth-supported\n#1024\nietf-tls-server2019-07-02urn:ietf:params:xml:ns:yang:ietf-tls-serverfile:///etc/sysrepo/yang/ietf-tls-server@2019-07-02.yanglocal-client-auth-supportedietf-netconf-server2019-07-02urn:ietf:params:xml:ns:yang:ietf-netconf-serverfile:///etc/sysrepo/yang/ietf-netconf-server@2019-07-02.yangssh-listentls-listenssh-call-hometls-call-homeietf-interfaces2018-02-20urn:ietf:params:xml:ns:yang:ietf-interfacesfile:///etc/sysrepo/yang/ietf-interfaces@2018-02-20.yangietf-ip2018-02-22urn:ietf:params:xml:ns:yang:ietf-ipfile:///etc/sysrepo/yang/ietf-ip@2018-02-22.yang\n#1018\nietf-network-instance2019-01-21urn:ietf:params:xml:ns:yang:ietf-network-instancefile:///etc/sysrepo/yang/ietf-network-instance@2019-01-21.yangietf-subscribed-notifications2019-09-09urn:ietf:params:xml:ns:yang:ietf-subscribed-notificationsfile:///etc/sysrepo/yang/ietf-subscribed-notifications@2019-09-09.yangencode-xmlreplaysubtreexpathietf-yang-push2019-09-09urn:ietf:params:xml:ns:yang:ietf-yang-pushfile:///etc/sysrepo/yang/ietf-yang-push@2019-09-09.yangon-changeietf-yang-metadata2016-08-05urn:ietf:params:xml:ns:yang:ietf-yang-metadata\n#1006\nietf-inet-types2013-07-15urn:ietf:params:xml:ns:yang:ietf-inet-typesietf-yang-types2013-07-15urn:ietf:params:xml:ns:yang:ietf-yang-typesietf-yang-structure-ext2020-06-17urn:ietf:params:xml:ns:yang:ietf-yang-structure-extietf-crypto-types2019-07-02urn:ietf:params:xml:ns:yang:ietf-crypto-typesfile:///etc/sysrepo/yang/ietf-crypto-types@2019-07-02.yangietf-ssh-common2019-07-02urn:ietf:params:xml:ns:yang:ietf-ssh-commonfile:///etc/sysrepo/yang/ietf-ssh-common@2019-07-02.yang\n#1018\niana-crypt-hash2014-08-06urn:ietf:params:xml:ns:yang:iana-crypt-hashfile:///etc/sysrepo/yang/iana-crypt-hash@2014-08-06.yangietf-tls-common2019-07-02urn:ietf:params:xml:ns:yang:ietf-tls-commonfile:///etc/sysrepo/yang/ietf-tls-common@2019-07-02.yangietf-tcp-client2019-07-02urn:ietf:params:xml:ns:yang:ietf-tcp-clientfile:///etc/sysrepo/yang/ietf-tcp-client@2019-07-02.yangietf-tcp-server2019-07-02urn:ietf:params:xml:ns:yang:ietf-tcp-serverfile:///etc/sysrepo/yang/ietf-tcp-server@2019-07-02.yangietf-restconf\n#1016\n2017-01-26urn:ietf:params:xml:ns:yang:ietf-restconffile:///etc/sysrepo/yang/ietf-restconf@2017-01-26.yangietf-yang-patch2017-02-22urn:ietf:params:xml:ns:yang:ietf-yang-patchfile:///etc/sysrepo/yang/ietf-yang-patch@2017-02-22.yangcompletecompleteds:runningcompleteds:candidatecompleteds:startupcompleteds:operationalcomplete\n#72\n1\n##\n"} +{"t": 0.209, "dir": "send", "data": "\n#90\n\n##\n", "phase": "logout"} +{"t": 0.21, "dir": "recv", "data": "\n#91\n\n##\n"} diff --git a/tools/record_session.py b/tools/record_session.py new file mode 100755 index 00000000..4af0ca49 --- /dev/null +++ b/tools/record_session.py @@ -0,0 +1,390 @@ +#!/usr/bin/env python3 +""" +Records a device session as JSON Lines for Exscript's replay tests. + +The recorder talks to the device directly with paramiko or a plain socket and +does not use Exscript, so it also works where Exscript fails. Passwords are +asked for interactively and never written: a sent secret is stored as +``{"dir": "send", "secret": true, "data": ""}``. Run +``tools/sanitize_recording.py`` on a recording before committing it. + +Examples:: + + uv run tools/record_session.py ssh 192.0.2.1 -u admin -c cmds.txt -o raw.jsonl --os ios + uv run tools/record_session.py telnet 192.0.2.1 -u admin -c cmds.txt -o raw.jsonl + uv run tools/record_session.py netconf 192.0.2.1 -u admin -c rpcs.xml -o raw.jsonl --os junos + +Command files for CLI sessions contain one command per line; empty lines and +lines starting with ``#`` are skipped. ``!secret`` sends the secret that is +asked for at the start (e.g. after ``enable``). ``!send TEXT`` sends TEXT +without a line end and understands Python escapes (``!send \\x20`` answers a +pager). NETCONF command files contain RPC bodies separated by ``---`` lines. +""" + +import argparse +import codecs +import datetime +import getpass +import json +import os +import re +import socket +import sys +import time + +import paramiko + +IAC, DONT, DO, WONT, WILL, SB, SE = 255, 254, 253, 252, 251, 250, 240 +ECHO, SGA, TTYPE, NAWS = 1, 3, 24, 31 +EOM = ']]>]]>' +NETCONF_BASE_10 = 'urn:ietf:params:netconf:base:1.0' +NETCONF_BASE_11 = 'urn:ietf:params:netconf:base:1.1' +USER_PROMPT = re.compile(r'(?:user ?name|login)\s*:\s*$', re.I) +PASSWORD_PROMPT = re.compile(r'password\s*:\s*$', re.I) +CHUNK_HEADER = re.compile(r'\n#(\d+)\n') + + +class Recording: + """Collects the events of a session and writes them as JSON Lines.""" + + def __init__(self, header): + self.header = header + self.events = [] + self.start = time.monotonic() + + def add(self, direction, data=b'', **extra): + event = { + 't': round(time.monotonic() - self.start, 3), + 'dir': direction, + 'data': data.decode('latin-1'), + } + event.update(extra) + self.events.append(event) + + def write(self, path): + with open(path, 'w', encoding='ascii', newline='\n') as fp: + for item in [self.header, *self.events]: + fp.write(json.dumps(item) + '\n') + + +class Link: + """A recorded byte stream to the device (an SSH channel or a socket).""" + + def __init__(self, conn, recording): + self.conn = conn + self.recording = recording + + def send(self, data, **extra): + self.conn.sendall(data) + self.recording.add('send', data, **extra) + + def send_secret(self, secret, **extra): + self.conn.sendall((secret + '\r').encode('latin-1')) + self.recording.add('send', secret=True, **extra) + + def read(self, timeout): + """Returns the text that arrived within the timeout, or None. Raises EOFError at EOF.""" + self.conn.settimeout(timeout) + try: + data = self.conn.recv(65536) + except TimeoutError: + return None + if not data: + self.recording.add('close') + raise EOFError + self.recording.add('recv', data) + return self.process(data) + + def process(self, data): + return data.decode('latin-1') + + +class TelnetLink(Link): + """Answers Telnet option negotiation; the raw bytes are recorded unchanged.""" + + def __init__(self, conn, recording, term, width, height): + super().__init__(conn, recording) + self.term = term.encode('ascii') + self.size = bytes([width >> 8, width & 255, height >> 8, height & 255]) + self.pending = b'' + self.answered = set() + + def send(self, data, **extra): + super().send(data.replace(b'\xff', b'\xff\xff'), **extra) + + def negotiate(self, reply): + if reply not in self.answered: + self.answered.add(reply) + Link.send(self, reply, phase='negotiation') + + def process(self, data): + data, self.pending, text = self.pending + data, b'', bytearray() + i = 0 + while i < len(data): + byte = data[i] + if byte != IAC: + text.append(byte) + i += 1 + continue + if i + 1 >= len(data): + break + command = data[i + 1] + if command == IAC: + text.append(IAC) + i += 2 + elif command in (DO, DONT, WILL, WONT): + if i + 2 >= len(data): + break + self.answer(command, data[i + 2]) + i += 3 + elif command == SB: + end = data.find(bytes([IAC, SE]), i) + if end < 0: + break + if data[i + 2 : i + 4] == bytes([TTYPE, 1]): # SEND terminal type + self.negotiate(bytes([IAC, SB, TTYPE, 0]) + self.term + bytes([IAC, SE])) + i = end + 2 + else: + i += 2 + self.pending = data[i:] + return text.decode('latin-1') + + def answer(self, command, option): + if command == DO: + accept = option in (TTYPE, NAWS) + self.negotiate(bytes([IAC, WILL if accept else WONT, option])) + if option == NAWS: + self.negotiate(bytes([IAC, SB, NAWS]) + self.size + bytes([IAC, SE])) + elif command == WILL: + self.negotiate(bytes([IAC, DO if option in (ECHO, SGA) else DONT, option])) + + +def read_until(link, idle, timeout, complete=None): + """Reads until `complete(text)` is true, or until no data arrived for `idle` seconds.""" + text = '' + deadline = time.monotonic() + timeout + while (remaining := deadline - time.monotonic()) > 0: + chunk = link.read(min(idle, remaining)) + if chunk is None: + if complete is None: + break + continue + text += chunk + if complete is not None and complete(text): + break + return text + + +def check_host_key(transport, host, port, assume_yes): + key = transport.get_remote_server_key() + name = host if port == 22 else f'[{host}]:{port}' + known = paramiko.HostKeys() + try: + known.load(os.path.expanduser('~/.ssh/known_hosts')) + except OSError: + pass + expected = known.lookup(name) + if expected and key.get_name() in expected: + if expected[key.get_name()] != key: + sys.exit(f'Host key mismatch for {name}; aborting.') + return + question = f'Unknown {key.get_name()} host key {key.fingerprint}. Continue? [y/N] ' + if not assume_yes and input(question) != 'y': + sys.exit('Aborted.') + + +def ssh_transport(args, password, recording): + port = args.port or 22 + transport = paramiko.Transport(socket.create_connection((args.host, port), args.timeout)) + transport.start_client(timeout=args.timeout) + check_host_key(transport, args.host, port, args.yes) + try: + transport.auth_none(args.username) + allowed = ['none'] + except paramiko.BadAuthenticationType as exc: + allowed = list(exc.allowed_types) + queries = [] + + def answer(title, instructions, prompts): + if prompts: + queries.append({'title': title, 'instructions': instructions, 'prompts': prompts}) + return [password] * len(prompts) + + if args.key: + try: + key = paramiko.PKey.from_path(args.key) + except paramiko.PasswordRequiredException: + key = paramiko.PKey.from_path(args.key, passphrase=getpass.getpass('Key passphrase: ')) + transport.auth_publickey(args.username, key) + elif 'none' not in allowed: + # Exscript's order; some servers offer keyboard-interactive without supporting it. + methods = [args.auth] if args.auth else ['keyboard-interactive', 'password'] + methods = [method for method in methods if method in allowed] + for method in methods: + try: + if method == 'keyboard-interactive': + transport.auth_interactive(args.username, answer) + else: + transport.auth_password(args.username, password) + break + except paramiko.AuthenticationException: + if method == methods[-1]: + raise + banner = transport.get_banner() + recording.header['ssh'] = { + 'server_version': transport.remote_version, + 'banner': banner.decode('latin-1') if isinstance(banner, bytes) else banner, + 'auth_methods': allowed, + 'interactive': queries[0] if queries else None, + 'host_key_type': transport.get_remote_server_key().get_name(), + } + return transport + + +def cli_session(link, args, password, secret, commands): + def read(): + return read_until(link, args.idle, args.timeout, args.prompt and args.prompt.search) + + text = read() + for _ in range(5): # app-level login prompts + if args.username and USER_PROMPT.search(text): + link.send((args.username + '\r').encode('latin-1'), phase='login') + elif password is not None and PASSWORD_PROMPT.search(text): + link.send_secret(password, phase='login') + else: + break + text = read() + for command in commands: + if command == '!secret': + link.send_secret(secret, phase='command') + elif command.startswith('!send '): + data = codecs.decode(command[6:], 'unicode_escape').encode('latin-1') + link.send(data, phase='command') + else: + link.send((command + '\r').encode('latin-1'), phase='command') + read() + + +def netconf_complete(text, chunked): + if not chunked: + return EOM in text + pos = 0 + while match := CHUNK_HEADER.match(text, pos): + pos = match.end() + int(match.group(1)) + return text.startswith('\n##\n', pos) + + +def netconf_frame(xml, chunked): + data = xml.encode('utf-8') + if not chunked: + return data + EOM.encode() + return b'\n#%d\n%s\n##\n' % (len(data), data) + + +def netconf_session(link, args, rpcs): + hello = read_until(link, args.idle, args.timeout, lambda text: EOM in text) + chunked = NETCONF_BASE_11 in hello + client_hello = ( + '' + '' + f'{NETCONF_BASE_10}{NETCONF_BASE_11}' + '' + ) + link.send(netconf_frame(client_hello, False), phase='login') + for number, body in enumerate([*rpcs, ''], 1): + rpc = f'{body}' + link.send(netconf_frame(rpc, chunked), phase='command' if number <= len(rpcs) else 'logout') + read_until(link, args.idle, args.timeout, lambda text: netconf_complete(text, chunked)) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__.strip().split('\n\n')[0]) + parser.add_argument('protocol', choices=['ssh', 'telnet', 'netconf']) + parser.add_argument('host') + parser.add_argument('-p', '--port', type=int, help='default: 22, 23 or 830') + parser.add_argument('-u', '--username') + parser.add_argument('--key', help='SSH private key instead of a password') + parser.add_argument( + '--auth', choices=['keyboard-interactive', 'password'], help='SSH method to use' + ) + parser.add_argument('--no-password', action='store_true', help='do not ask for a password') + parser.add_argument('-c', '--commands', required=True, help='command or RPC file') + parser.add_argument('-o', '--output', required=True, help='recording file (.jsonl)') + parser.add_argument('--os', help='ground truth: Exscript OS/driver name, e.g. ios') + parser.add_argument('--os-version', help='ground truth: OS version') + parser.add_argument('--platform', help='ground truth: hardware or image') + parser.add_argument('--prompt', type=re.compile, help='regex that ends a read early') + parser.add_argument( + '--idle', type=float, default=2.0, help='seconds without data that end a read' + ) + parser.add_argument('--timeout', type=float, default=60.0, help='maximum seconds per read') + parser.add_argument( + '--term', default='dumb', help="terminal type (default: Exscript's default)" + ) + parser.add_argument('--width', type=int, default=80) + parser.add_argument('--height', type=int, default=24) + parser.add_argument('-y', '--yes', action='store_true', help='accept unknown SSH host keys') + args = parser.parse_args() + if args.protocol == 'netconf' and not args.port: + args.port = 830 + + with open(args.commands, encoding='utf-8') as fp: + content = fp.read() + if args.protocol == 'netconf': + commands = [rpc.strip() for rpc in re.split(r'^---$', content, flags=re.M) if rpc.strip()] + else: + commands = [ + line for line in content.splitlines() if line.strip() and not line.startswith('#') + ] + password = None if args.no_password or args.key else getpass.getpass('Password: ') + secret = getpass.getpass('Secret for !secret: ') if '!secret' in commands else None + + recording = Recording( + { + 'format': 'exscript-recording', + 'version': 1, + 'protocol': args.protocol, + 'date': datetime.date.today().isoformat(), + 'sanitized': False, + 'device': {'os': args.os, 'version': args.os_version, 'platform': args.platform}, + } + ) + try: + if args.protocol == 'telnet': + recording.header['terminal'] = { + 'type': args.term, + 'width': args.width, + 'height': args.height, + } + conn = socket.create_connection((args.host, args.port or 23), args.timeout) + link = TelnetLink(conn, recording, args.term, args.width, args.height) + cli_session(link, args, password, secret, commands) + conn.close() + else: + transport = ssh_transport(args, password, recording) + channel = transport.open_session() + if args.protocol == 'netconf': + channel.invoke_subsystem('netconf') + netconf_session(Link(channel, recording), args, commands) + else: + recording.header['terminal'] = { + 'type': args.term, + 'width': args.width, + 'height': args.height, + } + channel.get_pty(args.term, args.width, args.height) + channel.invoke_shell() + cli_session(Link(channel, recording), args, password, secret, commands) + transport.close() + except EOFError: + pass # the device closed the session; already recorded + except BaseException: + recording.header['incomplete'] = True + raise + finally: + recording.write(args.output) + print(f'{len(recording.events)} events written to {args.output}', file=sys.stderr) + + +if __name__ == '__main__': + main() diff --git a/tools/sanitize_recording.py b/tools/sanitize_recording.py new file mode 100755 index 00000000..16decf14 --- /dev/null +++ b/tools/sanitize_recording.py @@ -0,0 +1,284 @@ +#!/usr/bin/env python3 +""" +Sanitizes a recording from tools/record_session.py before it is committed. + +The sanitizer replaces, consistently within a recording: + +- IPv4 and IPv6 addresses with documentation addresses (it keeps loopback, + unspecified, link-local, multicast, netmask and documentation addresses); +- MAC addresses with locally administered ones (it keeps multicast MACs); +- values after password, secret, key and community keywords, crypt hashes, + and the content of XML elements with such names; +- serial numbers after common labels; +- the literal strings given with ``--replace OLD=NEW`` (host names, user names, + site names, and anything else the patterns miss). + +Consecutive output events are merged first, so a value split between two reads +is still found. NETCONF 1.1 chunks are re-framed with their original sizes. + +The sanitizer prints every replacement and the prompts it saw, but it cannot +know every identifier: read the result before committing it. Example:: + + uv run tools/sanitize_recording.py raw.jsonl tests/recordings/ios/ios-15.9-ssh.jsonl \\ + --replace core-rtr-01=router1 --replace jdoe=admin +""" + +import argparse +import collections +import ipaddress +import itertools +import json +import re +import sys + +EOM = b']]>]]>' +NETCONF_BASE_11 = b'urn:ietf:params:netconf:base:1.1' +REMOVED = 'REMOVED' +FRAME_HEADER = re.compile(rb'\n#(#|\d+)\n') +IPV4 = re.compile(r'(?]*(?)([^<]+)(?= 1 and value[0] == value[-1] == '"' + if value.strip('"') == REMOVED or value in self.keep: + return value + self.report[kind, value[:2] + '...', REMOVED] += 1 + return f'"{REMOVED}"' if quoted else REMOVED + + +def merge_output(events): + merged = [] + for event in events: + if merged and event['dir'] == 'recv' and merged[-1]['dir'] == 'recv': + merged[-1]['data'] += event['data'] + else: + merged.append(dict(event)) + return merged + + +def decode_messages(stream, chunked): + """Splits NETCONF data into complete (message, chunk sizes) tuples and the rest.""" + if not chunked: + *complete, rest = stream.split(EOM) + return [(message, []) for message in complete], rest + messages, message, sizes = [], b'', [] + start = pos = 0 + while match := FRAME_HEADER.match(stream, pos): + if match.group(1) == b'#': + messages.append((message, sizes)) + message, sizes = b'', [] + start = pos = match.end() + continue + end = match.end() + int(match.group(1)) + if end > len(stream): + break + message += stream[match.end() : end] + sizes.append(end - match.end()) + pos = end + return messages, stream[start:] + + +def encode_message(message, chunked, sizes=()): + """Frames a NETCONF message, with the given chunk sizes if chunked (last chunk takes the rest).""" + if not chunked: + return message + EOM + chunks, pos = [], 0 + for size in sizes[:-1]: + if pos + size >= len(message): + break + chunks.append(message[pos : pos + size]) + pos += size + chunks.append(message[pos:]) + return b''.join(b'\n#%d\n%s' % (len(chunk), chunk) for chunk in chunks) + b'\n##\n' + + +def sanitize_netconf(events, clean): + hellos = {} + for event in events: + if event['dir'] not in ('recv', 'send') or event.get('secret'): + continue + data = event['data'].encode('latin-1') + if event['dir'] not in hellos: + hello, rest = data.split(EOM, 1) + hellos[event['dir']] = hello + messages, chunked = [(hello, [])], False + else: + chunked = all(NETCONF_BASE_11 in hello for hello in hellos.values()) + messages, rest = decode_messages(data, chunked) + if rest.strip(): + sys.exit(f'Incomplete NETCONF message at t={event["t"]}; record the session again.') + encoded = b''.join( + encode_message(clean(message), chunked, sizes) for message, sizes in messages + ) + event['data'] = (encoded + rest).decode('latin-1') + + +def prompts(events): + """Returns the last output lines before each input, which usually contain the host name.""" + seen = [] + for previous, event in itertools.pairwise(events): + if previous['dir'] == 'recv' and event['dir'] == 'send': + line = previous['data'].rsplit('\n', 1)[-1].strip() + if line and line not in seen: + seen.append(line) + return seen + + +def main(): + parser = argparse.ArgumentParser(description=__doc__.strip().split('\n\n')[0]) + parser.add_argument('input', help='raw recording') + parser.add_argument('output', help='sanitized recording') + parser.add_argument( + '--replace', action='append', default=[], metavar='OLD=NEW', help='literal replacement' + ) + parser.add_argument( + '--keep', action='append', default=[], metavar='VALUE', help='value to keep as is' + ) + args = parser.parse_args() + replacements = [tuple(item.split('=', 1)) for item in args.replace] + if any(len(item) != 2 or not item[0] for item in replacements): + parser.error('--replace needs OLD=NEW') + + with open(args.input, encoding='utf-8') as fp: + header, *events = [json.loads(line) for line in fp if line.strip()] + sanitizer = Sanitizer(replacements, args.keep) + events = merge_output(events) + if header['protocol'] == 'netconf': + sanitize_netconf( + events, lambda message: sanitizer.text(message.decode('latin-1')).encode('latin-1') + ) + else: + for event in events: + if event['data']: + event['data'] = sanitizer.text(event['data']) + ssh = header.get('ssh') or {} + if ssh.get('banner'): + ssh['banner'] = sanitizer.text(ssh['banner']) + if ssh.get('interactive'): + query = ssh['interactive'] + query['instructions'] = sanitizer.text(query['instructions']) + query['prompts'] = [[sanitizer.text(prompt), echo] for prompt, echo in query['prompts']] + header['sanitized'] = True + + with open(args.output, 'w', encoding='ascii', newline='\n') as fp: + for item in [header, *events]: + fp.write(json.dumps(item) + '\n') + + for (kind, old, new), count in sorted(sanitizer.report.items()): + print(f'{kind:10} {old!r} -> {new!r} ({count}x)') + if header['protocol'] != 'netconf': + print('Prompts (check for host names):', ', '.join(map(repr, prompts(events))) or 'none') + print(f'{len(events)} events written to {args.output}', file=sys.stderr) + + +if __name__ == '__main__': + main() From 23ef68b4e55fae97bec3f1e0e66beac598e35e95 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Sun, 1 Nov 2026 00:14:42 +0100 Subject: [PATCH 012/250] Verify SSH host keys before sending credentials The check was ineffective: known_hosts was loaded too late, [host]:port entries were never found, and unknown keys were accepted, so a man-in-the-middle could get the password. The new host_key_policy handles unknown hosts: 'reject' (default), 'warn' or 'auto-add'. --- CHANGES.md | 22 ++++ scripts/exscript | 16 ++- src/Exscript/host.py | 8 +- src/Exscript/protocols/protocol.py | 21 +++- src/Exscript/protocols/ssh2.py | 142 +++++++++++++------------ tests/Exscript/KnownIssuesTest.py | 21 ---- tests/Exscript/protocols/SSH2Test.py | 151 ++++++++++++++++++++++++++- tests/Exscript/servers/SSHdTest.py | 2 +- 8 files changed, 283 insertions(+), 100 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index cacbb1e9..4d946def 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -18,6 +18,28 @@ marked with **BREAKING**; a migration guide will accompany the release. execute, and close, and records operation durations. Span attributes may include host names, usernames, commands, and response contents. +## Security + +- **BREAKING - SSH Host Key Verification:** `SSH2` and `Netconf` now verify the host + key before any credentials are sent. Before, known_hosts was read only after the + check, and unknown keys were accepted silently. The key is checked against + `~/.ssh/known_hosts` or the file given with the new `known_hosts` argument, + including `[host]:port` entries for other ports than 22 and hashed entries. A + changed key raises `paramiko.BadHostKeyException`. For a host without a known key, + the new `host_key_policy` argument decides: + + - `'reject'` (default): raise `paramiko.SSHException`; + - `'warn'`: log a warning and accept the key; + - `'auto-add'`: accept the key and append it to known_hosts, like OpenSSH's + `StrictHostKeyChecking=accept-new`. + + Accepted keys are pinned for reconnects of the same object. `verify_fingerprint=False` + still disables the verification completely. Both new arguments are also `Host` + options, and the CLI has `--ssh-host-key-policy`. + + To migrate, add the hosts to known_hosts (for example with `ssh-keyscan`), or pass + `host_key_policy='auto-add'` or `host_key_policy='warn'`. + ## Protocols - **BREAKING - Default Protocol:** The default protocol changed across all relevant diff --git a/scripts/exscript b/scripts/exscript index 7366c715..cc9eec92 100755 --- a/scripts/exscript +++ b/scripts/exscript @@ -196,6 +196,7 @@ def run(): if args.ssh_preferred_auth: host.set_option("preferred_authentications", args.ssh_preferred_auth.split(",")) host.set_option("verify_fingerprint", not args.ssh_auto_verify) + host.set_option("host_key_policy", args.ssh_host_key_policy) # Prepare the code that is executed after the user script has completed. if not args.no_auto_logout: @@ -664,8 +665,19 @@ parser.add_argument( action="store_true", default=False, help=""" -Automatically confirms the 'Host key changed' SSH error -message with 'yes'. Highly insecure and not recommended. +Disables the verification of SSH host keys, so that any key is accepted. +Highly insecure and not recommended. +""".strip(), +) + +parser.add_argument( + "--ssh-host-key-policy", + choices=("reject", "warn", "auto-add"), + default="reject", + help=""" +What to do with SSH hosts that are not in ~/.ssh/known_hosts: reject the +connection (the default), warn and connect, or add the host key to +known_hosts and connect. """.strip(), ) diff --git a/src/Exscript/host.py b/src/Exscript/host.py index cd4c474a..9df512fb 100644 --- a/src/Exscript/host.py +++ b/src/Exscript/host.py @@ -103,7 +103,7 @@ def set_uri(self, uri): def get_uri(self): """ Returns a URI formatted representation of the host, including all - of it's attributes except for the name. Uses the + of its attributes except for the name. Uses the address, not the name of the host to build the URI. :rtype: str @@ -167,7 +167,7 @@ def get_name(self): def set_address(self, address): """ - Set the address of the remote host the is contacted, without + Set the address of the remote host that is contacted, without changing hostname, username, password, protocol, and TCP port number. This is the actual address that is used to open the connection. @@ -232,6 +232,8 @@ def set_option(self, name, value): disabled_algorithms: dict[str, list[str]] # SSH2 only driver: Driver()|str encoding: str + host_key_policy: str # SSH2 only + known_hosts: str # SSH2 only preferred_authentications: list[str]|None # SSH2 only termtype: str timeout: int @@ -252,6 +254,8 @@ def set_option(self, name, value): 'disabled_algorithms', 'driver', 'encoding', + 'host_key_policy', + 'known_hosts', 'preferred_authentications', 'termtype', 'timeout', diff --git a/src/Exscript/protocols/protocol.py b/src/Exscript/protocols/protocol.py index 5f48a851..2a806f1e 100644 --- a/src/Exscript/protocols/protocol.py +++ b/src/Exscript/protocols/protocol.py @@ -171,6 +171,9 @@ def __init__( account_factory=None, banner_timeout=20, # SSH2 only encoding='latin-1', + *, + host_key_policy='reject', # SSH2 only + known_hosts=None, # SSH2 only ): """ Constructor. @@ -205,7 +208,19 @@ def __init__( :type preferred_authentications: list[str]|None :keyword preferred_authentications: List of preferred SSH authentication methods, referring to SSH config option 'PreferredAuthentications'. - :keyword verify_fingerprint: Whether to verify the host's fingerprint. + :keyword verify_fingerprint: Whether to verify the host key of the + server. If False, any key is accepted, which allows + man-in-the-middle attacks. + :type host_key_policy: str + :keyword host_key_policy: What to do with a host key if known_hosts + has no key for the host: 'reject' (raise paramiko.SSHException), + 'warn' (log a warning and accept the key), or 'auto-add' (accept + the key and append it to known_hosts). A key that differs from the + known key always raises paramiko.BadHostKeyException. + :type known_hosts: str|None + :keyword known_hosts: The known_hosts file. Defaults to + ~/.ssh/known_hosts. Entries for other ports than 22 have the form + [host]:port, like in OpenSSH. :keyword account_factory: A function that produces a new :class:`Account`. :type banner_timeout: bool :keyword banner_timeout: The time to wait for the banner. @@ -259,6 +274,10 @@ def __init__( self.disabled_algorithms = disabled_algorithms self.preferred_authentications = preferred_authentications self.verify_fingerprint = verify_fingerprint + if host_key_policy not in ('reject', 'warn', 'auto-add'): + raise ValueError(f'invalid host_key_policy: {host_key_policy!r}') + self.host_key_policy = host_key_policy + self.known_hosts = known_hosts # Set manual_driver: if driver is not None: diff --git a/src/Exscript/protocols/ssh2.py b/src/Exscript/protocols/ssh2.py index 58dfc67b..e0f03fb0 100644 --- a/src/Exscript/protocols/ssh2.py +++ b/src/Exscript/protocols/ssh2.py @@ -28,6 +28,7 @@ import os import select import socket +import threading import time from binascii import hexlify @@ -48,6 +49,17 @@ _logger = logging.getLogger(__name__) _logger.addHandler(logging.NullHandler()) +_known_hosts_lock = threading.Lock() + + +def _prefer_key_type(options, keytype): + """Makes the server present a key of the known type, like paramiko's SSHClient.""" + types = list(options.key_types) + if keytype == 'ssh-rsa': # RSA keys are negotiated with SHA-2 signatures + keytype = next((t for t in ('rsa-sha2-512', 'rsa-sha2-256', 'ssh-rsa') if t in types), None) + if keytype in types: + options.key_types = [keytype] + [t for t in types if t != keytype] + # Register supported key types. keymap = {'rsa': paramiko.RSAKey, 'ecdsa': paramiko.ECDSAKey, 'ed25519': paramiko.Ed25519Key} @@ -85,68 +97,59 @@ def __init__(self, **kwargs): "password", ] - # Paramiko client stuff. - self._system_host_keys = paramiko.HostKeys() + # Host keys accepted for unknown hosts, pinned for reconnects of this object. self._host_keys = paramiko.HostKeys() - self._host_keys_filename = None - if self.verify_fingerprint: - self._missing_host_key = self._reject_host_key - else: - self._missing_host_key = self._add_host_key - - def _reject_host_key(self, key): - name = key.get_name() - fp = hexlify(key.get_fingerprint()) - msg = 'Rejecting %s host key for %s: %s' % (name, self.host, fp) - self._dbg(2, msg) - - def _add_host_key(self, key): - name = key.get_name() - fp = hexlify(key.get_fingerprint()) - msg = 'Adding %s host key for %s: %s' % (name, self.host, fp) - self._dbg(2, msg) - self._host_keys.add(self.host, name, key) - if self._host_keys_filename is not None: - self._save_host_keys() - - def _save_host_keys(self): - with open(self._host_keys_filename, 'w') as file: - file.write('# SSH host keys collected by Exscript\n') - for hostname, keys in list(self._host_keys.items()): - for keytype, key in list(keys.items()): - line = ' '.join((hostname, keytype, key.get_base64())) - file.write(line + '\n') - - def _load_system_host_keys(self, filename=None): - """ - Load host keys from a system (read-only) file. Host keys read with - this method will not be saved back by :class:`save_host_keys`. - - This method can be called multiple times. Each new set of host keys - will be merged with the existing set (new replacing old if there are - conflicts). + def _host_key_name(self): + # OpenSSH and paramiko store hosts on other ports than 22 as "[host]:port". + if self.port in (None, 22): + return self.host + return f'[{self.host}]:{self.port}' - If C{filename} is left as ``None``, an attempt will be made to read - keys from the user's local "known hosts" file, as used by OpenSSH, - and no exception will be raised if the file can't be read. This is - probably only useful on posix. + def _known_hosts_file(self): + return os.path.expanduser(self.known_hosts or '~/.ssh/known_hosts') - :param filename: the filename to read, or ``None`` - :type filename: str + def _load_known_hosts(self): + """Returns the keys in the known_hosts file; a missing file has none.""" + keys = paramiko.HostKeys() + try: + keys.load(self._known_hosts_file()) + except FileNotFoundError: + pass + return keys - @raise IOError: if a filename was provided and the file could not be - read + def _check_host_key(self, key, name, known): """ - if filename is None: - # try the user's .ssh key file, and mask exceptions - filename = os.path.expanduser('~/.ssh/known_hosts') - try: - self._system_host_keys.load(filename) - except OSError: - pass + Raises BadHostKeyException if the key differs from the known keys of + the host. If the host has no known keys, applies the host_key_policy. + """ + if known: + expected = known.get(key.get_name()) + if expected != key: # a changed key, or a key type that was never seen + raise BadHostKeyException(name, key, expected or next(iter(known.values()))) return - self._system_host_keys.load(filename) + msg = f'unknown {key.get_name()} host key for {name}: {key.fingerprint}' + if self.host_key_policy == 'reject': + raise SSHException(f'Rejecting {msg} (not in {self._known_hosts_file()})') + if self.host_key_policy == 'auto-add': + self._append_known_host(name, key) + self._dbg(1, f'Added {msg} to {self._known_hosts_file()}') + else: + self._logger.warning('Accepting %s (not in known_hosts)', msg) + self._host_keys.add(name, key.get_name(), key) + + def _append_known_host(self, name, key): + filename = self._known_hosts_file() + line = f'{name} {key.get_name()} {key.get_base64()}\n' + # shortcut: the lock serializes threads only; use separate files for parallel processes. + with _known_hosts_lock: + os.makedirs(os.path.dirname(filename) or os.curdir, mode=0o700, exist_ok=True) + with open(filename, 'ab+') as fp: + if fp.seek(0, os.SEEK_END): + fp.seek(-1, os.SEEK_END) + if fp.read(1) != b'\n': + line = '\n' + line + fp.write(line.encode()) def _paramiko_connect(self): # Find supported address families. @@ -173,20 +176,22 @@ def _paramiko_connect(self): t.logger.disabled = True if self.debug > 5: t.set_hexdump(True) - t.start_client() - # Check system host keys. - server_key = t.get_remote_server_key() - keytype = server_key.get_name() - our_server_key = self._system_host_keys.get(self.host, {}).get(keytype, None) - if our_server_key is None: - our_server_key = self._host_keys.get(self.host, {}).get(keytype, None) - if our_server_key is None: - self._missing_host_key(server_key) - # if the callback returns, assume the key is ok - our_server_key = server_key - if server_key != our_server_key: - raise BadHostKeyException(self.host, server_key, our_server_key) + # The known keys are loaded before the key exchange, so that the key is + # checked before any credentials are sent. + name = self._host_key_name() + known = None + if self.verify_fingerprint: + known = self._load_known_hosts().lookup(name) or self._host_keys.lookup(name) + if known: + _prefer_key_type(t.get_security_options(), next(iter(known))) + t.start_client() + if self.verify_fingerprint: + try: + self._check_host_key(t.get_remote_server_key(), name, known) + except Exception: + t.close() + raise t.set_keepalive(self.KEEPALIVE_INTERVAL) return t @@ -377,7 +382,6 @@ def _paramiko_channel(self): def _connect(self): self._transport = self._paramiko_connect() - self._load_system_host_keys() return True def _protocol_authenticate(self, user, password): diff --git a/tests/Exscript/KnownIssuesTest.py b/tests/Exscript/KnownIssuesTest.py index ea77eecc..b73322ce 100644 --- a/tests/Exscript/KnownIssuesTest.py +++ b/tests/Exscript/KnownIssuesTest.py @@ -10,14 +10,10 @@ import os import socket import sys -import tempfile import threading import time import unittest import uuid -from unittest import mock - -import paramiko from Exscript import Account, PrivateKey from Exscript.emulators import VirtualDevice @@ -59,23 +55,6 @@ def start_daemon(self, daemon_cls, device, **kwargs): return port -class HostKeyTest(ServerMixin, unittest.TestCase): - @unittest.expectedFailure # known_hosts is only loaded after the check; no [host]:port lookup - def testKnownHostsMismatchIsRejected(self): - port = self.start_daemon(SSHd, VirtualDevice(HOST), key=PrivateKey.from_file(KEYFILE)) - other_key = paramiko.RSAKey.generate(2048) - home = tempfile.mkdtemp() - os.mkdir(os.path.join(home, '.ssh')) - with open(os.path.join(home, '.ssh', 'known_hosts'), 'w') as fp: - fp.write('[%s]:%d %s %s\n' % (HOST, port, other_key.get_name(), other_key.get_base64())) - - conn = SSH2(timeout=1, connect_rate_limit=0) - self.addCleanup(conn.close, force=True) - with mock.patch.dict(os.environ, {'HOME': home, 'USERPROFILE': home}): - with self.assertRaises(paramiko.BadHostKeyException): - conn.connect(HOST, port) - - class ExpectTest(ServerMixin, unittest.TestCase): """expect() must find a pattern regardless of how much data follows it.""" diff --git a/tests/Exscript/protocols/SSH2Test.py b/tests/Exscript/protocols/SSH2Test.py index db00f1ee..ae3cddea 100644 --- a/tests/Exscript/protocols/SSH2Test.py +++ b/tests/Exscript/protocols/SSH2Test.py @@ -1,14 +1,22 @@ -import os.path +import os +import socket +import tempfile +import threading import unittest +from unittest import mock + +import paramiko from Exscript import PrivateKey -from Exscript.protocols import SSH2 +from Exscript.protocols import SSH2, Netconf +from Exscript.protocols.ssh2 import _prefer_key_type from Exscript.servers import SSHd, sshd from .ProtocolTest import ProtocolTest keyfile = os.path.join(os.path.dirname(__file__), 'id_rsa') key = PrivateKey.from_file(keyfile) +HOST = '127.0.0.1' class SSH2Test(ProtocolTest): @@ -19,7 +27,7 @@ def createDaemon(self): self.daemon.dbg = 5 def createProtocol(self): - self.protocol = SSH2(timeout=1, debug=5) + self.protocol = SSH2(timeout=1, debug=5, verify_fingerprint=False) def testConstructor(self): self.assertIsInstance(self.protocol, SSH2) @@ -36,8 +44,143 @@ def testAuthenticate(self): self.assertRaises(IOError, ProtocolTest.testAuthenticate, self) +class KeyServer: + """Completes the SSH key exchange with its host keys; nothing else.""" + + def __init__(self, test, *keys): + self.keys = keys + self._transports = [] + self._closing = threading.Event() + self._listener = socket.create_server((HOST, 0)) + self._listener.settimeout(0.1) + self.port = self._listener.getsockname()[1] + self._thread = threading.Thread(target=self._accept, daemon=True) + self._thread.start() + test.addCleanup(self.close) + + def _accept(self): + while not self._closing.is_set(): + try: + sock, _ = self._listener.accept() + except TimeoutError: + continue + transport = paramiko.Transport(sock) + for host_key in self.keys: + transport.add_server_key(host_key) + self._transports.append(transport) + # With an event, the key exchange runs in the transport's thread, and the + # transport stays open until the client or close() closes it. + transport.start_server(event=threading.Event(), server=paramiko.ServerInterface()) + + def close(self): + self._closing.set() + self._thread.join() + self._listener.close() + for transport in self._transports: + transport.close() + + +class HostKeyTest(unittest.TestCase): + """The host key is checked against known_hosts before any credentials are sent.""" + + server_key = paramiko.RSAKey(filename=keyfile) + other_key = paramiko.RSAKey.generate(1024) + + def setUp(self): + self.server = KeyServer(self, self.server_key) + self.name = f'[{HOST}]:{self.server.port}' + tmpdir = tempfile.TemporaryDirectory() + self.addCleanup(tmpdir.cleanup) + self.known_hosts = os.path.join(tmpdir.name, 'known_hosts') + + def writeKnownHosts(self, *entries): + with open(self.known_hosts, 'w') as fp: + for name, host_key in entries: + fp.write(f'{name} {host_key.get_name()} {host_key.get_base64()}\n') + + def connect(self, conn=None, cls=SSH2, **kwargs): + if conn is None: + conn = cls(timeout=1, connect_rate_limit=0, known_hosts=self.known_hosts, **kwargs) + self.addCleanup(conn.close, force=True) + conn.connect(HOST, self.server.port) + return conn + + def testUnknownHostIsRejectedByDefault(self): + for cls in SSH2, Netconf: + with self.assertRaisesRegex(paramiko.SSHException, 'Rejecting unknown ssh-rsa'): + self.connect(cls=cls) + + def testKnownHostIsAccepted(self): + self.writeKnownHosts((self.name, self.server_key)) + self.assertTrue(self.connect().connected) + + def testHashedKnownHostIsAccepted(self): + self.writeKnownHosts((paramiko.HostKeys.hash_host(self.name), self.server_key)) + self.assertTrue(self.connect().connected) + + def testChangedKeyIsRejected(self): + self.writeKnownHosts((self.name, self.other_key)) + for policy in 'reject', 'warn', 'auto-add': + with self.assertRaises(paramiko.BadHostKeyException): + self.connect(host_key_policy=policy) + self.connect(verify_fingerprint=False) + + def testDefaultKnownHostsFile(self): + home = os.path.dirname(self.known_hosts) + os.mkdir(os.path.join(home, '.ssh')) + self.known_hosts = os.path.join(home, '.ssh', 'known_hosts') + self.writeKnownHosts((self.name, self.other_key)) + with mock.patch.dict(os.environ, {'HOME': home, 'USERPROFILE': home}): + conn = SSH2(timeout=1, connect_rate_limit=0) + self.addCleanup(conn.close, force=True) + with self.assertRaises(paramiko.BadHostKeyException): + self.connect(conn) + + def testWarnPolicyPinsTheKey(self): + with self.assertLogs('Exscript.protocols.ssh2', 'WARNING') as logs: + conn = self.connect(host_key_policy='warn') + self.assertIn(self.name, logs.output[0]) + self.assertFalse(os.path.exists(self.known_hosts)) + + # A reconnect of the same object must present the same key. + conn.close(force=True) + self.server.keys = (self.other_key,) + with self.assertRaises(paramiko.BadHostKeyException): + self.connect(conn) + + def testAutoAddPolicyAppendsToKnownHosts(self): + with open(self.known_hosts, 'w') as fp: + fp.write(f'192.0.2.1 {self.other_key.get_name()} {self.other_key.get_base64()}') + self.connect(host_key_policy='auto-add') + known = paramiko.HostKeys(self.known_hosts) + self.assertEqual(known.lookup('192.0.2.1')['ssh-rsa'], self.other_key) + self.assertEqual(known.lookup(self.name)['ssh-rsa'], self.server_key) + self.connect() # now known + + def testKnownKeyTypeIsPreferred(self): + self.server.keys = (paramiko.ECDSAKey.generate(), self.server_key) + self.writeKnownHosts((self.name, self.server_key)) + self.connect() + + def testPreferKeyType(self): + options = mock.Mock(key_types=('ssh-ed25519', 'rsa-sha2-512', 'rsa-sha2-256')) + _prefer_key_type(options, 'ssh-rsa') + self.assertEqual(options.key_types, ['rsa-sha2-512', 'ssh-ed25519', 'rsa-sha2-256']) + _prefer_key_type(options, 'ssh-dss') # not offered: unchanged + self.assertEqual(options.key_types, ['rsa-sha2-512', 'ssh-ed25519', 'rsa-sha2-256']) + + def testInvalidPolicy(self): + self.assertRaises(ValueError, SSH2, host_key_policy='yes') + + def suite(): - return unittest.TestLoader().loadTestsFromTestCase(SSH2Test) + loader = unittest.TestLoader() + return unittest.TestSuite( + [ + loader.loadTestsFromTestCase(SSH2Test), + loader.loadTestsFromTestCase(HostKeyTest), + ] + ) if __name__ == '__main__': diff --git a/tests/Exscript/servers/SSHdTest.py b/tests/Exscript/servers/SSHdTest.py index e897c619..3248c64c 100644 --- a/tests/Exscript/servers/SSHdTest.py +++ b/tests/Exscript/servers/SSHdTest.py @@ -13,7 +13,7 @@ def _create_daemon(self): self.daemon = SSHd(self.host, self.port, self.device) def _create_client(self): - return SSH2() + return SSH2(verify_fingerprint=False) def suite(): From 24500b27d69e90c78a67b60ba965194c8fcef420 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Sun, 1 Nov 2026 00:21:11 +0100 Subject: [PATCH 013/250] Keep passwords and device output out of logs and exceptions Protocol.send() logged passwords at debug level 4; they are logged as now. InvalidCommandException contains only the line that matched an error pattern instead of the whole response, and repr(Account) shows only the account name. --- CHANGES.md | 7 +++++ src/Exscript/account.py | 37 +++++++++++++++------------ src/Exscript/protocols/netconf.py | 2 +- src/Exscript/protocols/protocol.py | 33 +++++++++++++++++++----- tests/Exscript/AccountTest.py | 5 ++++ tests/Exscript/KnownIssuesTest.py | 11 -------- tests/Exscript/protocols/DummyTest.py | 37 ++++++++++++++++++++++++++- 7 files changed, 95 insertions(+), 37 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 4d946def..d9d770dd 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -39,6 +39,13 @@ marked with **BREAKING**; a migration guide will accompany the release. To migrate, add the hosts to known_hosts (for example with `ssh-keyscan`), or pass `host_key_policy='auto-add'` or `host_key_policy='warn'`. +- **Passwords in Debug Logs:** Passwords and S/Key responses sent by the app-level + login are logged as ``. Before, debug level 4 logged them in clear text. +- **BREAKING - `InvalidCommandException` Message:** The message contains only the + line that matched an error pattern, for example `Error in the response from the + device: % Invalid input detected`. Before, it contained the full response, which + may include configuration secrets. The full response is still in `conn.response`. +- **`repr(Account)`:** Shows only the account name, for example `Account('admin')`. ## Protocols diff --git a/src/Exscript/account.py b/src/Exscript/account.py index 99a9df8e..b43267bb 100644 --- a/src/Exscript/account.py +++ b/src/Exscript/account.py @@ -82,6 +82,9 @@ def __init__(self, name, password='', password2=None, key=None, needs_lock=True) self.lock = multiprocessing.Lock() self.needs_lock = needs_lock + def __repr__(self): + return f'{type(self).__name__}({self.name!r})' # never the passwords + def __enter__(self): if self.needs_lock: self.acquire() @@ -587,34 +590,34 @@ def reset(self): def add_pool(self, pool, match=None): """ Adds a new account pool. If the given match argument is - None, the pool the default pool. Otherwise, the match argument is + None, the pool is the default pool. Otherwise, the match argument is a callback function that is invoked to decide whether or not the given pool should be used for a host. When Exscript logs into a host, the account is chosen in the following order: - # Exscript checks whether an account was attached to the - :class:`Host` object using :class:`Host.set_account()`), and uses that. + #. Exscript checks whether an account was attached to the + :class:`Host` object using :class:`Host.set_account()`, and uses that. - # If the :class:`Host` has no account attached, Exscript walks - through all pools that were passed to :class:`Queue.add_account_pool()`. - For each pool, it passes the :class:`Host` to the function in the - given match argument. If the return value is True, the account - pool is used to acquire an account. - (Accounts within each pool are taken in a round-robin - fashion.) + #. If the :class:`Host` has no account attached, Exscript walks + through all pools that were passed to :class:`Queue.add_account_pool()`. + For each pool, it passes the :class:`Host` to the function in the + given match argument. If the return value is True, the account + pool is used to acquire an account. + (Accounts within each pool are taken in a round-robin + fashion.) - # If no matching account pool is found, an account is taken - from the default account pool. + #. If no matching account pool is found, an account is taken + from the default account pool. - # Finally, if all that fails and the default account pool - contains no accounts, an error is raised. + #. Finally, if all that fails and the default account pool + contains no accounts, an error is raised. Example usage:: - def do_nothing(conn): - conn.autoinit() + def log_in(job, host, conn): + conn.login() def use_this_pool(host): return host.get_name().startswith('foo') @@ -630,7 +633,7 @@ def use_this_pool(host): queue.account_manager.add_pool(other_pool, use_this_pool) host = Host('localhost') - queue.run(host, do_nothing) + queue.run(host, log_in) In the example code, the host has no account attached. As a result, the queue checks whether use_this_pool() returns True. Because the diff --git a/src/Exscript/protocols/netconf.py b/src/Exscript/protocols/netconf.py index 086138bb..d170672f 100644 --- a/src/Exscript/protocols/netconf.py +++ b/src/Exscript/protocols/netconf.py @@ -418,7 +418,7 @@ def _validate_rpc_body(self, data): ) raise err - def _raise_on_response_with_errors(self): + def _raise_on_response_with_errors(self, line): # For device handlers that apply XSLT (e.g. Junos) or any transformation, # dispatch() returns NCElement instead of RPCReply. response = self._ncc_response diff --git a/src/Exscript/protocols/protocol.py b/src/Exscript/protocols/protocol.py index 2a806f1e..29d0383b 100644 --- a/src/Exscript/protocols/protocol.py +++ b/src/Exscript/protocols/protocol.py @@ -24,6 +24,7 @@ An abstract base class for all protocols. """ +import contextlib import errno import logging import os @@ -244,6 +245,7 @@ def __init__( self._manual_login_error_re = None self._manual_driver = None self._receives_echo = True # does the remote shell echo sent commands? + self._sending_secret = False # set by _secret() while a password is sent self._write_lock = threading.Lock() # Init remaining per-session attributes. self._init() @@ -1129,7 +1131,8 @@ def _app_authenticate(self, account, password, flush=True, bailout=False): # A password prompt is now required. self.expect(self.get_password_prompt()) - self.send(phrase + '\r') + with self._secret(): + self.send(phrase + '\r') if bailout: break continue @@ -1138,7 +1141,8 @@ def _app_authenticate(self, account, password, flush=True, bailout=False): elif section == 'password': self._dbg(2, "Password prompt received.") self.expect(prompt) # consume the prompt from the buffer - self.send(password + '\r') + with self._secret(): + self.send(password + '\r') if bailout: break continue @@ -1303,9 +1307,21 @@ def send(self, data): :param data: The data that is sent to the remote host. """ self.response = None - self._dbg(4, 'Sending: %s' % repr(data)) + if self._sending_secret: + self._dbg(4, 'Sending: ') + else: + self._dbg(4, 'Sending: %s' % repr(data)) self._send(data) + @contextlib.contextmanager + def _secret(self): + """Marks the data sent within this context as secret, so it is not logged.""" + self._sending_secret = True + try: + yield + finally: + self._sending_secret = False + def execute(self, command, consume=True): """ Sends the given command to the remote host (with a CR appended) and @@ -1474,7 +1490,7 @@ def _check_response_for_errors(self): continue args = repr(prompt.pattern), lineno, repr(line) self._dbg(3, "Error prompt %s matches line %s: %s" % args) - self._raise_on_response_with_errors() + self._raise_on_response_with_errors(line) def add_monitor(self, pattern, callback, limit=80): """ @@ -1515,9 +1531,12 @@ def _raise_on_canceled_expect(self): raise DriverReplacedException() raise ExpectCancelledException() - def _raise_on_response_with_errors(self): - msg = 'Response from device:\n' + self.response - raise InvalidCommandException(msg) + def _raise_on_response_with_errors(self, line): + # Only the error line: the full response (self.response) may contain secrets. + line = line.strip() + if len(line) > 200: + line = line[:200] + '...' + raise InvalidCommandException('Error in the response from the device: ' + line) def _raise_on_response_timeout(self): buffer = str(self._buffer)[-200:] diff --git a/tests/Exscript/AccountTest.py b/tests/Exscript/AccountTest.py index d6bd1b49..baf3fbe3 100644 --- a/tests/Exscript/AccountTest.py +++ b/tests/Exscript/AccountTest.py @@ -58,6 +58,11 @@ def testSetName(self): def testGetName(self): self.assertEqual(self.user, self.account.get_name()) + def testRepr(self): + self.assertEqual(repr(self.account), f'Account({self.user!r})') + self.assertNotIn(self.password1, repr(self.account)) + self.assertNotIn(self.password2, repr(self.account)) + def testSetPassword(self): self.assertEqual(self.password1, self.account.get_password()) self.account.set_password('foo') diff --git a/tests/Exscript/KnownIssuesTest.py b/tests/Exscript/KnownIssuesTest.py index b73322ce..05a23e8f 100644 --- a/tests/Exscript/KnownIssuesTest.py +++ b/tests/Exscript/KnownIssuesTest.py @@ -88,17 +88,6 @@ def testSSH2(self): self.expectMarker(conn, port) -class SecretHygieneTest(unittest.TestCase): - @unittest.expectedFailure # Protocol.send() logs passwords at debug level 4 - def testDebugLogRedactsPassword(self): - conn = Dummy(device=VirtualDevice('r1', echo=False, strict=False), debug=4) - with self.assertLogs('Exscript.protocols', level='DEBUG') as logs: - conn.connect('r1') - conn.login(Account('admin', PASSWORD, needs_lock=False)) - conn.close(force=True) - self.assertEqual([line for line in logs.output if PASSWORD in line], []) - - @unittest.skipIf(ExscriptInstrumentor is None, 'requires the "otel" extra') class OtelTest(unittest.TestCase): def setUp(self): diff --git a/tests/Exscript/protocols/DummyTest.py b/tests/Exscript/protocols/DummyTest.py index b06d0ff7..9ce444af 100644 --- a/tests/Exscript/protocols/DummyTest.py +++ b/tests/Exscript/protocols/DummyTest.py @@ -1,8 +1,10 @@ import re import unittest +from Exscript import Account from Exscript.emulators import VirtualDevice from Exscript.protocols import Dummy +from Exscript.protocols.exception import InvalidCommandException from .ProtocolTest import ProtocolTest @@ -122,8 +124,41 @@ def testDummy(self): protocol.close() +class SecretHygieneTest(unittest.TestCase): + PASSWORD = 'S3cr3t-P@ss' + + def testDebugLogRedactsPassword(self): + conn = Dummy(device=VirtualDevice('r1', echo=False, strict=False), debug=4) + with self.assertLogs('Exscript.protocols', level='DEBUG') as logs: + conn.connect('r1') + conn.login(Account('admin', self.PASSWORD, needs_lock=False)) + conn.close(force=True) + self.assertIn('Sending: ', '\n'.join(logs.output)) + self.assertEqual([line for line in logs.output if self.PASSWORD in line], []) + + def testErrorMessageShowsOnlyTheErrorLine(self): + device = VirtualDevice('r1', echo=False, strict=False) + device.add_command('show run', 'enable secret PRIVATE\r\n% Invalid input detected') + conn = Dummy(device=device) + conn.connect('r1') + conn.login(Account('admin', self.PASSWORD, needs_lock=False)) + with self.assertRaises(InvalidCommandException) as cm: + conn.execute('show run') + self.assertEqual( + str(cm.exception), 'Error in the response from the device: % Invalid input detected' + ) + self.assertIn('PRIVATE', conn.response) + conn.close(force=True) + + def suite(): - return unittest.TestLoader().loadTestsFromTestCase(DummyTest) + loader = unittest.TestLoader() + return unittest.TestSuite( + [ + loader.loadTestsFromTestCase(DummyTest), + loader.loadTestsFromTestCase(SecretHygieneTest), + ] + ) if __name__ == '__main__': From 9e06b6f7903a4b03a2bba4d9570696bd15af378a Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Sun, 1 Nov 2026 00:35:13 +0100 Subject: [PATCH 014/250] Redesign the OpenTelemetry instrumentation It exported passwords and device output, crashed on login() with a positional account, and never exported the spans of a session without close(). Each call now gets its own client span; payloads are recorded only with capture_payload=True. The otel extra installs only the API. --- CHANGES.md | 19 +- pyproject.toml | 9 +- src/Exscript/contrib/otel.py | 450 +++++++++-------------------- tests/Exscript/KnownIssuesTest.py | 44 --- tests/Exscript/contrib/OtelTest.py | 137 +++++++++ tests/Exscript/contrib/__init__.py | 0 uv.lock | 14 +- 7 files changed, 301 insertions(+), 372 deletions(-) create mode 100644 tests/Exscript/contrib/OtelTest.py create mode 100644 tests/Exscript/contrib/__init__.py diff --git a/CHANGES.md b/CHANGES.md index d9d770dd..6c515404 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -13,10 +13,21 @@ marked with **BREAKING**; a migration guide will accompany the release. - **Dependencies:** - **NETCONF Protocol:** - **OpenTelemetry Tracing (optional):** Install the `otel` extra (`Exscript[otel]`) - and use `Exscript.contrib.otel.ExscriptInstrumentor` to instrument `Protocol` and - `Netconf` operations. It creates spans for connect, login/authentication, send, - execute, and close, and records operation durations. Span attributes may include - host names, usernames, commands, and response contents. + and call `Exscript.contrib.otel.ExscriptInstrumentor().instrument()`. Every call of + `connect()`, `login()`, `authenticate()`, `send()`, `execute()` and `close()` creates + a client span such as `SSH2 execute`, with the attributes `server.address`, + `server.port`, `exscript.protocol`, `exscript.operation`, `exscript.os` and + `error.type`. The histogram `exscript.client.operation.duration` records the + durations in seconds. Commands and responses may contain secrets, so they are + recorded only with `instrument(capture_payload=True)`: truncated to + `max_payload_length` characters (default 1024), and with passwords as ``. + The extra installs only the OpenTelemetry API; the application provides the SDK. + + Changes against the 2.9 development versions: spans are children of the current + span instead of the `connect` span, which also ended only on `close()`; the + `protocol.*` attributes are replaced by the ones above; payloads are no longer + recorded by default; and the histogram `exscript.client.duration` in milliseconds + is replaced. `login(account)` with a positional account no longer crashes. ## Security diff --git a/pyproject.toml b/pyproject.toml index aa780ccb..4a8e11a7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,9 +45,8 @@ dependencies = [ [project.optional-dependencies] otel = [ - "opentelemetry-api>=1.12", - "opentelemetry-sdk>=1.12", - "opentelemetry-instrumentation>=0.42b0", + "opentelemetry-api>=1.27", + "opentelemetry-instrumentation>=0.48b0", ] [project.urls] @@ -60,8 +59,8 @@ Changelog = "https://github.com/knipknap/exscript/blob/master/CHANGES.md" test = [ "pytest>=9.2", "pytest-cov>=7.1", - "opentelemetry-sdk>=1.12", - "opentelemetry-instrumentation>=0.42b0", + "opentelemetry-sdk>=1.27", + "opentelemetry-instrumentation>=0.48b0", ] types = ["mypy>=2.4"] dev = [{ include-group = "test" }, { include-group = "types" }] diff --git a/src/Exscript/contrib/otel.py b/src/Exscript/contrib/otel.py index 9c8df7b5..25eb6c6b 100644 --- a/src/Exscript/contrib/otel.py +++ b/src/Exscript/contrib/otel.py @@ -21,355 +21,183 @@ # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ -OpenTelemetry instrumentation for Exscript. - -Automatically creates spans and metrics for ``Protocol.connect``, -``login``, ``send``, ``execute`` and ``close`` (plus the same on -``Netconf``). +OpenTelemetry instrumentation for Exscript. Requires the ``otel`` extra. Usage:: from Exscript.contrib.otel import ExscriptInstrumentor ExscriptInstrumentor().instrument() + +Every call of ``connect()``, ``login()``, ``authenticate()``, ``send()``, +``execute()`` and ``close()`` on a protocol adapter creates a client span named +like ``SSH2 execute``, as a child of the current span. The duration of each call +is recorded in the histogram ``exscript.client.operation.duration``, in seconds. + +Span attributes are ``server.address``, ``server.port``, ``exscript.protocol`` +(the class name, e.g. ``SSH2``), ``exscript.operation``, ``exscript.os`` (the +guessed OS, once known), and ``error.type`` if the call raised. The histogram +has the attributes ``exscript.protocol``, ``exscript.operation`` and +``error.type``. + +Commands and responses may contain secrets, so they are recorded only with +``instrument(capture_payload=True)``, as ``exscript.command`` and +``exscript.response``. They are truncated to ``max_payload_length`` characters +(default: 1024), and passwords sent by the login are recorded as ````. + +Further options of ``instrument()``: + +- ``tracer_provider`` and ``meter_provider``, which default to the global ones; +- ``request_hook(span, conn)``, called before each call; +- ``response_hook(span, conn, result)``, called after each call that did not + raise. """ from __future__ import annotations +import contextvars import functools -import types +import inspect +import time from collections.abc import Callable, Collection -from timeit import default_timer +from typing import Any from opentelemetry.instrumentation.instrumentor import ( # type: ignore[attr-defined] BaseInstrumentor, # mypy cannot resolve it in this namespace package ) -from opentelemetry.metrics import Histogram, get_meter -from opentelemetry.trace import SpanKind, Tracer, get_tracer, set_span_in_context -from opentelemetry.trace.span import Span - -from Exscript.protocols.protocol import Protocol - -# TODO (Netconf): Check whether Netconf is now similar enough to Protocol. -try: - from Exscript.protocols.netconf import Netconf -except ImportError: # ncclient may not be installed - Netconf = None # type: ignore[misc,assignment] - -from Exscript.version import __version__ as _exscript_version - -# --------------------------------------------------------------------------- -# Public type aliases -# --------------------------------------------------------------------------- -_RequestHookT = Callable[[Span, Protocol], None] | None -_ResponseHookT = Callable[[Span, Protocol], None] | None - -# Sentinel attribute set on patched methods -_APPLIED_ATTR = "opentelemetry_instrumentation_exscript_applied" -# Attribute stored on the Protocol *instance* by the connect wrapper -_CONNECT_SPAN_ATTR = "opentelemetry_instrumentation_exscript_connect_span" - -# Methods to instrument — same set for Protocol and Netconf -_INSTRUMENTED_METHODS = ("connect", "login", "authenticate", "close", "send", "execute") - - -# --------------------------------------------------------------------------- -# Helpers -# --------------------------------------------------------------------------- -def _get_parent_span(proto: Protocol) -> Span | None: - """Return the span started by ``connect``, if any.""" - return getattr(proto, _CONNECT_SPAN_ATTR, None) - - -def _get_protocol_attributes(proto: Protocol) -> dict: - """Collect common span attributes from a Protocol instance.""" - attrs: dict = { - "protocol.type": proto.__class__.__name__, - "protocol.host": proto.host, - "protocol.port": proto.port, - "protocol.encoding": proto.encoding, - } - # These calls may fail before connect — be defensive - try: - attrs["protocol.timeout"] = proto.get_timeout() - except Exception: # noqa: BLE001, S110 - pass - try: - attrs["protocol.connect_timeout"] = proto.get_connect_timeout() - except Exception: # noqa: BLE001, S110 - pass - try: - attrs["protocol.os"] = proto.guess_os() - except Exception: # noqa: BLE001, S110 - pass - try: - prompt = proto.get_prompt() - attrs["protocol.prompt"] = ( - [getattr(p, "pattern", str(p)) for p in prompt] - if isinstance(prompt, (list, tuple)) - else getattr(prompt, "pattern", str(prompt)) - ) - except Exception: # noqa: BLE001, S110 - pass - try: - err_prompt = proto.get_error_prompt() - attrs["protocol.error_prompt"] = ( - [getattr(p, "pattern", str(p)) for p in err_prompt] - if isinstance(err_prompt, (list, tuple)) - else getattr(err_prompt, "pattern", str(err_prompt)) - ) - except Exception: # noqa: BLE001, S110 - pass - try: - attrs["protocol.protocol_authenticated"] = proto.is_protocol_authenticated() - except Exception: # noqa: BLE001, S110 - pass - try: - attrs["protocol.app_authenticated"] = proto.is_app_authenticated() - except Exception: # noqa: BLE001, S110 - pass - try: - attrs["protocol.app_authorized"] = proto.is_app_authorized() - except Exception: # noqa: BLE001, S110 - pass - return attrs - - -def _span_name(protocol_name: str, action: str) -> str: - return f"{protocol_name.strip()} {action.strip()}" - - -# --------------------------------------------------------------------------- -# Generic wrapper factory -# --------------------------------------------------------------------------- -def _make_wrapper( - original, - action: str, - tracer: Tracer, - duration_histogram: Histogram, - request_hook: _RequestHookT, - response_hook: _ResponseHookT, -): - """Return an instrumented replacement for *original* method.""" - - @functools.wraps(original) - def wrapper(self, *args, **kwargs): - protocol_name = self.__class__.__name__ - name = _span_name(protocol_name, action) - - # --- span attributes --- - span_attributes: dict = {"protocol.type": protocol_name} - span_attributes.update(_get_protocol_attributes(self)) - - # action-specific attributes - if action == "connect": - hostname = kwargs.get("hostname") or (args[0] if args else None) - port = kwargs.get("port") or (args[1] if len(args) > 1 else None) - if hostname: - span_attributes["protocol.host"] = hostname - if port: - span_attributes["protocol.port"] = port - elif action in ("login", "authenticate"): - account = kwargs.get("account") or (args[0] if args else None) - app_account = kwargs.get("app_account") or (args[1] if args else None) - if account is not None: - try: - span_attributes["protocol.username"] = account.get_name() - except Exception: # noqa: BLE001, S110 - pass - if app_account is not None: - try: - span_attributes["protocol.app_username"] = app_account.get_name() - except Exception: # noqa: BLE001, S110 - pass - elif action == "close": - force = kwargs.get("force", False) or (args[0] if args else False) - span_attributes["protocol.force_close"] = force - elif action in ("send", "execute"): - data = kwargs.get("data") - command = kwargs.get("command", data) or (args[0] if args else None) - if command is not None: - span_attributes["protocol.command"] = command - - metric_labels = {"protocol.type": protocol_name} - - # --- connect uses a long-lived span (ended in close) --- - if action == "connect": - span = tracer.start_span(name, kind=SpanKind.CLIENT, attributes=span_attributes) - exception = None - if callable(request_hook): - request_hook(span, self) - start_time = default_timer() - try: - result = original(self, *args, **kwargs) - except Exception as exc: # noqa: BLE001 - exception = exc - result = getattr(exc, "response", None) - finally: - elapsed = max(round((default_timer() - start_time) * 1000), 0) +from opentelemetry.metrics import get_meter +from opentelemetry.trace import Span, SpanKind, get_tracer + +from Exscript.protocols import Protocol +from Exscript.version import __version__ + +RequestHook = Callable[[Span, Protocol], None] +ResponseHook = Callable[[Span, Protocol, Any], None] + +_OPERATIONS = ('connect', 'login', 'authenticate', 'send', 'execute', 'close') +_APPLIED = '_exscript_otel_applied' +_DISABLED = '_exscript_otel_disabled' +# Calls take from milliseconds (send) to minutes (execute on a slow device). +_BUCKETS = (0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60, 120, 300) +# The call in progress in this context, as (id(conn), operation). +_active: contextvars.ContextVar[tuple[int, str] | None] = contextvars.ContextVar( + 'exscript_otel_active', default=None +) - if callable(response_hook): - response_hook(span, self, result) - duration_histogram.record(elapsed, attributes=metric_labels) - if exception is not None: - raise exception.with_traceback(exception.__traceback__) +def _protocol_classes(): + """Returns Protocol and its subclasses, which may override instrumented methods.""" + classes, todo = [], [Protocol] + while todo: + cls = todo.pop() + classes.append(cls) + todo.extend(cls.__subclasses__()) + return classes - # Store so child spans can use it as parent - setattr(self, _CONNECT_SPAN_ATTR, span) - return result - # --- all other actions use a child span scoped to the call --- - parent = _get_parent_span(self) - ctx = set_span_in_context(parent) if parent else None +def _truncate(value, limit): + value = str(value) + return value if len(value) <= limit else value[:limit] + '...' - span_kwargs: dict = { - "kind": SpanKind.CLIENT, - "attributes": span_attributes, - } - if ctx: - span_kwargs["context"] = ctx - - with tracer.start_as_current_span(name, **span_kwargs) as span: - exception = None - if callable(request_hook): - request_hook(span, self) - start_time = default_timer() - try: - result = original(self, *args, **kwargs) - except Exception as exc: # noqa: BLE001 - exception = exc - result = getattr(exc, "response", None) - finally: - elapsed = max(round((default_timer() - start_time) * 1000), 0) - # Capture response on execute, send does not update response by design! - if action == "execute": - try: - span.set_attribute("protocol.response", str(self.response)) - except Exception: # noqa: BLE001, S110 - pass +def _set_connection_attributes(span, conn): + if conn.host: + span.set_attribute('server.address', str(conn.host)) + if conn.port: + span.set_attribute('server.port', int(conn.port)) + os_name = conn.guess_os() + if os_name and os_name != 'unknown': + span.set_attribute('exscript.os', os_name) - if callable(response_hook): - response_hook(span, self, result) - duration_histogram.record(elapsed, attributes=metric_labels) - if exception is not None: - raise exception.with_traceback(exception.__traceback__) +def _make_wrapper(original, operation, tracer, histogram, options): + signature = inspect.signature(original) + capture, limit = options['capture_payload'], options['max_payload_length'] - # End the connect span when close is called - if action == "close" and parent is not None: + @functools.wraps(original) + def wrapper(conn, *args, **kwargs): + key = (id(conn), operation) + if getattr(conn, _DISABLED, False) or _active.get() == key: + return original(conn, *args, **kwargs) # e.g. an override that calls super() + try: + arguments = signature.bind(conn, *args, **kwargs).arguments + except TypeError: + return original(conn, *args, **kwargs) # raises the TypeError for the caller + + protocol = type(conn).__name__ + labels = {'exscript.protocol': protocol, 'exscript.operation': operation} + attributes = dict(labels) + if capture and operation == 'send': + secret = getattr(conn, '_sending_secret', False) + attributes['exscript.command'] = ( + '' if secret else _truncate(arguments['data'], limit) + ) + elif capture and operation == 'execute': + attributes['exscript.command'] = _truncate(arguments['command'], limit) + + with tracer.start_as_current_span( + f'{protocol} {operation}', kind=SpanKind.CLIENT, attributes=attributes + ) as span: + if options['request_hook']: + options['request_hook'](span, conn) + token = _active.set(key) + start = time.perf_counter() try: - parent.end() - except Exception: # noqa: BLE001, S110 - pass - - return result + result = original(conn, *args, **kwargs) + except BaseException as exc: + labels['error.type'] = type(exc).__qualname__ + span.set_attribute('error.type', labels['error.type']) + raise + finally: + _active.reset(token) + histogram.record(time.perf_counter() - start, labels) + _set_connection_attributes(span, conn) + if capture and operation == 'execute': + span.set_attribute('exscript.response', _truncate(conn.response, limit)) + if options['response_hook']: + options['response_hook'](span, conn, result) + return result - setattr(wrapper, _APPLIED_ATTR, True) + setattr(wrapper, _APPLIED, True) return wrapper -# --------------------------------------------------------------------------- -# Patch / un-patch logic -# --------------------------------------------------------------------------- -def _instrument_class( - cls, - tracer: Tracer, - duration_histogram: Histogram, - request_hook: _RequestHookT, - response_hook: _ResponseHookT, -): - """Monkey-patch all target methods on *cls*.""" - for method_name in _INSTRUMENTED_METHODS: - original = getattr(cls, method_name, None) - if original is None: - continue - if getattr(original, _APPLIED_ATTR, False): - continue # already instrumented - wrapped = _make_wrapper( - original, - method_name, - tracer, - duration_histogram, - request_hook, - response_hook, - ) - setattr(cls, method_name, wrapped) - - -def _uninstrument_class(cls): - """Restore original methods on *cls*.""" - for method_name in _INSTRUMENTED_METHODS: - func = getattr(cls, method_name, None) - if func is None or not getattr(func, _APPLIED_ATTR, False): - continue - setattr(cls, method_name, func.__wrapped__) - - -# --------------------------------------------------------------------------- -# Public instrumentor -# --------------------------------------------------------------------------- class ExscriptInstrumentor(BaseInstrumentor): - """OpenTelemetry instrumentor for Exscript Protocol & Netconf. - - Usage:: - - ExscriptInstrumentor().instrument() - # optional hooks: - ExscriptInstrumentor().instrument( - request_hook=my_request_hook, - response_hook=my_response_hook, - ) - """ + """Instruments all protocol adapters; see the module documentation.""" def instrumentation_dependencies(self) -> Collection[str]: - return ("exscript",) + return ('exscript',) def _instrument(self, **kwargs): - tracer_provider = kwargs.get("tracer_provider") - tracer = get_tracer( - __name__, - _exscript_version, - tracer_provider, + tracer = get_tracer(__name__, __version__, kwargs.get('tracer_provider')) + meter = get_meter(__name__, __version__, kwargs.get('meter_provider')) + histogram = meter.create_histogram( + 'exscript.client.operation.duration', + unit='s', + description='Duration of Exscript protocol operations', + explicit_bucket_boundaries_advisory=_BUCKETS, ) - meter_provider = kwargs.get("meter_provider") - meter = get_meter(__name__, _exscript_version, meter_provider) - duration_histogram = meter.create_histogram( - name="exscript.client.duration", - unit="ms", - description="Duration of Exscript protocol operations", - ) - request_hook = kwargs.get("request_hook") - response_hook = kwargs.get("response_hook") - - _instrument_class( - Protocol, - tracer, - duration_histogram, - request_hook, - response_hook, - ) - if Netconf is not None: - _instrument_class( - Netconf, - tracer, - duration_histogram, - request_hook, - response_hook, - ) + options = { + 'capture_payload': kwargs.get('capture_payload', False), + 'max_payload_length': kwargs.get('max_payload_length', 1024), + 'request_hook': kwargs.get('request_hook'), + 'response_hook': kwargs.get('response_hook'), + } + for cls in _protocol_classes(): + for operation in _OPERATIONS: + method = cls.__dict__.get(operation) + if method is not None and not getattr(method, _APPLIED, False): + wrapper = _make_wrapper(method, operation, tracer, histogram, options) + setattr(cls, operation, wrapper) def _uninstrument(self, **kwargs): - _uninstrument_class(Protocol) - if Netconf is not None: - _uninstrument_class(Netconf) + for cls in _protocol_classes(): + for operation in _OPERATIONS: + method = cls.__dict__.get(operation) + if getattr(method, _APPLIED, False): + setattr(cls, operation, method.__wrapped__) @staticmethod def uninstrument_protocol(protocol): - """Disable instrumentation on a specific *protocol* instance.""" - for method_name in _INSTRUMENTED_METHODS: - func = getattr(protocol, method_name, None) - if func is None or not getattr(func, _APPLIED_ATTR, False): - continue - original = types.MethodType(func.__wrapped__, protocol) - setattr(protocol, method_name, original) + """Disables the instrumentation for the given protocol adapter.""" + setattr(protocol, _DISABLED, True) diff --git a/tests/Exscript/KnownIssuesTest.py b/tests/Exscript/KnownIssuesTest.py index 05a23e8f..ead24ceb 100644 --- a/tests/Exscript/KnownIssuesTest.py +++ b/tests/Exscript/KnownIssuesTest.py @@ -21,17 +21,7 @@ from Exscript.protocols.drivers import driver_map, isam from Exscript.servers import SSHd, Telnetd -try: - from opentelemetry.sdk.trace import TracerProvider - from opentelemetry.sdk.trace.export import SimpleSpanProcessor - from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter - - from Exscript.contrib.otel import ExscriptInstrumentor -except ImportError: # the optional "otel" extra is not installed - ExscriptInstrumentor = None - HOST = '127.0.0.1' -PASSWORD = 'S3cr3t-P@ss' KEYFILE = os.path.join(os.path.dirname(__file__), 'protocols', 'id_rsa') @@ -88,40 +78,6 @@ def testSSH2(self): self.expectMarker(conn, port) -@unittest.skipIf(ExscriptInstrumentor is None, 'requires the "otel" extra') -class OtelTest(unittest.TestCase): - def setUp(self): - self.exporter = InMemorySpanExporter() - provider = TracerProvider() - provider.add_span_processor(SimpleSpanProcessor(self.exporter)) - instrumentor = ExscriptInstrumentor() - instrumentor.instrument(tracer_provider=provider, skip_dep_check=True) - self.addCleanup(instrumentor.uninstrument) - - def runSession(self, login): - device = VirtualDevice('r1', echo=False, strict=False) - device.add_command('show run', 'snmp-server community PRIVATE-COMMUNITY RW') - conn = Dummy(device=device) - conn.connect('r1') - login(conn, Account('admin', PASSWORD, needs_lock=False)) - conn.execute('show run') - conn.close(force=True) - - @unittest.expectedFailure # the wrapper reads args[1] for app_account -> IndexError - def testLoginWithPositionalAccount(self): - self.runSession(lambda conn, account: conn.login(account)) - - @unittest.expectedFailure # commands (incl. passwords) and responses are exported - def testNoSecretsOrDeviceOutputExportedByDefault(self): - self.runSession(lambda conn, account: conn.login(account=account)) - exported = [ - str(value) - for span in self.exporter.get_finished_spans() - for value in (span.attributes or {}).values() - ] - self.assertEqual([v for v in exported if PASSWORD in v or 'PRIVATE-COMMUNITY' in v], []) - - class DriverTest(unittest.TestCase): @unittest.expectedFailure # assigned to self._error_re instead of self.error_re def testIsamUsesItsErrorPatterns(self): diff --git a/tests/Exscript/contrib/OtelTest.py b/tests/Exscript/contrib/OtelTest.py new file mode 100644 index 00000000..30c91791 --- /dev/null +++ b/tests/Exscript/contrib/OtelTest.py @@ -0,0 +1,137 @@ +import unittest +from unittest import mock + +from Exscript import Account +from Exscript.emulators import VirtualDevice +from Exscript.protocols import Dummy + +try: + from opentelemetry.sdk.metrics import MeterProvider + from opentelemetry.sdk.metrics.export import InMemoryMetricReader + from opentelemetry.sdk.trace import TracerProvider + from opentelemetry.sdk.trace.export import SimpleSpanProcessor + from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter + from opentelemetry.trace import SpanKind, StatusCode + + from Exscript.contrib.otel import ExscriptInstrumentor +except ImportError: # the optional "otel" extra is not installed + ExscriptInstrumentor = None + +PASSWORD = 'S3cr3t-P@ss' + + +class Child(Dummy): + """An adapter whose override calls the instrumented base method.""" + + def authenticate(self, *args, **kwargs): + return super().authenticate(*args, **kwargs) + + +@unittest.skipIf(ExscriptInstrumentor is None, 'requires the "otel" extra') +class OtelTest(unittest.TestCase): + def instrument(self, **kwargs): + self.exporter = InMemorySpanExporter() + tracer_provider = TracerProvider() + tracer_provider.add_span_processor(SimpleSpanProcessor(self.exporter)) + self.reader = InMemoryMetricReader() + instrumentor = ExscriptInstrumentor() + instrumentor.instrument( + tracer_provider=tracer_provider, + meter_provider=MeterProvider(metric_readers=[self.reader]), + skip_dep_check=True, + **kwargs, + ) + self.addCleanup(instrumentor.uninstrument) + + def runSession(self, cls=Dummy, disable=False): + device = VirtualDevice('r1', echo=False, strict=False) + device.add_command('show run', 'snmp-server community PRIVATE RW') + conn = cls(device=device, connect_rate_limit=0) + if disable: + ExscriptInstrumentor.uninstrument_protocol(conn) + conn.connect('r1') + conn.login(Account('admin', PASSWORD, needs_lock=False)) # a positional account + conn.execute('show run') + conn.close(force=True) + + def spans(self, name=None): + spans = self.exporter.get_finished_spans() + return [span for span in spans if name is None or span.name == name] + + def testSpans(self): + self.instrument() + self.runSession() + names = {span.name for span in self.spans()} + self.assertLessEqual( + {'Dummy connect', 'Dummy login', 'Dummy execute', 'Dummy close'}, names + ) + (execute,) = self.spans('Dummy execute') + self.assertEqual(execute.kind, SpanKind.CLIENT) + self.assertEqual(execute.attributes['server.address'], 'r1') + self.assertEqual(execute.attributes['exscript.protocol'], 'Dummy') + self.assertEqual(execute.attributes['exscript.operation'], 'execute') + sends = self.spans('Dummy send') + children = [span for span in sends if span.parent.span_id == execute.context.span_id] + self.assertEqual(len(children), 1) + + def testNoPayloadByDefault(self): + self.instrument() + self.runSession() + exported = [str(value) for span in self.spans() for value in span.attributes.values()] + leaks = [ + value + for value in exported + if PASSWORD in value or 'show run' in value or 'PRIVATE' in value + ] + self.assertEqual(leaks, []) + + def testCapturePayload(self): + self.instrument(capture_payload=True, max_payload_length=12) + self.runSession() + (execute,) = self.spans('Dummy execute') + self.assertEqual(execute.attributes['exscript.command'], 'show run') + self.assertEqual(execute.attributes['exscript.response'], 'snmp-server ...') + commands = [span.attributes.get('exscript.command') for span in self.spans('Dummy send')] + self.assertIn('', commands) + exported = [str(value) for span in self.spans() for value in span.attributes.values()] + self.assertEqual([value for value in exported if PASSWORD in value], []) + + def testFailedConnectEndsItsSpan(self): + self.instrument() + conn = Dummy(connect_rate_limit=0) + with mock.patch.object(Dummy, '_connect', side_effect=OSError('unreachable')): + self.assertRaises(OSError, conn.connect, 'r1') + (span,) = self.spans('Dummy connect') + self.assertEqual(span.status.status_code, StatusCode.ERROR) + self.assertEqual(span.attributes['error.type'], 'OSError') + + def testDurationMetric(self): + self.instrument() + self.runSession() + (metric,) = [ + metric + for resource in self.reader.get_metrics_data().resource_metrics + for scope in resource.scope_metrics + for metric in scope.metrics + ] + self.assertEqual(metric.name, 'exscript.client.operation.duration') + self.assertEqual(metric.unit, 's') + operations = {point.attributes['exscript.operation'] for point in metric.data.data_points} + self.assertLessEqual({'connect', 'login', 'send', 'execute', 'close'}, operations) + + def testOverrideCallingSuperCreatesOneSpan(self): + self.instrument() + self.runSession(Child) + self.assertEqual(len(self.spans('Child authenticate')), 1) + + def testUninstrumentProtocol(self): + self.instrument() + self.runSession(disable=True) + self.assertEqual(self.spans(), []) + + def testUninstrument(self): + original = Dummy.connect + self.instrument() + self.assertIsNot(Dummy.connect, original) + ExscriptInstrumentor().uninstrument() + self.assertIs(Dummy.connect, original) diff --git a/tests/Exscript/contrib/__init__.py b/tests/Exscript/contrib/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/uv.lock b/uv.lock index 2af54347..c50b949a 100644 --- a/uv.lock +++ b/uv.lock @@ -418,7 +418,6 @@ dependencies = [ otel = [ { name = "opentelemetry-api" }, { name = "opentelemetry-instrumentation" }, - { name = "opentelemetry-sdk" }, ] [package.dev-dependencies] @@ -443,9 +442,8 @@ types = [ requires-dist = [ { name = "lxml" }, { name = "ncclient", specifier = ">=0.7,<1" }, - { name = "opentelemetry-api", marker = "extra == 'otel'", specifier = ">=1.12" }, - { name = "opentelemetry-instrumentation", marker = "extra == 'otel'", specifier = ">=0.42b0" }, - { name = "opentelemetry-sdk", marker = "extra == 'otel'", specifier = ">=1.12" }, + { name = "opentelemetry-api", marker = "extra == 'otel'", specifier = ">=1.27" }, + { name = "opentelemetry-instrumentation", marker = "extra == 'otel'", specifier = ">=0.48b0" }, { name = "paramiko", specifier = ">=3.4,<6" }, { name = "pycryptodomex" }, ] @@ -454,14 +452,14 @@ provides-extras = ["otel"] [package.metadata.requires-dev] dev = [ { name = "mypy", specifier = ">=2.4" }, - { name = "opentelemetry-instrumentation", specifier = ">=0.42b0" }, - { name = "opentelemetry-sdk", specifier = ">=1.12" }, + { name = "opentelemetry-instrumentation", specifier = ">=0.48b0" }, + { name = "opentelemetry-sdk", specifier = ">=1.27" }, { name = "pytest", specifier = ">=9.2" }, { name = "pytest-cov", specifier = ">=7.1" }, ] test = [ - { name = "opentelemetry-instrumentation", specifier = ">=0.42b0" }, - { name = "opentelemetry-sdk", specifier = ">=1.12" }, + { name = "opentelemetry-instrumentation", specifier = ">=0.48b0" }, + { name = "opentelemetry-sdk", specifier = ">=1.27" }, { name = "pytest", specifier = ">=9.2" }, { name = "pytest-cov", specifier = ">=7.1" }, ] From b3412cd6f81a8f603261de8dc49aa034885de692 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Sun, 1 Nov 2026 00:47:40 +0100 Subject: [PATCH 015/250] Run templates in secure mode by default Templates could run local commands and access local files. sys.exec, sys.env and file.* except file.exists now need secure=False or the new CLI option --insecure. sys.exec runs its command without a shell, so template variables cannot inject shell syntax. --- CHANGES.md | 11 ++++++ doc/exscript.rst | 12 +++++- pyproject.toml | 2 +- scripts/exscript | 25 +++++++++++-- src/Exscript/stdlib/mysys.py | 20 +++++----- src/Exscript/util/template.py | 57 ++++++++++++++++++++--------- tests/Exscript/util/templateTest.py | 40 ++++++++++++++++++++ 7 files changed, 133 insertions(+), 34 deletions(-) create mode 100644 tests/Exscript/util/templateTest.py diff --git a/CHANGES.md b/CHANGES.md index 6c515404..626085c7 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -57,6 +57,17 @@ marked with **BREAKING**; a migration guide will accompany the release. device: % Invalid input detected`. Before, it contained the full response, which may include configuration secrets. The full response is still in `conn.response`. - **`repr(Account)`:** Shows only the account name, for example `Account('admin')`. +- **BREAKING - Templates Run in Secure Mode:** `Exscript.util.template.eval()`, + `eval_file()`, `paste()`, `paste_file()`, `test()` and `test_file()` forbid the + template functions that run local commands or access local files: `sys.exec`, + `sys.env`, and `file.*` except `file.exists`. Templates that call them raise + `Exscript.interpreter.exception.PermissionError` when they are compiled. Functions + that the caller passes as template variables are allowed. Before, secure mode was + available only for `test()` and off by default. To migrate, pass `secure=False`, + or use the new CLI option `--insecure`. +- **BREAKING - `sys.exec` Without a Shell:** The template function `sys.exec` runs + the command without a shell and waits for it to finish. Pipes, redirections and + shell variables no longer work; use for example `sys.exec("sh -c '...'")`. ## Protocols diff --git a/doc/exscript.rst b/doc/exscript.rst index d02f2ba4..57107865 100644 --- a/doc/exscript.rst +++ b/doc/exscript.rst @@ -46,6 +46,9 @@ following options are currently provided: Interprets the given string as the script. --hosts=FILE Loads a list of hostnames from the given file (one host per line). + --insecure Allow the template to call the functions that run + local commands or access local files: sys.exec, + sys.env and file.* (except file.exists). -i, --non-interactive Do not ask for a username or password. -l DIR, --logdir=DIR Logs any communication into the directory with the @@ -82,9 +85,14 @@ following options are currently provided: --sleep=TIME Waits for the specified time before running the script. TIME is a timespec as specified by the 'sleep' Unix command. - --ssh-auto-verify Automatically confirms the 'Host key changed' SSH - error message with 'yes'. Highly insecure and not + --ssh-auto-verify Disables the verification of SSH host keys, so that + any key is accepted. Highly insecure and not recommended. + --ssh-host-key-policy={reject,warn,auto-add} + What to do with SSH hosts that are not in + ~/.ssh/known_hosts: reject the connection (the + default), warn and connect, or add the host key to + known_hosts and connect. --ssh-key=FILE Specify a key file that is passed to the SSH client. This is equivalent to using the "-i" parameter of the openssh command line client. diff --git a/pyproject.toml b/pyproject.toml index 4a8e11a7..32c340cf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -107,7 +107,7 @@ ignore = [ "src/Exscript/protocols/ssh2.py" = ["S110"] "src/Exscript/protocols/telnet.py" = ["S101", "S110"] "src/Exscript/servers/httpd.py" = ["S105", "S324"] -"src/Exscript/stdlib/mysys.py" = ["S602"] +"src/Exscript/stdlib/mysys.py" = ["S603"] "src/Exscript/util/crypt.py" = ["S303"] "src/Exscript/util/file.py" = ["S107"] "src/Exscript/util/pidutil.py" = ["S110"] diff --git a/scripts/exscript b/scripts/exscript index cc9eec92..30c45ca4 100755 --- a/scripts/exscript +++ b/scripts/exscript @@ -220,8 +220,8 @@ def run(): vars["__filename__"] = args.filename vars.update(hosts[0].get_all()) try: - template.test(script_content, **vars) - template.test(script_extra_content, **vars) + template.test(script_content, secure=not args.insecure, **vars) + template.test(script_extra_content, secure=not args.insecure, **vars) except Exception as e: if args.parser_verbose > 0: raise @@ -302,7 +302,9 @@ def run(): if args.filename: kwargs["__filename__"] = args.filename kwargs.update(host.get_all()) - return template.paste(conn, script_content, args.no_escaping, **kwargs) + return template.paste( + conn, script_content, args.no_escaping, secure=not args.insecure, **kwargs + ) else: @@ -313,7 +315,12 @@ def run(): kwargs["__filename__"] = args.filename kwargs.update(host.get_all()) return template.eval( - conn, script_content, not args.no_strip, args.no_escaping, **kwargs + conn, + script_content, + not args.no_strip, + args.no_escaping, + secure=not args.insecure, + **kwargs, ) # Wrap the template processor such that the login procedure is automated. @@ -494,6 +501,16 @@ functions to be made available in the Exscript template. """.strip(), ) +parser.add_argument( + "--insecure", + action="store_true", + default=False, + help=""" +Allow the template to call the functions that run local commands or access +local files: sys.exec, sys.env and file.* (except file.exists). +""".strip(), +) + parser.add_argument( "--load-driver", default=None, diff --git a/src/Exscript/stdlib/mysys.py b/src/Exscript/stdlib/mysys.py index 4ff19b71..9e07416d 100644 --- a/src/Exscript/stdlib/mysys.py +++ b/src/Exscript/stdlib/mysys.py @@ -21,9 +21,10 @@ # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. import os +import shlex import sys import time -from subprocess import PIPE, STDOUT, Popen +from subprocess import DEVNULL, PIPE, STDOUT, run from .util import secure_function @@ -41,18 +42,17 @@ def env(scope, varname): def execute(scope, command): """ - Executes the given command locally. + Executes the given command locally, without a shell, and waits for it to + finish. Its output is stored in the __response__ variable. :type command: string - :param command: A shell command. + :param command: A command line. On POSIX systems it is split like a shell + would split it, but pipes, redirections and variables are not + supported; use e.g. sh -c '...' to run a shell. """ - process = Popen( - command[0], shell=True, stdin=PIPE, stdout=PIPE, stderr=STDOUT, text=True, close_fds=True - ) - if process.stdout: - scope.define(__response__=process.stdout.readlines()) - else: - scope.define(__response__=['']) + args = command[0] if os.name == 'nt' else shlex.split(command[0]) + result = run(args, stdin=DEVNULL, stdout=PIPE, stderr=STDOUT, text=True, check=False) + scope.define(__response__=result.stdout.splitlines(keepends=True)) return True diff --git a/src/Exscript/util/template.py b/src/Exscript/util/template.py index 8e60378a..07139013 100644 --- a/src/Exscript/util/template.py +++ b/src/Exscript/util/template.py @@ -22,14 +22,29 @@ # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ Executing Exscript templates on a connection. + +All functions compile templates in secure mode by default: templates may not +call the functions of the standard library that run local commands or access +local files (``sys.exec``, ``sys.env``, and ``file.*`` except ``file.exists``). +Such calls raise :class:`Exscript.interpreter.exception.PermissionError` when +the template is compiled. Functions that the caller passes as variables are +allowed. Pass ``secure=False`` to allow all functions. """ from Exscript import stdlib from Exscript.interpreter import Parser +from Exscript.stdlib.util import secure_function debug = 0 +def _trusted(value): + # Functions passed by the caller are allowed in secure mode. + if callable(value) and not hasattr(value, '_is_secure'): + return secure_function(value) + return value + + def _compile(conn, filename, template, parser_kwargs, **kwargs): undefined = 'undefined' if conn: @@ -57,7 +72,7 @@ def _compile(conn, filename, template, parser_kwargs, **kwargs): parser.define_object(**stdlib.functions) # Allow for overriding built-in variables. - parser.define(**kwargs) + parser.define(**{name: _trusted(value) for name, value in kwargs.items()}) # Compile the template. return parser.parse(template, parser.variables.get('__filename__')[0]) @@ -68,20 +83,16 @@ def _run(conn, filename, template, parser_kwargs, **kwargs): return compiled.execute() -def test(string, secure=False, **kwargs): +def test(string, secure=True, **kwargs): """ Compiles the given template, and raises an exception if that failed. Does nothing otherwise. - With secure mode enabled it makes sure that each function that is used in - the template has the Exscript.stdlib.util.safe_function decorator. - Raises Exscript.interpreter.Exception.PermissionError if any function - lacks the decorator. - :type string: string :param string: The template to compile. :type secure: bool - :param secure: Whether to enable secure mode. + :param secure: Whether to forbid the insecure functions (see the module + documentation). :type kwargs: dict :param kwargs: Variables to define in the template. """ @@ -89,13 +100,15 @@ def test(string, secure=False, **kwargs): _compile(None, None, string, parser_args, **kwargs) -def test_file(filename, secure=False, **kwargs): +def test_file(filename, secure=True, **kwargs): """ Convenience wrapper around test() that reads the template from a file instead. :type filename: string :param filename: The name of the template file. + :type secure: bool + :param secure: Whether to forbid the insecure functions. :type kwargs: dict :param kwargs: Variables to define in the template. """ @@ -104,7 +117,7 @@ def test_file(filename, secure=False, **kwargs): _compile(None, filename, fp.read(), parser_args, **kwargs) -def eval(conn, string, strip_command=True, no_escaping=False, **kwargs): +def eval(conn, string, strip_command=True, no_escaping=False, *, secure=True, **kwargs): r""" Compiles the given template and executes it on the given connection. Raises an exception if the compilation fails. @@ -138,16 +151,19 @@ def eval(conn, string, strip_command=True, no_escaping=False, **kwargs): :param no_escaping: Whether to disable advanced escape evaluation for control characters, octal, hexadecimal and unicode character sequences. Evaluation for CR, LF and TAB in strings is always enabled. + :type secure: bool + :param secure: Whether to forbid the insecure functions (see the module + documentation). :type kwargs: dict :param kwargs: Variables to define in the template. :rtype: dict :return: The variables that are defined after execution of the script. """ - parser_args = {'strip_command': strip_command, 'no_escaping': no_escaping} + parser_args = {'strip_command': strip_command, 'no_escaping': no_escaping, 'secure': secure} return _run(conn, None, string, parser_args, **kwargs) -def eval_file(conn, filename, strip_command=True, no_escaping=False, **kwargs): +def eval_file(conn, filename, strip_command=True, no_escaping=False, *, secure=True, **kwargs): """ Convenience wrapper around eval() that reads the template from a file instead. @@ -162,15 +178,17 @@ def eval_file(conn, filename, strip_command=True, no_escaping=False, **kwargs): :param no_escaping: Whether to disable advanced escape evaluation for control characters, octal, hexadecimal and unicode character sequences. Evaluation for CR, LF and TAB in strings is always enabled. + :type secure: bool + :param secure: Whether to forbid the insecure functions. :type kwargs: dict :param kwargs: Variables to define in the template. """ - parser_args = {'strip_command': strip_command, 'no_escaping': no_escaping} + parser_args = {'strip_command': strip_command, 'no_escaping': no_escaping, 'secure': secure} with open(filename) as fp: return _run(conn, filename, fp.read(), parser_args, **kwargs) -def paste(conn, string, no_escaping=False, **kwargs): +def paste(conn, string, no_escaping=False, *, secure=True, **kwargs): """ Compiles the given template and executes it on the given connection. This function differs from eval() such that it does not @@ -190,16 +208,19 @@ def paste(conn, string, no_escaping=False, **kwargs): :param no_escaping: Whether to disable advanced escape evaluation for control characters, octal, hexadecimal and unicode character sequences. Evaluation for CR, LF and TAB in strings is always enabled. + :type secure: bool + :param secure: Whether to forbid the insecure functions (see the module + documentation). :type kwargs: dict :param kwargs: Variables to define in the template. :rtype: dict :return: The variables that are defined after execution of the script. """ - parser_args = {'no_prompt': True, 'no_escaping': no_escaping} + parser_args = {'no_prompt': True, 'no_escaping': no_escaping, 'secure': secure} return _run(conn, None, string, parser_args, **kwargs) -def paste_file(conn, filename, no_escaping=False, **kwargs): +def paste_file(conn, filename, no_escaping=False, *, secure=True, **kwargs): """ Convenience wrapper around paste() that reads the template from a file instead. @@ -212,9 +233,11 @@ def paste_file(conn, filename, no_escaping=False, **kwargs): :param no_escaping: Whether to disable advanced escape evaluation for control characters, octal, hexadecimal and unicode character sequences. Evaluation for CR, LF and TAB in strings is always enabled. + :type secure: bool + :param secure: Whether to forbid the insecure functions. :type kwargs: dict :param kwargs: Variables to define in the template. """ - parser_args = {'no_prompt': True, 'no_escaping': no_escaping} + parser_args = {'no_prompt': True, 'no_escaping': no_escaping, 'secure': secure} with open(filename) as fp: return _run(conn, filename, fp.read(), parser_args, **kwargs) diff --git a/tests/Exscript/util/templateTest.py b/tests/Exscript/util/templateTest.py new file mode 100644 index 00000000..7752c2b5 --- /dev/null +++ b/tests/Exscript/util/templateTest.py @@ -0,0 +1,40 @@ +import os +import unittest + +from Exscript.emulators import VirtualDevice +from Exscript.interpreter.exception import PermissionError +from Exscript.protocols import Dummy +from Exscript.util import template + + +def shout(scope, string): + """A function that the caller passes to the template.""" + return [string[0].upper()] + + +class templateTest(unittest.TestCase): + CORRELATE = template + + def setUp(self): + self.conn = Dummy(device=VirtualDevice('r1', strict=False), connect_rate_limit=0) + self.conn.connect('r1') + self.addCleanup(self.conn.close, force=True) + + def testSecureModeIsTheDefault(self): + for source in '{sys.exec("true")}', '{sys.env("HOME")}', '{file.rm("x")}': + self.assertRaises(PermissionError, template.test, source) + self.assertRaises(PermissionError, template.eval, self.conn, source) + self.assertRaises(PermissionError, template.paste, self.conn, source) + template.test(source, secure=False) + + def testSecureFunctionsAreAllowed(self): + template.test('{sys.message("hello")}{file.exists("x")}') + + def testFunctionsOfTheCallerAreAllowed(self): + variables = template.eval(self.conn, '{x = shout("hi")}', shout=shout) + self.assertEqual(variables['x'], ['HI']) + + @unittest.skipIf(os.name == 'nt', 'echo is a shell builtin on Windows') + def testExecDoesNotUseAShell(self): + variables = template.eval(self.conn, '{sys.exec("echo a;echo b")}', secure=False) + self.assertEqual(variables['__response__'], ['a;echo b\n']) From a1da72688f9e54dc76d1e58fb63458e687ecb1f3 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Sun, 1 Nov 2026 00:54:33 +0100 Subject: [PATCH 016/250] Parse XML without entities, DTDs or network access A reply from a malicious device could make Exscript read local files (XXE), as lxml before 5.0 resolves external entities by default. lxml 5.0 or later is required now, which also covers the parsers in ncclient. --- CHANGES.md | 4 ++++ pyproject.toml | 2 +- src/Exscript/protocols/netconf.py | 10 +++++----- src/Exscript/util/xml.py | 16 +++++++++++++++- tests/Exscript/util/xmlTest.py | 28 ++++++++++++++++++++++++++++ uv.lock | 2 +- 6 files changed, 54 insertions(+), 8 deletions(-) create mode 100644 tests/Exscript/util/xmlTest.py diff --git a/CHANGES.md b/CHANGES.md index 626085c7..8db90e72 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -68,6 +68,10 @@ marked with **BREAKING**; a migration guide will accompany the release. - **BREAKING - `sys.exec` Without a Shell:** The template function `sys.exec` runs the command without a shell and waits for it to finish. Pipes, redirections and shell variables no longer work; use for example `sys.exec("sh -c '...'")`. +- **XML Parsing:** Exscript parses NETCONF RPCs and replies without resolving + entities, loading DTDs, or accessing the network, also with `huge_tree`. lxml 5.0 or + later is required, which no longer resolves external entities by default, also in + the parsers of ncclient. ## Protocols diff --git a/pyproject.toml b/pyproject.toml index 32c340cf..8393099b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,7 +37,7 @@ classifiers = [ "Topic :: Software Development :: Libraries :: Python Modules", ] dependencies = [ - "lxml", + "lxml>=5.0", # 5.0 stopped resolving external entities by default, also in ncclient "ncclient>=0.7,<1", "paramiko>=3.4,<6", "pycryptodomex", diff --git a/src/Exscript/protocols/netconf.py b/src/Exscript/protocols/netconf.py index d170672f..a771af0b 100644 --- a/src/Exscript/protocols/netconf.py +++ b/src/Exscript/protocols/netconf.py @@ -36,10 +36,10 @@ from ncclient.operations.errors import TimeoutExpiredError from ncclient.operations.rpc import RaiseMode, RPCReply from ncclient.transport.ssh import SSHSession -from ncclient.xml_ import NCElement, to_ele, to_xml +from ncclient.xml_ import NCElement, to_xml from paramiko.ssh_exception import SSHException -from ..util.xml import XMLValidationError, validate_xml +from ..util.xml import XMLValidationError, _parse, validate_xml from .exception import ( LoginFailure, ProtocolException, @@ -141,9 +141,9 @@ def get_response_element(self): if self.response is None: return None # send() was called without expect_prompt() if isinstance(self._ncc_response, RPCReply): - return to_ele(self._ncc_response.xml, huge_tree=self.huge_tree) + return _parse(self._ncc_response.xml, huge_tree=self.huge_tree) if isinstance(self._ncc_response, NCElement): - return to_ele(self._ncc_response.data_xml, huge_tree=self.huge_tree) + return _parse(self._ncc_response.data_xml, huge_tree=self.huge_tree) return None @staticmethod @@ -378,7 +378,7 @@ def _send(self, data): self._ncc_response = None self._validate_rpc_body(data) try: - element = to_ele(data) + element = _parse(data) self._ncc_response = self._ncc_manager.dispatch(element) # wrappers handle the self.data_sent_event except TimeoutExpiredError: diff --git a/src/Exscript/util/xml.py b/src/Exscript/util/xml.py index deabfd59..3a335f7f 100644 --- a/src/Exscript/util/xml.py +++ b/src/Exscript/util/xml.py @@ -26,6 +26,20 @@ from lxml import etree +# Entities are never resolved and DTDs and other resources are never loaded, +# which rules out XXE attacks; also with huge_tree, which only lifts size limits. +_parsers = { + huge_tree: etree.XMLParser( + resolve_entities=False, load_dtd=False, no_network=True, huge_tree=huge_tree + ) + for huge_tree in (False, True) +} + + +def _parse(string, huge_tree=False): + """Parses an XML string with the safe parser and returns the root element.""" + return etree.fromstring(string.encode('utf-8'), _parsers[huge_tree]) + class XMLValidationError(Exception): """ @@ -52,7 +66,7 @@ def validate_xml(string, *, silent=False): :return: Whether the string is valid XML in UTF-8. """ try: - etree.fromstring(string.encode("utf-8")) + _parse(string) except UnicodeEncodeError: if not silent: raise diff --git a/tests/Exscript/util/xmlTest.py b/tests/Exscript/util/xmlTest.py new file mode 100644 index 00000000..9d9e871c --- /dev/null +++ b/tests/Exscript/util/xmlTest.py @@ -0,0 +1,28 @@ +import os +import pathlib +import tempfile +import unittest + +from lxml import etree + +from Exscript.util import xml +from Exscript.util.xml import _parse, validate_xml + + +class xmlTest(unittest.TestCase): + CORRELATE = xml + + def testValidateXml(self): + self.assertTrue(validate_xml('')) + self.assertRaises(etree.XMLSyntaxError, validate_xml, '') + self.assertFalse(validate_xml('', silent=True)) + self.assertFalse(validate_xml('\udcff', silent=True)) + + def testExternalEntitiesAreNotResolved(self): + with tempfile.NamedTemporaryFile('w', suffix='.txt', delete=False) as fp: + fp.write('XXE-MARKER') + self.addCleanup(os.remove, fp.name) + url = pathlib.Path(fp.name).as_uri() + document = f']>&x;' + for huge_tree in False, True: + self.assertNotIn(b'XXE-MARKER', etree.tostring(_parse(document, huge_tree))) diff --git a/uv.lock b/uv.lock index c50b949a..d2877ba6 100644 --- a/uv.lock +++ b/uv.lock @@ -440,7 +440,7 @@ types = [ [package.metadata] requires-dist = [ - { name = "lxml" }, + { name = "lxml", specifier = ">=5.0" }, { name = "ncclient", specifier = ">=0.7,<1" }, { name = "opentelemetry-api", marker = "extra == 'otel'", specifier = ">=1.27" }, { name = "opentelemetry-instrumentation", marker = "extra == 'otel'", specifier = ">=0.48b0" }, From 0b4a2e87703bae0b75f3dfa8db9c4825d4e17915 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Sun, 1 Nov 2026 01:00:00 +0100 Subject: [PATCH 017/250] Label the telnetlib code derived from CPython The PSF License requires its notice and a summary of the changes in derived works; the file had only Exscript's MIT header. The package now declares the license expression MIT AND PSF-2.0. --- CHANGES.md | 3 ++ LICENSES/PSF-2.0.txt | 48 +++++++++++++++++++++++++++++ pyproject.toml | 4 +-- src/Exscript/protocols/telnetlib.py | 28 +++++++---------- 4 files changed, 64 insertions(+), 19 deletions(-) create mode 100644 LICENSES/PSF-2.0.txt diff --git a/CHANGES.md b/CHANGES.md index 8db90e72..88b9b07c 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -8,6 +8,9 @@ marked with **BREAKING**; a migration guide will accompany the release. ## General +- **License Labels:** Exscript remains MIT-licensed. `Exscript.protocols.telnetlib` is + derived from CPython's telnetlib, so it carries the PSF notice, the license text is + in `LICENSES/PSF-2.0.txt`, and the package declares `MIT AND PSF-2.0`. - **Python 2.7 Support:** - **Python 3.11+ Support:** - **Dependencies:** diff --git a/LICENSES/PSF-2.0.txt b/LICENSES/PSF-2.0.txt new file mode 100644 index 00000000..5fe8a31f --- /dev/null +++ b/LICENSES/PSF-2.0.txt @@ -0,0 +1,48 @@ +PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2 +-------------------------------------------- + +1. This LICENSE AGREEMENT is between the Python Software Foundation +("PSF"), and the Individual or Organization ("Licensee") accessing and +otherwise using this software ("Python") in source or binary form and +its associated documentation. + +2. Subject to the terms and conditions of this License Agreement, PSF hereby +grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce, +analyze, test, perform and/or display publicly, prepare derivative works, +distribute, and otherwise use Python alone or in any derivative version, +provided, however, that PSF's License Agreement and PSF's notice of copyright, +i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, +2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022, 2023 Python Software Foundation; +All Rights Reserved" are retained in Python alone or in any derivative version +prepared by Licensee. + +3. In the event Licensee prepares a derivative work that is based on +or incorporates Python or any part thereof, and wants to make +the derivative work available to others as provided herein, then +Licensee hereby agrees to include in any such work a brief summary of +the changes made to Python. + +4. PSF is making Python available to Licensee on an "AS IS" +basis. PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR +IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND +DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS +FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT +INFRINGE ANY THIRD PARTY RIGHTS. + +5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON +FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS +A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON, +OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. + +6. This License Agreement will automatically terminate upon a material +breach of its terms and conditions. + +7. Nothing in this License Agreement shall be deemed to create any +relationship of agency, partnership, or joint venture between PSF and +Licensee. This License Agreement does not grant permission to use PSF +trademarks or trade name in a trademark sense to endorse or promote +products or services of Licensee, or any third party. + +8. By copying, installing or otherwise using Python, Licensee +agrees to be bound by the terms and conditions of this License +Agreement. diff --git a/pyproject.toml b/pyproject.toml index 8393099b..e7b0919c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,8 +8,8 @@ dynamic = ["version"] description = "Automating Telnet, SSH and NETCONF" readme = "README.md" requires-python = ">=3.12" -license = "MIT" -license-files = ["COPYING"] +license = "MIT AND PSF-2.0" # PSF-2.0: protocols/telnetlib.py is derived from CPython +license-files = ["COPYING", "LICENSES/*"] authors = [{ name = "Samuel Abels", email = "knipknap@gmail.com" }] keywords = [ "exscript", diff --git a/src/Exscript/protocols/telnetlib.py b/src/Exscript/protocols/telnetlib.py index 2b851bea..b44d3b16 100644 --- a/src/Exscript/protocols/telnetlib.py +++ b/src/Exscript/protocols/telnetlib.py @@ -1,25 +1,19 @@ # +# Copyright (C) 2001-2023 Python Software Foundation; All Rights Reserved # Copyright (C) 2010-2017 Samuel Abels -# The MIT License (MIT) # -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files -# (the "Software"), to deal in the Software without restriction, -# including without limitation the rights to use, copy, modify, merge, -# publish, distribute, sublicense, and/or sell copies of the Software, -# and to permit persons to whom the Software is furnished to do so, -# subject to the following conditions: +# SPDX-License-Identifier: PSF-2.0 AND MIT # -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. +# This module is derived from the telnetlib module of CPython, which is +# licensed under the Python Software Foundation License Version 2; see +# LICENSES/PSF-2.0.txt. The changes for Exscript are licensed under the MIT +# License; see COPYING. # -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY -# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, -# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE -# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# Summary of the changes made to Python's telnetlib, as required by the PSF +# License: the data is decoded with a configurable encoding; the terminal type +# (TTYPE) and window size (NAWS) options are negotiated; a callback receives +# all data; and the client waits for any of several regular expressions in a +# search window that a cleanup function can process. """TELNET client class. Based on RFC 854: TELNET Protocol Specification, by J. Postel and From 154a5b770b750fd6cd204065546a959f55b76b1a Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Sun, 1 Nov 2026 01:04:43 +0100 Subject: [PATCH 018/250] Add a security policy, CodeQL, dependency audits and Dependabot SECURITY.md asks for private vulnerability reports through GitHub. Dependabot updates only the uv lockfile, so the lower bounds in pyproject.toml stay a deliberate decision. --- .github/dependabot.yml | 22 ++++++++++++ .github/workflows/security.yml | 63 ++++++++++++++++++++++++++++++++++ SECURITY.md | 28 +++++++++++++++ 3 files changed, 113 insertions(+) create mode 100644 .github/dependabot.yml create mode 100644 .github/workflows/security.yml create mode 100644 SECURITY.md diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..5357b601 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,22 @@ +version: 2 +updates: + - package-ecosystem: uv + directory: / + schedule: + interval: weekly + # A library keeps its lower bounds; only the lockfile used for development moves. + versioning-strategy: lockfile-only + cooldown: + default-days: 7 + groups: + python: + patterns: ["*"] + - package-ecosystem: github-actions + directory: / + schedule: + interval: weekly + cooldown: + default-days: 7 + groups: + actions: + patterns: ["*"] diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml new file mode 100644 index 00000000..8ff95115 --- /dev/null +++ b/.github/workflows/security.yml @@ -0,0 +1,63 @@ +name: Security + +on: + push: + branches: [master] + pull_request: + schedule: + - cron: "17 4 * * 1" # weekly: new advisories appear without code changes + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + codeql: + name: CodeQL + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write # uploads the results to code scanning + strategy: + fail-fast: false + matrix: + language: [python, actions] + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: github/codeql-action/init@9b3254085494b55a0f7c4c2b3d2d52cc712ce370 # v4.39.1 + with: + languages: ${{ matrix.language }} + build-mode: none + - uses: github/codeql-action/analyze@9b3254085494b55a0f7c4c2b3d2d52cc712ce370 # v4.39.1 + with: + category: /language:${{ matrix.language }} + + audit: + name: Known vulnerabilities in dependencies + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3 + - run: >- + uv export --frozen --all-extras --all-groups --no-emit-project + --format requirements.txt -o requirements.txt + - run: uvx pip-audit --disable-pip --strict -r requirements.txt + + dependency-review: + name: Dependency review + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/dependency-review-action@a1d282b36b6f3519aa1f3fc636f609c47dddb294 # v5.0.0 diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 00000000..f16738ae --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,28 @@ +# Security Policy + +## Supported Versions + +Security fixes are made for the latest release of Exscript. + +## Reporting a Vulnerability + +Please report vulnerabilities privately: open the +[Security tab](https://github.com/knipknap/exscript/security) of the repository and +choose "Report a vulnerability". Do not open a public issue or pull request. + +Please include the affected version, a description of the problem, and if possible +the steps or a script to reproduce it. Remove passwords, host names and other data of +your network from logs and recordings first. + +The maintainers confirm the report, keep you informed about the fix, and credit you +in the release notes unless you prefer otherwise. + +## Security Defaults + +- SSH host keys are verified against known_hosts before any credentials are sent; + unknown hosts are rejected by default (`host_key_policy`). +- Templates run in secure mode by default: they cannot run local commands or access + local files unless `secure=False` (CLI: `--insecure`) is given. +- XML is parsed without resolving entities, loading DTDs, or accessing the network. +- Passwords are not logged, and the OpenTelemetry instrumentation records commands + and responses only with `capture_payload=True`. From 23eab1698705eecdfb65572ce1589cc7368708af Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Sun, 1 Nov 2026 01:11:03 +0100 Subject: [PATCH 019/250] Remove eval() and the shell from the CLI A value entered at a VAR={#"PROMPT"} prompt went to eval(), which runs any Python code; ast.literal_eval() parses it now. --sleep ran "sleep" through os.system(), which also failed on Windows; Python parses the time now, with the units of sleep (s, m, h, d). --- CHANGES.md | 4 ++++ doc/exscript.rst | 5 +++-- pyproject.toml | 1 - scripts/exscript | 34 +++++++++++++++++++++-------- tests/scripts/exscriptTest.py | 40 +++++++++++++++++++++++++++++++++++ 5 files changed, 72 insertions(+), 12 deletions(-) create mode 100644 tests/scripts/exscriptTest.py diff --git a/CHANGES.md b/CHANGES.md index 88b9b07c..a1d8276f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -75,6 +75,10 @@ marked with **BREAKING**; a migration guide will accompany the release. entities, loading DTDs, or accessing the network, also with `huge_tree`. lxml 5.0 or later is required, which no longer resolves external entities by default, also in the parsers of ncclient. +- **CLI Without `eval()` and Shell:** A value entered at a `-d VAR={#"PROMPT"}` prompt + is parsed as a Python literal, such as a number or a list, instead of being run as + Python code. `--sleep` no longer runs the `sleep` command in a shell; it accepts + seconds with an optional suffix `m`, `h` or `d`, also on Windows. ## Protocols diff --git a/doc/exscript.rst b/doc/exscript.rst index 57107865..4a18e92c 100644 --- a/doc/exscript.rst +++ b/doc/exscript.rst @@ -83,8 +83,9 @@ following options are currently provided: --retry-login=NUM Defines the number of retries per host on login failure. Default is 0. --sleep=TIME Waits for the specified time before running the - script. TIME is a timespec as specified by the 'sleep' - Unix command. + script. TIME is a number of seconds, optionally with + the suffix m, h or d, like for the 'sleep' Unix + command; e.g. "90" or "1h 30m". --ssh-auto-verify Disables the verification of SSH host keys, so that any key is accepted. Highly insecure and not recommended. diff --git a/pyproject.toml b/pyproject.toml index e7b0919c..ce6509c6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -97,7 +97,6 @@ ignore = [ "tests/**" = ["S101", "S104", "S105", "S106", "S107", "S108", "S311"] # Security baseline: existing findings per file, to be reviewed; new code is checked. "demos/template/template.py" = ["S101"] -"scripts/exscript" = ["S307", "S605"] "src/Exscript/emulators/iosemu.py" = ["S105"] "src/Exscript/emulators/vdevice.py" = ["S105"] "src/Exscript/interpreter/string.py" = ["S105"] diff --git a/scripts/exscript b/scripts/exscript index 30c45ca4..f48b576e 100755 --- a/scripts/exscript +++ b/scripts/exscript @@ -23,11 +23,13 @@ # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. import argparse +import ast import getpass import importlib.util import os import re import sys +import time from pathlib import Path import Exscript.protocols.drivers @@ -53,6 +55,19 @@ bracket_expression = r"\{([^\]]*)\}" define_expression = rf"^\S+=(?:\S+|{bracket_expression})$" bracket_expression_re = re.compile(rf"^{bracket_expression}$") define_expression_re = re.compile(define_expression) +duration_re = re.compile(r"(\d+(?:\.\d*)?|\.\d+)([smhd]?)") +duration_units = {"": 1, "s": 1, "m": 60, "h": 3600, "d": 86400} + + +def duration(text): + """Returns the seconds of a time like the sleep command takes it, e.g. "1h 30m".""" + seconds = 0.0 + for part in text.split(): + match = duration_re.fullmatch(part) + if match is None: + raise ValueError(part) + seconds += float(match.group(1)) * duration_units[match.group(2)] + return seconds def expand_bracket(value_name, value): @@ -67,8 +82,8 @@ def expand_bracket(value_name, value): value = input(f"Please enter {prompt.lstrip('#')}: ") if prompt.startswith("#"): try: - value = eval(value) - except (SyntaxError, NameError): + value = ast.literal_eval(value) + except (ValueError, TypeError, SyntaxError, MemoryError, RecursionError): pass return value @@ -344,12 +359,10 @@ def run(): decorator = log_to_file(args.logdir, mode, args.delete_logs) function = decorator(function) - # Wait until the specified time. We are using os.system because - # it allows for specifying absolute times, not just the number - # of seconds like Python's time.sleep(). + # Wait until the specified time. if args.sleep: - print(f"Waiting for {args.sleep}...", end="", flush=True) - os.system(f"sleep {args.sleep}") + print(f"Waiting for {args.sleep:g} seconds...", end="", flush=True) + time.sleep(args.sleep) print("time expired, starting script.") # Run the template. @@ -458,7 +471,8 @@ Defines a variable that is passed to the script. PAIR has the following syntax: STRING=STRING. Use a prompt with the following syntax: VAR={"PROMPT"}. Use a password prompt with the following syntax: VAR={\\!"PROMPT"}. -Evaluate the value of a prompt with the following syntax: VAR={#"PROMPT"}. +Evaluate the value of a prompt as a Python literal (e.g. a number or a list) +with the following syntax: VAR={#"PROMPT"}. """.strip(), ) @@ -670,10 +684,12 @@ Defines the number of retries per host on login failure. Default is 0. parser.add_argument( "--sleep", metavar="TIME", + type=duration, default="", help=""" Waits for the specified time before running the script. -TIME is a timespec as specified by the 'sleep' Unix command. +TIME is a number of seconds, optionally with the suffix m, h or d, +like for the 'sleep' Unix command; e.g. "90" or "1h 30m". """.strip(), ) diff --git a/tests/scripts/exscriptTest.py b/tests/scripts/exscriptTest.py new file mode 100644 index 00000000..2e324d23 --- /dev/null +++ b/tests/scripts/exscriptTest.py @@ -0,0 +1,40 @@ +import importlib.machinery +import importlib.util +import os +import unittest +from unittest import mock + +SCRIPT = os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, 'scripts', 'exscript') + + +def load_cli(): + loader = importlib.machinery.SourceFileLoader('exscript_cli', SCRIPT) + spec = importlib.util.spec_from_file_location('exscript_cli', SCRIPT, loader=loader) + module = importlib.util.module_from_spec(spec) + loader.exec_module(module) + return module + + +class exscriptTest(unittest.TestCase): + cli = load_cli() + + def testDuration(self): + duration = self.cli.duration + self.assertEqual(duration(''), 0) + self.assertEqual(duration('90'), 90) + self.assertEqual(duration('1.5m'), 90) + self.assertEqual(duration('1h 30m'), 5400) + self.assertEqual(duration('2d'), 172800) + for invalid in 'tomorrow', '1x', '-1', '1; rm -rf ~': + self.assertRaises(ValueError, duration, invalid) + + def testPromptValuesAreLiteralsOnly(self): + cases = { + '[1, 2]': [1, 2], + '42': 42, + 'plain text': 'plain text', + '__import__("os").getcwd()': '__import__("os").getcwd()', + } + for typed, expected in cases.items(): + with mock.patch('builtins.input', return_value=typed): + self.assertEqual(self.cli.expand_bracket('count', '{#count}'), expected) From aa0532b0faca8d8fec87c5280c5ad8b1bd304a6d Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 3 Nov 2026 00:09:00 +0100 Subject: [PATCH 020/250] Fire job_started before the job runs MainLoop fired job_started after starting the job's thread, so a fast job could complete first, and the queue showed no status bar for it. Job.start() now calls an on_start callback before it starts the child. --- src/Exscript/workqueue/job.py | 4 +++- src/Exscript/workqueue/mainloop.py | 6 ++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Exscript/workqueue/job.py b/src/Exscript/workqueue/job.py index a307d0f9..560a887e 100644 --- a/src/Exscript/workqueue/job.py +++ b/src/Exscript/workqueue/job.py @@ -105,9 +105,11 @@ def __init__(self, function, name, times, data): self.child = None self.watcher = None - def start(self, child_cls, on_complete): + def start(self, child_cls, on_complete, on_start=None): self.child = child_cls(self.id, self.func, self.name, self.data) self.child.failures = self.failures + if on_start is not None: + on_start(self.child) # before the child runs, so it always precedes on_complete self.watcher = _ChildWatcher(self.child, partial(on_complete, self)) self.watcher.start() diff --git a/src/Exscript/workqueue/mainloop.py b/src/Exscript/workqueue/mainloop.py index 8d132cff..15baeea2 100644 --- a/src/Exscript/workqueue/mainloop.py +++ b/src/Exscript/workqueue/mainloop.py @@ -119,8 +119,7 @@ def _on_job_completed(self, job, exc_info): # Remove the watcher from the queue, and re-enque if needed. if exc_info and job.failures < job.times: self._dbg(3, 'Restarting job "%s"' % job.name) - job.start(self.job_cls, self._on_job_completed) - self.job_started_event(job.child) + job.start(self.job_cls, self._on_job_completed, self.job_started_event) else: self.collection.task_done(job) @@ -135,6 +134,5 @@ def run(self): break # self.collection.stop() was called. self.job_init_event(job) - job.start(self.job_cls, self._on_job_completed) - self.job_started_event(job.child) + job.start(self.job_cls, self._on_job_completed, self.job_started_event) self._dbg(3, 'Job "%s" started.' % job.name) From 898bc40ce07b205909e193847f770038c153b987 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 3 Nov 2026 00:09:00 +0100 Subject: [PATCH 021/250] Make the API report work against the 2.x branch layout The api-diff job searched for the package only in src/, but master still has it in the repository root, so griffe failed to load the base version. --- .github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d9a34b90..7eb0f786 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -82,4 +82,5 @@ jobs: - name: Compare with the target branch env: BASE_REF: ${{ github.base_ref }} - run: uvx griffe check Exscript --search src --against "origin/$BASE_REF" --verbose + # Both search paths: the target branch may still have the package in the repository root. + run: uvx griffe check Exscript --search src --search . --against "origin/$BASE_REF" --verbose From 5b25d11b35e437d8f6257b6cff274bdf77473ecf Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 3 Nov 2026 00:48:16 +0100 Subject: [PATCH 022/250] Base the Telnet client on CPython 3.12's telnetlib The old copy blocked on Telnet commands split across reads, crashed on IAC NOP, broke multibyte characters, connected over IPv4 only and logged passwords. The new code parses whole reads with a state machine and keeps an incomplete command for the next read. --- CHANGES.md | 29 +- pyproject.toml | 5 +- src/Exscript/protocols/telnet.py | 1 - src/Exscript/protocols/telnetlib.py | 925 +++++++--------------- tests/Exscript/protocols/telnetlibTest.py | 150 ++++ uv.lock | 94 +++ 6 files changed, 548 insertions(+), 656 deletions(-) create mode 100644 tests/Exscript/protocols/telnetlibTest.py diff --git a/CHANGES.md b/CHANGES.md index a1d8276f..60584e97 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -9,7 +9,7 @@ marked with **BREAKING**; a migration guide will accompany the release. ## General - **License Labels:** Exscript remains MIT-licensed. `Exscript.protocols.telnetlib` is - derived from CPython's telnetlib, so it carries the PSF notice, the license text is + derived from CPython's telnetlib (3.12), so it carries the PSF notice, the license text is in `LICENSES/PSF-2.0.txt`, and the package declares `MIT AND PSF-2.0`. - **Python 2.7 Support:** - **Python 3.11+ Support:** @@ -97,6 +97,33 @@ marked with **BREAKING**; a migration guide will accompany the release. - `Exscript.util.url.Url.from_string(url, default_protocol='ssh')` - `exscript` CLI: `-p`/`--protocol` +### Telnet + +- **Telnet Client Based on CPython 3.12:** `Exscript.protocols.telnetlib` is now based + on the last telnetlib of CPython (3.12; the module was removed in 3.13). Fixes: + - connections use IPv6, too, and try all addresses of the host; + - Telnet commands are parsed in whole reads instead of byte by byte, which cuts the + CPU time of large responses by about a quarter; + - a command split across two reads no longer blocks until more data arrives; + - `IAC NOP` and option code 0 no longer crash the connection with a `TypeError`; a + `DO` for option 0 is refused with `WONT` instead of being ignored; + - subnegotiations end at `IAC SE` only, as RFC 855 requires; + - multibyte characters split across reads are decoded correctly, e.g. with + `encoding='utf-8'`; + - sent data is written completely (`sendall()`), and window sizes that contain + the byte 255 are escaped; + - passwords are no longer logged at debug level 5. +- **BREAKING - `Exscript.protocols.telnetlib` Is Internal:** The module contains only + what the Telnet adapter uses. Removed: the methods `read_until()`, `read_some()`, + `read_eager()`, `read_very_eager()`, `read_lazy()`, `read_very_lazy()`, `interact()` + and `mt_interact()` with their helpers, `sock_avail()`, `get_socket()`, `fileno()` + and `set_receive_callback()`; the function `test()`; the option constants other + than those Exscript negotiates; and the `stdout` and `stderr` arguments. The + remaining constants are integers instead of bytes, and the options of the + constructor are keyword-only. Use `Exscript.protocols.Telnet`. If you need a drop-in + replacement for the telnetlib module that Python 3.13 removed, use the package + `standard-telnetlib`. + ### Protocol Drivers - **BREAKING - Graceful Logout:** `Protocol.close(force=False)` now calls the driver's diff --git a/pyproject.toml b/pyproject.toml index ce6509c6..9f927335 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -61,6 +61,7 @@ test = [ "pytest-cov>=7.1", "opentelemetry-sdk>=1.27", "opentelemetry-instrumentation>=0.48b0", + "hypothesis>=6.170.14", ] types = ["mypy>=2.4"] dev = [{ include-group = "test" }, { include-group = "types" }] @@ -123,10 +124,6 @@ ignore_missing_imports = true # paramiko, ncclient and others ship no type hint warn_unused_ignores = true warn_redundant_casts = true -[[tool.mypy.overrides]] -module = ["Exscript.protocols.telnetlib"] -ignore_errors = true # vendored copy of the former standard library module - [tool.pytest.ini_options] testpaths = ["tests"] python_files = ["*Test.py"] diff --git a/src/Exscript/protocols/telnet.py b/src/Exscript/protocols/telnet.py index 530d7de3..c9896f84 100644 --- a/src/Exscript/protocols/telnet.py +++ b/src/Exscript/protocols/telnet.py @@ -60,7 +60,6 @@ def _connect(self): connect_timeout=self.connect_timeout, termsize=(rows, cols), termtype=self.termtype, - stderr=self.stderr, receive_callback=self._telnetlib_received, ) self._tn.cancel_expect = self._expect_canceled # sync states diff --git a/src/Exscript/protocols/telnetlib.py b/src/Exscript/protocols/telnetlib.py index b44d3b16..4b1798a9 100644 --- a/src/Exscript/protocols/telnetlib.py +++ b/src/Exscript/protocols/telnetlib.py @@ -4,654 +4,316 @@ # # SPDX-License-Identifier: PSF-2.0 AND MIT # -# This module is derived from the telnetlib module of CPython, which is +# This module is derived from the telnetlib module of CPython 3.12, which is # licensed under the Python Software Foundation License Version 2; see # LICENSES/PSF-2.0.txt. The changes for Exscript are licensed under the MIT # License; see COPYING. # # Summary of the changes made to Python's telnetlib, as required by the PSF -# License: the data is decoded with a configurable encoding; the terminal type -# (TTYPE) and window size (NAWS) options are negotiated; a callback receives -# all data; and the client waits for any of several regular expressions in a -# search window that a cleanup function can process. -"""TELNET client class. - -Based on RFC 854: TELNET Protocol Specification, by J. Postel and -J. Reynolds - -Example: - ->>> from telnetlib import Telnet ->>> tn = Telnet('www.python.org', 79) # connect to finger port ->>> tn.write('guido\r\n') ->>> print tn.read_all() -Login Name TTY Idle When Where -guido Guido van Rossum pts/2 snag.cnri.reston.. - ->>> - -Note that read_all() won't read until eof -- it just reads some data --- but it guarantees to read at least one byte unless EOF is hit. - -It is possible to pass a Telnet object to select.select() in order to -wait until more data is available. Note that in this case, -read_eager() may return '' even if there was data on the socket, -because the protocol negotiation may have eaten the data. This is why -EOFError is needed in some cases to distinguish between "no data" and -"connection closed" (since the socket also appears ready for reading -when it is closed). - -Bugs: -- may hang when connection is slow in the middle of an IAC sequence - -To do: -- option negotiation -- timeout should be intrinsic to the connection object instead of an - option on one of the read calls only +# License: received data is decoded with a configurable encoding and passed to +# a callback, and NUL bytes are removed from it (the original also removed +# XON); the terminal type (TTYPE) and window size (NAWS) options are +# negotiated and the echo option is accepted, all other options are refused; +# commands are parsed in whole reads instead of byte by byte; the client waits +# for any of several regular expressions in a search window that a cleanup +# function can process; the read_*() variants, interact(), the option +# negotiation callback and the deprecation warning were removed. +""" +The Telnet client (RFC 854) of Exscript's Telnet adapter, derived from the +telnetlib module of CPython. +This module is internal; use :class:`Exscript.protocols.Telnet` instead. """ +import codecs import logging +import re import select -import selectors import socket import struct -import sys -import time from io import StringIO -from time import monotonic as _time +from time import monotonic -__all__ = ["Telnet"] +__all__ = ['Telnet'] _logger = logging.getLogger(__name__) _logger.addHandler(logging.NullHandler()) -# Tunable parameters -DEBUGLEVEL = 0 - -# Telnet protocol defaults TELNET_PORT = 23 -# Telnet protocol characters (don't change) -IAC = chr(255).encode('latin-1') # "Interpret As Command" -DONT = chr(254).encode('latin-1') -DO = chr(253).encode('latin-1') -WONT = chr(252).encode('latin-1') -WILL = chr(251).encode('latin-1') -SB = chr(250).encode('latin-1') -SE = chr(240).encode('latin-1') -theNULL = chr(0).encode('latin-1') - -# Telnet protocol options code (don't change).encode('latin-1') -# These ones all come from arpa/telnet.h -BINARY = chr(0).encode('latin-1') # 8-bit data path -ECHO = chr(1).encode('latin-1') # echo -RCP = chr(2).encode('latin-1') # prepare to reconnect -SGA = chr(3).encode('latin-1') # suppress go ahead -NAMS = chr(4).encode('latin-1') # approximate message size -STATUS = chr(5).encode('latin-1') # give status -TM = chr(6).encode('latin-1') # timing mark -RCTE = chr(7).encode('latin-1') # remote controlled transmission and echo -NAOL = chr(8).encode('latin-1') # negotiate about output line width -NAOP = chr(9).encode('latin-1') # negotiate about output page size -NAOCRD = chr(10).encode('latin-1') # negotiate about CR disposition -NAOHTS = chr(11).encode('latin-1') # negotiate about horizontal tabstops -NAOHTD = chr(12).encode('latin-1') # negotiate about horizontal tab disposition -NAOFFD = chr(13).encode('latin-1') # negotiate about formfeed disposition -NAOVTS = chr(14).encode('latin-1') # negotiate about vertical tab stops -NAOVTD = chr(15).encode('latin-1') # negotiate about vertical tab disposition -NAOLFD = chr(16).encode('latin-1') # negotiate about output LF disposition -XASCII = chr(17).encode('latin-1') # extended ascii character set -LOGOUT = chr(18).encode('latin-1') # force logout -BM = chr(19).encode('latin-1') # byte macro -DET = chr(20).encode('latin-1') # data entry terminal -SUPDUP = chr(21).encode('latin-1') # supdup protocol -SUPDUPOUTPUT = chr(22).encode('latin-1') # supdup output -SNDLOC = chr(23).encode('latin-1') # send location -TTYPE = chr(24).encode('latin-1') # terminal type -EOR = chr(25).encode('latin-1') # end or record -TUID = chr(26).encode('latin-1') # TACACS user identification -OUTMRK = chr(27).encode('latin-1') # output marking -TTYLOC = chr(28).encode('latin-1') # terminal location number -VT3270REGIME = chr(29).encode('latin-1') # 3270 regime -X3PAD = chr(30).encode('latin-1') # X.3 PAD -NAWS = chr(31).encode('latin-1') # window size -TSPEED = chr(32).encode('latin-1') # terminal speed -LFLOW = chr(33).encode('latin-1') # remote flow control -LINEMODE = chr(34).encode('latin-1') # Linemode option -XDISPLOC = chr(35).encode('latin-1') # X Display Location -OLD_ENVIRON = chr(36).encode('latin-1') # Old - Environment variables -AUTHENTICATION = chr(37).encode('latin-1') # Authenticate -ENCRYPT = chr(38).encode('latin-1') # Encryption option -NEW_ENVIRON = chr(39).encode('latin-1') # New - Environment variables -# the following ones come from -# http://www.iana.org/assignments/telnet-options -# Unfortunately, that document does not assign identifiers -# to all of them, so we are making them up -TN3270E = chr(40).encode('latin-1') # TN3270E -XAUTH = chr(41).encode('latin-1') # XAUTH -CHARSET = chr(42).encode('latin-1') # CHARSET -RSP = chr(43).encode('latin-1') # Telnet Remote Serial Port -COM_PORT_OPTION = chr(44).encode('latin-1') # Com Port Control Option -SUPPRESS_LOCAL_ECHO = chr(45).encode('latin-1') # Telnet Suppress Local Echo -TLS = chr(46).encode('latin-1') # Telnet Start TLS -KERMIT = chr(47).encode('latin-1') # KERMIT -SEND_URL = chr(48).encode('latin-1') # SEND-URL -FORWARD_X = chr(49).encode('latin-1') # FORWARD_X -PRAGMA_LOGON = chr(138).encode('latin-1') # TELOPT PRAGMA LOGON -SSPI_LOGON = chr(139).encode('latin-1') # TELOPT SSPI LOGON -PRAGMA_HEARTBEAT = chr(140).encode('latin-1') # TELOPT PRAGMA HEARTBEAT -EXOPL = chr(255).encode('latin-1') # Extended-Options-List - -SEND_TTYPE = chr(1).encode('latin-1') - -# poll/select have the advantage of not requiring any extra file descriptor, -# contrarily to epoll/kqueue (also, they require a single syscall). -if hasattr(selectors, 'PollSelector'): - _TelnetSelector = selectors.PollSelector -else: - _TelnetSelector = selectors.SelectSelector +# Telnet commands (RFC 854) and the options that Exscript negotiates. +IAC = 255 # interpret as command +DONT = 254 +DO = 253 +WONT = 252 +WILL = 251 +SB = 250 # subnegotiation begin +SE = 240 # subnegotiation end +ECHO = 1 +TTYPE = 24 # terminal type (RFC 1091) +NAWS = 31 # window size (RFC 1073) +TTYPE_IS = 0 +TTYPE_SEND = 1 +_NEGOTIATION = {DO: 'DO', DONT: 'DONT', WILL: 'WILL', WONT: 'WONT'} +_IAC = bytes([IAC]) -class Telnet: - """Telnet interface class. - - An instance of this class represents a connection to a telnet - server. The instance is initially not connected; the open() - method must be used to establish a connection. Alternatively, the - host name and optional port number can be passed to the - constructor, too. - - Don't try to reopen an already connected instance. - - This class has many read_*() methods. Note that some of them - raise EOFError when the end of the connection is read, because - they can return an empty string for other reasons. See the - individual doc strings. - - read_until(expected, [timeout]) - Read until the expected string has been seen, or a timeout is - hit (default is no timeout); may block. +# _waitfor() searches the end of the received text after every read, so its +# search window covers all data only with small reads. +_READ_SIZE = 64 - read_all() - Read all data until EOF; may block. - read_some() - Read at least one byte or EOF; may block. +def _command(*codes): + return bytes([IAC, *codes]) - read_very_eager() - Read all data available already queued or on the socket, - without blocking. - read_eager() - Read either data already queued or some data available on the - socket, without blocking. - - read_lazy() - Read all data in the raw queue (processing it first), without - doing any socket I/O. - - read_very_lazy() - Reads all data in the cooked queue, without doing any socket - I/O. +class Telnet: + """ + A Telnet connection that answers option negotiation and delivers the + received text to a callback and to expect(). """ - def __init__(self, host=None, port=0, encoding='latin-1', **kwargs): - """Constructor. - - When called without arguments, create an unconnected instance. - With a hostname argument, it connects the instance; a port - number is optional. - + def __init__( + self, + host=None, + port=0, + encoding='latin-1', + *, + connect_timeout=None, + termsize=None, + termtype='dumb', + receive_callback=None, + ): """ - self._logger = _logger - self.debuglevel = DEBUGLEVEL - self.can_naws = False + Constructor. Connects to the host if one is given. + + :type host: str|None + :param host: The host name or address. + :type port: int + :param port: The TCP port; 0 is the Telnet port 23. + :type encoding: str + :param encoding: The encoding of the sent and received text. + :type connect_timeout: float|None + :param connect_timeout: The timeout of socket operations in seconds. + :type termsize: tuple(int, int)|None + :param termsize: The rows and columns of the terminal (NAWS). + :type termtype: str + :param termtype: The terminal type (TTYPE). + :type receive_callback: callable|None + :param receive_callback: Called with every received text. + """ + self.debuglevel = 0 self.host = host self.port = port self.sock = None + self.encoding = encoding + self.connect_timeout = connect_timeout + self.window_size = termsize + self.termtype = termtype + self.data_callback = receive_callback self.cancel_expect = False - self.rawq = b'' - self.irawq = 0 + self.can_naws = False + self.eof = False + self.rawq = b'' # received bytes that end with an incomplete command self.cookedq = StringIO() - self.eof = 0 - self.encoding = encoding - self.connect_timeout = kwargs.get('connect_timeout', None) - self.window_size = kwargs.get('termsize') - self.stdout = kwargs.get('stdout', sys.stdout) - self.stderr = kwargs.get('stderr', sys.stderr) - self.termtype = kwargs.get('termtype', 'dumb') - self.data_callback = kwargs.get('receive_callback', None) - self.data_callback_kwargs = {} + self._decoder = codecs.getincrementaldecoder(encoding)() + self._sb = None # the data of the subnegotiation in progress if host: self.open(host, port) def open(self, host, port=0): - """Connect to a host. - - The optional second argument is the port number, which - defaults to the standard telnet port (23). - - Don't try to reopen an already connected instance. + """ + Connects to the host, trying all of its addresses (IPv4 and IPv6). + :type host: str + :param host: The host name or address. + :type port: int + :param port: The TCP port; 0 is the Telnet port 23. """ - self.eof = 0 - if not port: - port = TELNET_PORT + self.eof = False self.host = host - self.port = port - msg = "getaddrinfo returns an empty list" - for res in socket.getaddrinfo(host, port, socket.AF_INET, socket.SOCK_STREAM): - af, socktype, proto, canonname, sa = res - try: - self.sock = socket.socket(af, socktype, proto) - self.sock.settimeout(self.connect_timeout) - self.sock.connect(sa) - except OSError as msg_: - msg = f'{msg_} => telnet://{host}:{port}' - if self.sock: - self.sock.close() - self.sock = None - continue - break - if not self.sock: - raise OSError(msg) + self.port = port or TELNET_PORT + try: + self.sock = socket.create_connection((host, self.port), self.connect_timeout) + except OSError as exc: + raise OSError(f'{exc} => telnet://{host}:{self.port}') from exc def msg(self, msg, *args): - """Print a debug message, when the debug level is > 0. - - If extra arguments are present, they are substituted in the - message using the standard string formatting operator. - - """ + """Logs a debug message if the debug level is above 0.""" if self.debuglevel > 0: - msg = '' + msg - if args: - msg = msg % args - self._logger.debug(msg) + _logger.debug(msg, *args) def set_debuglevel(self, debuglevel): - """Set the debug level. - - The higher it is, the more debug output you get (on stdout). + """ + Sets the debug level; above 0, the received data and the option + negotiation are logged. + :type debuglevel: int + :param debuglevel: The debug level. """ self.debuglevel = debuglevel def close(self): - """Close the connection.""" - if self.sock: - self.sock.close() - self.sock = 0 - self.eof = 1 - - def get_socket(self): - """Return the socket object used internally.""" - return self.sock - - def fileno(self): - """Return the fileno() of the socket object used internally.""" - return self.sock.fileno() + """Closes the connection.""" + sock, self.sock = self.sock, None + self.eof = True + if sock: + sock.close() def write(self, buffer): - """Write a string to the socket, doubling any IAC characters. - - Can block if the connection is blocked. May raise - socket.error if the connection is closed. - - """ - if type(buffer) == type(0): - buffer = chr(buffer) - elif not isinstance(buffer, bytes): - buffer = buffer.encode(self.encoding) - if IAC in buffer: - buffer = buffer.replace(IAC, IAC + IAC) - self.msg("Send: %s", repr(buffer)) - self.sock.send(buffer) - - def read_until(self, match, timeout=None): - """Read until a given string is encountered or until timeout. - When no match is found, return whatever is available instead, - possibly the empty string. Raise EOFError if the connection - is closed and no cooked data is available. """ - n = len(match) - self.process_rawq() - i = self.rawq.find(match) - if i >= 0: - i = i + n - buf = self.rawq[:i] - self.rawq = self.rawq[i:] - return buf - if timeout is not None: - deadline = _time() + timeout - with _TelnetSelector() as selector: - selector.register(self, selectors.EVENT_READ) - while not self.eof: - if selector.select(timeout): - i = max(0, len(self.rawq) - n) - self.fill_rawq() - self.process_rawq() - i = self.rawq.find(match, i) - if i >= 0: - i = i + n - buf = self.rawq[:i] - self.rawq = self.rawq[i:] - return buf - if timeout is not None: - timeout = deadline - _time() - if timeout < 0: - break - return self.read_very_lazy() - - def read_all(self): - """Read all data until EOF; block until connection closed.""" - self.process_rawq() - while not self.eof: - self.fill_rawq() - self.process_rawq() - buf = self.cookedq.getvalue() - self.cookedq.seek(0) - self.cookedq.truncate() - return buf - - def read_some(self): - """Read at least one byte of cooked data unless EOF is hit. - - Return '' if EOF is hit. Block if no data is immediately - available. + Sends the given text or bytes, doubling the IAC bytes. The data is not + logged, because it may contain a password. + :type buffer: str|bytes + :param buffer: The data to send. """ - self.process_rawq() - while self.cookedq.tell() == 0 and not self.eof: - self.fill_rawq() - self.process_rawq() - buf = self.cookedq.getvalue() - self.cookedq.seek(0) - self.cookedq.truncate() - return buf - - def read_very_eager(self): - """Read everything that's possible without blocking in I/O (eager). - - Raise EOFError if connection closed and no cooked data - available. Return '' if no cooked data available otherwise. - Don't block unless in the midst of an IAC sequence. - - """ - self.process_rawq() - while not self.eof and self.sock_avail(): - self.fill_rawq() - self.process_rawq() - return self.read_very_lazy() - - def read_eager(self): - """Read readily available data. - - Raise EOFError if connection closed and no cooked data - available. Return '' if no cooked data available otherwise. - Don't block unless in the midst of an IAC sequence. - - """ - self.process_rawq() - while self.cookedq.tell() == 0 and not self.eof and self.sock_avail(): - self.fill_rawq() - self.process_rawq() - return self.read_very_lazy() - - def read_lazy(self): - """Process and return data that's already in the queues (lazy). - - Raise EOFError if connection closed and no data available. - Return '' if no cooked data available otherwise. Don't block - unless in the midst of an IAC sequence. - - """ - self.process_rawq() - return self.read_very_lazy() - - def read_very_lazy(self): - """Return any data available in the cooked queue (very lazy). - - Raise EOFError if connection closed and no data available. - Return '' if no cooked data available otherwise. Don't block. - - """ - buf = self.cookedq.getvalue() - self.cookedq.seek(0) - self.cookedq.truncate() - if not buf and self.eof and not self.rawq: - raise EOFError('telnet connection closed') - return buf - - def set_receive_callback(self, callback, *args, **kwargs): - """The callback function called after each receipt of any data.""" - self.data_callback = callback - self.data_callback_kwargs = kwargs + if isinstance(buffer, str): + buffer = buffer.encode(self.encoding) + self.sock.sendall(buffer.replace(_IAC, _IAC + _IAC)) def set_window_size(self, rows, cols): """ - Change the size of the terminal window, if the remote end supports - NAWS. If it doesn't, the method returns silently. + Sets the window size, and sends it if the server asked for it (NAWS). + + :type rows: int + :param rows: The number of rows. + :type cols: int + :param cols: The number of columns. """ + self.window_size = rows, cols if not self.can_naws: return - self.window_size = rows, cols - size = struct.pack('!HH', cols, rows) - self.sock.send(IAC + SB + NAWS + size + IAC + SE) + size = struct.pack('!HH', min(cols, 0xFFFF), min(rows, 0xFFFF)) + self.sock.sendall(_command(SB, NAWS) + size.replace(_IAC, _IAC + _IAC) + _command(SE)) + + def _negotiate(self, command, option): + self.msg('IAC %s %d', _NEGOTIATION[command], option) + if command == DO and option in (TTYPE, NAWS): + self.sock.sendall(_command(WILL, option)) + if option == NAWS: + self.can_naws = True + if self.window_size: + self.set_window_size(*self.window_size) + elif command in (DO, DONT): + self.sock.sendall(_command(WONT, option)) + elif option == ECHO: + # Exscript always asks for the echo, also when the server refuses it. + self.sock.sendall(_command(DO, option)) + else: + self.sock.sendall(_command(DONT, option)) + + def _subnegotiate(self, data): + self.msg('IAC SB %r', data) + if data == bytes([TTYPE, TTYPE_SEND]): + self.msg('Indicating support for terminal type %s.', self.termtype) + ttype = self.termtype.encode('latin-1') + self.sock.sendall(_command(SB, TTYPE, TTYPE_IS) + ttype + _command(SE)) def process_rawq(self): - """Transfer from raw queue to cooked queue. - - Set self.eof when connection is closed. Don't block unless in - the midst of an IAC sequence. """ - buf = b'' - try: - while self.rawq: - # Handle non-IAC first (normal data). - char = self.rawq_getchar() - if char != IAC: - buf = buf + char - continue - - # Interpret the command byte that follows after the IAC code. - command = self.rawq_getchar() - if command == theNULL: - self.msg('IAC NOP') - continue - elif command == IAC: - self.msg('IAC DATA') - buf = buf + command + Moves the received data from the raw queue to the cooked queue, + removes the Telnet commands and answers the option negotiation. + Keeps an incomplete command in the raw queue until more data arrives, + so it never blocks. + """ + data, self.rawq = self.rawq, b'' + cooked = bytearray() + pos, end = 0, len(data) + while pos < end: + iac = data.find(IAC, pos) + if self._sb is not None: # inside a subnegotiation + if iac < 0: + self._sb += data[pos:] + break + self._sb += data[pos:iac] + if iac + 1 == end: + self.rawq = data[iac:] + break + code = data[iac + 1] + if code == IAC: + self._sb.append(IAC) + pos = iac + 2 continue - - # DO: Indicates the request that the other party perform, - # or confirmation that you are expecting the other party - # to perform, the indicated option. - elif command == DO: - opt = self.rawq_getchar() - # int 0 produces an empty byte string which then has no ord - # we can skip its processing - if opt == b"": - continue - self.msg('IAC DO %s', ord(opt)) - if opt == TTYPE: - self.sock.send(IAC + WILL + opt) - elif opt == NAWS: - self.sock.send(IAC + WILL + opt) - self.can_naws = True - if self.window_size: - self.set_window_size(*self.window_size) - else: - self.sock.send(IAC + WONT + opt) - - # DON'T: Indicates the demand that the other party stop - # performing, or confirmation that you are no longer - # expecting the other party to perform, the indicated - # option. - elif command == DONT: - opt = self.rawq_getchar() - self.msg('IAC DONT %s', ord(opt)) - self.sock.send(IAC + WONT + opt) - - # SB: Indicates that what follows is subnegotiation of the - # indicated option. - elif command == SB: - opt = self.rawq_getchar() - self.msg('IAC SUBCOMMAND %d', ord(opt)) - - # We only handle the TTYPE command, so skip all other - # commands. - if opt != TTYPE: - while self.rawq_getchar() != SE: - pass - continue - - # We also only handle the SEND_TTYPE option of TTYPE, - # so skip everything else. - subopt = self.rawq_getchar() - if subopt != SEND_TTYPE: - while self.rawq_getchar() != SE: - pass - continue - - # Mandatory end of the IAC subcommand. - iac = self.rawq_getchar() - end = self.rawq_getchar() - if (iac, end) != (IAC, SE): - # whoops, that's an unexpected response... - self.msg('Expected IAC SE, but got %d %d.', ord(iac), ord(end)) - self.msg('IAC SUBCOMMAND_END') - - # Send the next supported terminal. - ttype = self.termtype.encode('latin-1') - self.msg('Indicating support for terminal type %s.', ttype) - self.sock.send(IAC + SB + TTYPE + theNULL + ttype + IAC + SE) - elif command in (WILL, WONT): - opt = self.rawq_getchar() - self.msg('IAC %s %d', command == WILL and 'WILL' or 'WONT', ord(opt)) - if opt == ECHO: - self.sock.send(IAC + DO + opt) - else: - self.sock.send(IAC + DONT + opt) + sb, self._sb = bytes(self._sb), None + if code == SE: + self._subnegotiate(sb) + pos = iac + 2 else: - self.msg('IAC %d not recognized.' % ord(command)) - except EOFError: # raised by self.rawq_getchar() - pass - buf = buf.decode(self.encoding) - self.cookedq.write(buf) - if self.data_callback is not None: - self.data_callback(buf, **self.data_callback_kwargs) - - def rawq_getchar(self): - """Get next char from raw queue. - - Block if no data is immediately available. Raise EOFError when - connection is closed. - - """ - if not self.rawq: - self.fill_rawq() - if self.eof: - raise EOFError - - c = self.rawq[self.irawq] - c = c.to_bytes((c.bit_length() + 7) // 8, 'big') - - self.irawq += 1 - if self.irawq >= len(self.rawq): + pos = iac # the subnegotiation is not terminated: handle the command + continue + if iac < 0: + cooked += data[pos:] + break + cooked += data[pos:iac] + if iac + 1 == end or (iac + 2 == end and data[iac + 1] in _NEGOTIATION): + self.rawq = data[iac:] + break + code = data[iac + 1] + pos = iac + 2 + if code == IAC: + cooked.append(IAC) + elif code in _NEGOTIATION: + self._negotiate(code, data[pos]) + pos += 1 + elif code == SB: + self._sb = bytearray() + else: + self.msg('IAC %d ignored', code) # e.g. NOP or go ahead + if self.eof: # an incomplete command can no longer complete self.rawq = b'' - self.irawq = 0 - return c + self._sb = None + text = self._decoder.decode(bytes(cooked).replace(b'\0', b''), self.eof) + if text: + self.cookedq.write(text) + if self.data_callback is not None: + self.data_callback(text) def fill_rawq(self): - """Fill raw queue from exactly one recv() system call. + """ + Reads once from the socket into the raw queue. Blocks if no data is + available, and sets eof when the connection was closed. + """ + data = self.sock.recv(_READ_SIZE) + self.msg('Recv: %r', data) + self.eof = not data + self.rawq += data - Block if no data is immediately available. Set self.eof when - connection is closed. + def _read_cooked(self): + text = self.cookedq.getvalue() + self.cookedq.seek(0) + self.cookedq.truncate() + return text + def read_all(self): """ - if self.irawq >= len(self.rawq): - self.rawq = b'' - self.irawq = 0 - # The buffer size should be fairly small so as to avoid quadratic - # behavior in process_rawq() above. - buf = self.sock.recv(64) - self.msg("Recv: %s", repr(buf)) - self.eof = not buf - self.rawq = self.rawq + buf - - def sock_avail(self): - """Test whether data is available on the socket.""" - return select.select([self], [], [], 0) == ([self], [], []) - - def interact(self): - """Interaction function, emulates a very dumb telnet client.""" - if sys.platform == "win32": - self.mt_interact() - return - while True: - rfd, wfd, xfd = select.select([self, sys.stdin], [], []) - if self in rfd: - try: - text = self.read_eager() - except EOFError: - print('*** Connection closed by remote host ***') - break - if text: - self.stdout.write(text) - self.stdout.flush() - if sys.stdin in rfd: - line = sys.stdin.readline() - if not line: - break - self.write(line) - - def mt_interact(self): - """Multithreaded version of interact().""" - import _thread + Reads until the connection is closed. - _thread.start_new_thread(self.listener, ()) - while 1: - line = sys.stdin.readline() - if not line: - break - self.write(line) - - def listener(self): - """Helper for mt_interact() -- this executes in the other thread.""" - while 1: - try: - data = self.read_eager() - except EOFError: - print('*** Connection closed by remote host ***') - return - if data: - self.stdout.write(data) - else: - self.stdout.flush() + :rtype: str + :return: The received text that expect() did not consume. + """ + self.process_rawq() + while not self.eof: + self.fill_rawq() + self.process_rawq() + return self._read_cooked() def _wait_for_data(self, timeout): - end = time.time() + timeout + # Waits in steps of one second; a single select() with the full timeout + # did not always return when data arrived. + end = monotonic() + timeout while True: - readable, writeable, excp = select.select([self.sock], [], [], 1) + readable, _, _ = select.select([self.sock], [], [], 1) if readable: return True - if time.time() > end: + if monotonic() > end: return False def _waitfor(self, relist, timeout=None, flush=False, cleanup=None): - re = None - relist = relist[:] - indices = list(range(len(relist))) + relist = [regex if hasattr(regex, 'search') else re.compile(regex) for regex in relist] search_window_size = 150 - head_loockback_size = 10 - for i in indices: - if not hasattr(relist[i], "search"): - if not re: - import re - relist[i] = re.compile(relist[i]) - self.msg("Expecting: %s" % [l.pattern for l in relist]) + head_lookback_size = 10 + self.msg('Expecting: %s', [regex.pattern for regex in relist]) incomplete_tail = '' clean_sw_size = search_window_size while True: @@ -663,7 +325,7 @@ def _waitfor(self, relist, timeout=None, flush=False, cleanup=None): qlen = self.cookedq.tell() if cleanup: while True: - pos = qlen - clean_sw_size - len(incomplete_tail) - head_loockback_size + pos = qlen - clean_sw_size - len(incomplete_tail) - head_lookback_size self.cookedq.seek(max(0, pos)) search_window = self.cookedq.read() search_window, incomplete_tail = cleanup(search_window) @@ -673,104 +335,67 @@ def _waitfor(self, relist, timeout=None, flush=False, cleanup=None): if search_window_len > search_window_size: clean_sw_size -= search_window_size break - else: - clean_sw_size += search_window_size + clean_sw_size += search_window_size else: - self.cookedq.seek(qlen - search_window_size) + self.cookedq.seek(max(0, qlen - search_window_size)) search_window = self.cookedq.read() - for i in indices: - m = relist[i].search(search_window) - if m is not None: - self.msg("Match found: %s" % repr(m.group())) - e = m.end() - m.start() - e = qlen - e + for index, regex in enumerate(relist): + match = regex.search(search_window) + if match is None: + continue + self.msg('Match found: %r', match.group()) + self.cookedq.seek(0) + text = self.cookedq.read(qlen - (match.end() - match.start())) + if flush: self.cookedq.seek(0) - text = self.cookedq.read(e) - if flush: - self.cookedq.seek(0) - self.cookedq.truncate() - self.cookedq.write(search_window[m.end() :]) - else: - self.cookedq.seek(qlen) - return i, m, text + self.cookedq.truncate() + self.cookedq.write(search_window[match.end() :]) + else: + self.cookedq.seek(qlen) + return index, match, text if self.eof: break - if timeout is not None: - if not self._wait_for_data( - timeout - ): # Workaround for the problem with select() below. - break - # The following will sometimes lock even if data is available - # and I have no idea why. Do NOT reverse this unless you are sure - # that you found the reason. The error is rare, but it does happen. - # r, w, x = select.select([self.sock], [], [], timeout) - # if not r: - # break + if timeout is not None and not self._wait_for_data(timeout): + break self.fill_rawq() - text = self.read_very_lazy() + text = self._read_cooked() if not text and self.eof: - raise EOFError + raise EOFError('telnet connection closed') return -1, None, text def waitfor(self, relist, timeout=None, cleanup=None): - """Read until one from a list of a regular expressions matches. - - The first argument is a list of regular expressions, either - compiled (re.RegexObject instances) or uncompiled (strings). - The optional second argument is a timeout, in seconds; default - is no timeout. - - Return a tuple of three items: the index in the list of the - first regular expression that matches; the match object - returned; and the text read up till and including the match. - - If EOF is read and no text was read, raise EOFError. - Otherwise, when nothing matches, return (-1, None, text) where - text is the text received so far (may be the empty string if a - timeout happened). - - If a regular expression ends with a greedy match (e.g. '.*') - or if more than one expression can match the same input, the - results are undeterministic, and may depend on the I/O timing. + """ + Reads until one of the regular expressions matches the end of the + received text, and keeps the text. + + Returns a tuple of the index of the first regular expression that + matched, the match object, and the text up to the match. When nothing + matched until the timeout, returns (-1, None, text) with the received + text; when expect was cancelled, returns (-2, None, ''). Raises + EOFError if the connection was closed and no text was received. + + :type relist: list(str|re.Pattern) + :param relist: The regular expressions. + :type timeout: float|None + :param timeout: Seconds without received data until the timeout. + :type cleanup: callable|None + :param cleanup: Returns the text to search and an incomplete tail. + :rtype: tuple(int, re.Match|None, str) + :return: The index, the match, and the text. """ return self._waitfor(relist, timeout, flush=False, cleanup=cleanup) def expect(self, relist, timeout=None, cleanup=None): """ - Like waitfor(), but removes the matched data from the incoming - buffer. + Like waitfor(), but removes the text up to the end of the match. + + :type relist: list(str|re.Pattern) + :param relist: The regular expressions. + :type timeout: float|None + :param timeout: Seconds without received data until the timeout. + :type cleanup: callable|None + :param cleanup: Returns the text to search and an incomplete tail. + :rtype: tuple(int, re.Match|None, str) + :return: The index, the match, and the text. """ return self._waitfor(relist, timeout, flush=True, cleanup=cleanup) - - -def test(): - """Test program for telnetlib. - - Usage: python telnetlib.py [-d] ... [host [port]] - - Default host is localhost; default port is 23. - - """ - debuglevel = 0 - while sys.argv[1:] and sys.argv[1] == '-d': - debuglevel = debuglevel + 1 - del sys.argv[1] - host = 'localhost' - if sys.argv[1:]: - host = sys.argv[1] - port = 0 - if sys.argv[2:]: - portstr = sys.argv[2] - try: - port = int(portstr) - except ValueError: - port = socket.getservbyname(portstr, 'tcp') - tn = Telnet() - tn.set_debuglevel(debuglevel) - tn.open(host, port) - tn.interact() - tn.close() - - -if __name__ == '__main__': - test() diff --git a/tests/Exscript/protocols/telnetlibTest.py b/tests/Exscript/protocols/telnetlibTest.py new file mode 100644 index 00000000..eaab54b2 --- /dev/null +++ b/tests/Exscript/protocols/telnetlibTest.py @@ -0,0 +1,150 @@ +import logging +import socket +import unittest + +from hypothesis import given +from hypothesis import strategies as st + +from Exscript.protocols import telnetlib +from Exscript.protocols.telnetlib import DO, DONT, IAC, NAWS, SB, SE, TTYPE, WILL, WONT, Telnet + +ECHO, SGA, BINARY, NOP = 1, 3, 0, 241 + + +class FakeSock: + """Returns the given chunks from recv(), then EOF, and records the sent data.""" + + def __init__(self, chunks): + self.chunks = list(chunks) + self.sent = [] + + def recv(self, size): + return self.chunks.pop(0) if self.chunks else b'' + + def sendall(self, data): + self.sent.append(bytes(data)) + + +def process(chunks, encoding='latin-1'): + """Returns the cooked text, the text passed to the callback, and the sent bytes.""" + tn = Telnet(encoding=encoding, termtype='vt100', termsize=(24, 80)) + tn.sock = FakeSock(chunks) + received = [] + tn.data_callback = received.append + while not tn.eof: + tn.fill_rawq() + tn.process_rawq() + return tn.cookedq.getvalue(), ''.join(received), b''.join(tn.sock.sent) + + +def iac(*codes): + return bytes([IAC, *codes]) + + +segments = st.one_of( + st.binary(max_size=30), + st.sampled_from([iac(IAC), iac(NOP), iac(0), iac(SE), iac(SB, TTYPE, 1) + iac(SE)]), + st.builds(iac, st.sampled_from([DO, DONT, WILL, WONT]), st.integers(0, 255)), + st.builds( + lambda data: iac(SB) + data.replace(b'\xff', b'\xff\xff') + iac(SE), st.binary(max_size=8) + ), +) + + +class telnetlibTest(unittest.TestCase): + CORRELATE = telnetlib + + def testData(self): + stream = b'a\x00b\x11c' + iac(IAC) + b'd' + iac(NOP) + iac(0) + b'e' + text, received, sent = process([stream]) + self.assertEqual(text, 'ab\x11c\xffde') # NUL is removed, XON is kept + self.assertEqual(received, text) + self.assertEqual(sent, b'') + + def testNegotiation(self): + replies = { + iac(DO, TTYPE): iac(WILL, TTYPE), + iac(DO, NAWS): iac(WILL, NAWS) + iac(SB, NAWS, 0, 80, 0, 24) + iac(SE), + iac(DO, SGA): iac(WONT, SGA), + iac(DO, BINARY): iac(WONT, BINARY), + iac(DONT, NAWS): iac(WONT, NAWS), + iac(DONT, BINARY): iac(WONT, BINARY), + iac(WILL, ECHO): iac(DO, ECHO), + iac(WONT, ECHO): iac(DO, ECHO), + iac(WILL, SGA): iac(DONT, SGA), + iac(WONT, BINARY): iac(DONT, BINARY), + iac(SB, TTYPE, 1) + iac(SE): iac(SB, TTYPE, 0) + b'vt100' + iac(SE), + iac(SB, TTYPE, 0) + b'xterm' + iac(SE): b'', + iac(SB, BINARY, 1) + iac(SE): b'', + } + for stream, reply in replies.items(): + self.assertEqual(process([b'x' + stream + b'y']), ('xy', 'xy', reply), stream) + + def testSubnegotiation(self): + # An escaped IAC inside the subnegotiation does not end it. + stream = iac(SB, NAWS, IAC, IAC, SE) + b'data' + iac(SE) + b'x' + self.assertEqual(process([stream])[0], 'x') + # An unterminated subnegotiation ends at the next command. + stream = iac(SB, TTYPE, 1) + iac(DO, TTYPE) + b'x' + self.assertEqual(process([stream]), ('x', 'x', iac(WILL, TTYPE))) + + def testIncompleteCommandAtEof(self): + for stream in iac(), iac(DO), iac(SB, TTYPE, 1), iac(SB, TTYPE, 1, IAC): + self.assertEqual(process([b'x' + stream]), ('x', 'x', b'')) + + def testEveryReadSplit(self): + stream = ( + b'a' + iac(DO, NAWS) + iac(SB, TTYPE, 1) + iac(SE) + iac(IAC) + iac(WILL, ECHO) + b'z' + ) + expected = process([stream]) + for cut in range(1, len(stream)): + self.assertEqual(process([stream[:cut], stream[cut:]]), expected, cut) + + @given(st.lists(segments, max_size=20), st.lists(st.integers(1, 60), max_size=8)) + def testSplitDoesNotMatter(self, parts, sizes): + stream = b''.join(parts) + chunks, pos = [], 0 + for size in sizes: + chunks.append(stream[pos : pos + size]) + pos += size + chunks.append(stream[pos:]) + self.assertEqual(process([chunk for chunk in chunks if chunk]), process([stream])) + + def testMultibyteCharacterSplit(self): + stream = 'Grüße 20 €'.encode() + for cut in range(1, len(stream)): + text, received, _ = process([stream[:cut], stream[cut:]], encoding='utf-8') + self.assertEqual((text, received), ('Grüße 20 €', 'Grüße 20 €'), cut) + + def testWrite(self): + tn = Telnet() + tn.sock = FakeSock([]) + tn.set_debuglevel(1) + with self.assertNoLogs('Exscript.protocols.telnetlib', logging.DEBUG): # may be a password + tn.write('sécret\xff\r') + tn.write(b'\xff') + self.assertEqual(tn.sock.sent, [b's\xe9cret\xff\xff\r', b'\xff\xff']) + + def testSetWindowSize(self): + tn = Telnet() + tn.sock = FakeSock([]) + tn.set_window_size(50, 120) + self.assertEqual(tn.sock.sent, []) # the server did not ask for it yet + tn.can_naws = True + tn.set_window_size(1, 255) + self.assertEqual(tn.sock.sent, [iac(SB, NAWS, 0, IAC, IAC, 0, 1) + iac(SE)]) + + def testOpen(self): + for family, address in (socket.AF_INET, '127.0.0.1'), (socket.AF_INET6, '::1'): + try: + listener = socket.create_server((address, 0), family=family) + except OSError: + continue # no IPv6 on this host + with listener: + tn = Telnet(address, listener.getsockname()[1], connect_timeout=5) + tn.close() + self.assertTrue(tn.eof) + with socket.create_server(('127.0.0.1', 0)) as listener: + port = listener.getsockname()[1] + with self.assertRaisesRegex(OSError, f'=> telnet://127.0.0.1:{port}'): + Telnet('127.0.0.1', port, connect_timeout=5) diff --git a/uv.lock b/uv.lock index d2877ba6..609c4979 100644 --- a/uv.lock +++ b/uv.lock @@ -422,6 +422,7 @@ otel = [ [package.dev-dependencies] dev = [ + { name = "hypothesis" }, { name = "mypy" }, { name = "opentelemetry-instrumentation" }, { name = "opentelemetry-sdk" }, @@ -429,6 +430,7 @@ dev = [ { name = "pytest-cov" }, ] test = [ + { name = "hypothesis" }, { name = "opentelemetry-instrumentation" }, { name = "opentelemetry-sdk" }, { name = "pytest" }, @@ -451,6 +453,7 @@ provides-extras = ["otel"] [package.metadata.requires-dev] dev = [ + { name = "hypothesis", specifier = ">=6.170.14" }, { name = "mypy", specifier = ">=2.4" }, { name = "opentelemetry-instrumentation", specifier = ">=0.48b0" }, { name = "opentelemetry-sdk", specifier = ">=1.27" }, @@ -458,6 +461,7 @@ dev = [ { name = "pytest-cov", specifier = ">=7.1" }, ] test = [ + { name = "hypothesis", specifier = ">=6.170.14" }, { name = "opentelemetry-instrumentation", specifier = ">=0.48b0" }, { name = "opentelemetry-sdk", specifier = ">=1.27" }, { name = "pytest", specifier = ">=9.2" }, @@ -465,6 +469,87 @@ test = [ ] types = [{ name = "mypy", specifier = ">=2.4" }] +[[package]] +name = "hypothesis" +version = "6.170.14" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "sortedcontainers" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d6/0f/10b2076a1314c554eee58c26e1dbe630fd26a32773df2a63bd5dba5d9947/hypothesis-6.170.14.tar.gz", hash = "sha256:8272913eb34c4399a88cf8a9ecdb351c9ef8f621c061f2ac323362d4a66b755d", size = 521490, upload-time = "2026-10-30T06:51:55.943Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c9/d6/eae1191f3398b8cc9f10b20c9b93dd1ceae037565e94ababf521730e59a4/hypothesis-6.170.14-cp311-abi3-macosx_10_12_x86_64.whl", hash = "sha256:b1c4990f9fafad83cdac33bc564d41066f5824a92ddd45ded7e0e84277a3ec0e", size = 800123, upload-time = "2026-10-30T06:50:33.682Z" }, + { url = "https://files.pythonhosted.org/packages/56/78/434915150cecd5b9ba41837106a3da2782f515a9aed583d25860bfa6d71c/hypothesis-6.170.14-cp311-abi3-macosx_11_0_arm64.whl", hash = "sha256:ab414b45a56853fcdd6b602d9c5033e3c7d127ee8b8e7e3397223b746440b37c", size = 795278, upload-time = "2026-10-30T06:50:22.237Z" }, + { url = "https://files.pythonhosted.org/packages/95/6b/07f5e5e667b4b3e90bea16b4db09367b5bb1dfae63eb75a7152b053b0c50/hypothesis-6.170.14-cp311-abi3-manylinux1_i686.manylinux_2_5_i686.whl", hash = "sha256:cceb1b3c19ea0a146c7bb2fb2b44e3ef231bcecd42c97d95154e5d233e209572", size = 1162031, upload-time = "2026-10-30T06:50:08.828Z" }, + { url = "https://files.pythonhosted.org/packages/22/ba/51e0dc10befcc4ca9b7fb81b76ca6aef4ffc9c7a741191a8b41e878ee2f2/hypothesis-6.170.14-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:376dbf5ffa1e3ad71af2a1af0877176faabe3e63d345324a33c82e408dbfd604", size = 1124181, upload-time = "2026-10-30T06:51:51.244Z" }, + { url = "https://files.pythonhosted.org/packages/ee/37/77834f102b0bca702f723dd519b8f590e83ec6392e59565384cfdc5bbec7/hypothesis-6.170.14-cp311-abi3-manylinux2014_armv7l.manylinux_2_17_armv7l.whl", hash = "sha256:5f0f6f6666fa15ec13ca1d8430f70660fce8b3f4f133e25fdf674be6ad77295e", size = 1154664, upload-time = "2026-10-30T06:51:50.027Z" }, + { url = "https://files.pythonhosted.org/packages/12/76/83edc11f33319a2febd025fb504aa1ed8e245b7e25905da80513615e4b5e/hypothesis-6.170.14-cp311-abi3-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:c4ff70fbb94f54cd35045a40e5a77dd8f876d4cc6a39dcdc4ac1af6f8f5cc6fe", size = 1148576, upload-time = "2026-10-30T06:50:18.505Z" }, + { url = "https://files.pythonhosted.org/packages/ce/35/d4086d546c0abe406f0ece17e33eca7051e87dc7501485a696385aed34fd/hypothesis-6.170.14-cp311-abi3-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:8ecec6be3a02e22c70bc7dfb15afeeeb708aeccb994c9d57db438199cad9bfbb", size = 1201336, upload-time = "2026-10-30T06:50:17.221Z" }, + { url = "https://files.pythonhosted.org/packages/dd/85/117b0dde708240854ce0f1ad95d43bc132f59e6bffc82c2a20e76b984020/hypothesis-6.170.14-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:3bbf06c4f8e3aab37c144fe3bc1d89a051a2dfbefef8c2a25160b11da854f949", size = 1167592, upload-time = "2026-10-30T06:50:23.393Z" }, + { url = "https://files.pythonhosted.org/packages/29/bb/105ee91bf92e3b7a5bc388827e774b7c5744f39ce36f50f94918b584813c/hypothesis-6.170.14-cp311-abi3-manylinux_2_31_riscv64.whl", hash = "sha256:07869d66bf1e92e39447808fcc1977bfd10bfdff4970d6969a49e89a84cf04a7", size = 1123748, upload-time = "2026-10-30T06:51:07.034Z" }, + { url = "https://files.pythonhosted.org/packages/cd/9b/c60e868ba0f93ff5bc7cec3e413c5e824863bbd9bd45dcc86bc18adb4266/hypothesis-6.170.14-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:0ce9ddf0d0e99c532c2cf00d97b63316830810d18f11c80c7bcf0f771969f3a4", size = 1299121, upload-time = "2026-10-30T06:51:00.068Z" }, + { url = "https://files.pythonhosted.org/packages/e9/f6/a2b9630ddc70aa7642931ac6c353fc57df38c6de655718a50337b5b7d4e0/hypothesis-6.170.14-cp311-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:420b4a77157e94f124498fff639284d514b60926f61925dd7aafb75411e7ce92", size = 1427779, upload-time = "2026-10-30T06:50:15.652Z" }, + { url = "https://files.pythonhosted.org/packages/db/36/b3e7d3fef4ce3f44f70a61399a3549c02a71e430122f688880a24336e9a6/hypothesis-6.170.14-cp311-abi3-musllinux_1_2_i686.whl", hash = "sha256:2a8ed08715a9a27d3a4730f48c99133ef25b3561d8fe992d1eda91e22cf6be99", size = 1381908, upload-time = "2026-10-30T06:50:58.685Z" }, + { url = "https://files.pythonhosted.org/packages/ad/52/b044ccaf1cb9d89f319b737f1fee7cc83e16c2fd086e1bf1ce149092eb44/hypothesis-6.170.14-cp311-abi3-musllinux_1_2_ppc64le.whl", hash = "sha256:71081b46714e95e9a10d922bedd08d5a6e49835bbc2ef37a81d30d62f8596829", size = 1277802, upload-time = "2026-10-30T06:51:53.644Z" }, + { url = "https://files.pythonhosted.org/packages/27/2f/989ef755683c7cefd195670861a2fbd72b6b7952c124256152e94660e62c/hypothesis-6.170.14-cp311-abi3-musllinux_1_2_riscv64.whl", hash = "sha256:ce563cb34dedc9163811b03ce05a955eededd6f429db7c80c5c31d5836a9c05a", size = 1293571, upload-time = "2026-10-30T06:50:03.076Z" }, + { url = "https://files.pythonhosted.org/packages/a5/00/7426641c28dfc52e0a0d32748a1e47d3c189f9cf7eef81c0ecb7e90eb4cc/hypothesis-6.170.14-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:06a752e3e879f1b69dda9ea203d4e8e1562cf72f694513f01c8c99d2e5e3f3cc", size = 1333329, upload-time = "2026-10-30T06:50:50.362Z" }, + { url = "https://files.pythonhosted.org/packages/7b/55/88307f535bb0914824b0eb2c6b58aa4a482b483c8d25d8022e7b4f641f8c/hypothesis-6.170.14-cp311-abi3-win32.whl", hash = "sha256:0f418c6bff7cd8a117dd0f526c20f100c32df1ff9f7bc8e01a5eeb3855278039", size = 687305, upload-time = "2026-10-30T06:50:43.079Z" }, + { url = "https://files.pythonhosted.org/packages/25/61/ba5bb3e8593afa9212fad51dc70f219ef6da7527d806c62a42c1cf12247b/hypothesis-6.170.14-cp311-abi3-win_amd64.whl", hash = "sha256:1e9f7619dff3072685e4f36e8ae7b134a0c1ca41074fc54ade81e5355c72cd8c", size = 693378, upload-time = "2026-10-30T06:51:36.729Z" }, + { url = "https://files.pythonhosted.org/packages/ab/72/482a68ae0e94b24acd964196cca830d1dd65efbf3d4821599d1c9527f8e2/hypothesis-6.170.14-cp311-abi3-win_arm64.whl", hash = "sha256:7dbaec9a62b7792b0b04b7ebc933aeafd3ff019b81d67d763c36ebcf4d56f864", size = 691696, upload-time = "2026-10-30T06:50:32.53Z" }, + { url = "https://files.pythonhosted.org/packages/8e/41/aa2c8ffc1490f02ec2b0e283e0856a0822c2b0707a83831e286493f888f5/hypothesis-6.170.14-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:76b03ded37a7eb14d01b03edca02b9a6ce6b8ab0ad8d2f9ac5f2e9de6769a33a", size = 801893, upload-time = "2026-10-30T06:51:10.965Z" }, + { url = "https://files.pythonhosted.org/packages/81/45/a82212e5773214d3ad1e0cee626098c1068adc93cdb37449e3c0629aa591/hypothesis-6.170.14-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a6055172afdca41cda869652bd2f2c24ec984005cd16bbec5a8f38483f23b9bc", size = 793892, upload-time = "2026-10-30T06:50:55.958Z" }, + { url = "https://files.pythonhosted.org/packages/8a/cc/1889082d37e946e006845163a91e4091d1cff4f1692e3580729454cb02f4/hypothesis-6.170.14-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b6490c64811a2176a5af9fe5a5d59ab2aa20d35d691328cbb336f6fb398355c4", size = 1123577, upload-time = "2026-10-30T06:50:11.524Z" }, + { url = "https://files.pythonhosted.org/packages/7f/7a/311a4ec14dbde1ce82cbd4d8957665c4d68e9139524b10e0a622db98bc4e/hypothesis-6.170.14-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:cde7e98694a9baa1d9761c1b595b3d617467c876200c0798a40f2016a0e85304", size = 1167206, upload-time = "2026-10-30T06:50:05.235Z" }, + { url = "https://files.pythonhosted.org/packages/f5/0d/1afb5e9f470898cf296f10545cfbf76e6b8d4d7025e0c41a491bbe969a22/hypothesis-6.170.14-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:7e39d46bdebb3853fb3000aab3c354e4d780705dc8b2674773ffcdfd04fd5122", size = 1298455, upload-time = "2026-10-30T06:51:30.733Z" }, + { url = "https://files.pythonhosted.org/packages/f5/f6/ff7e9568f7a7d9ec375c68636c66e1b1a3baa06f44652535ff434110f6cd/hypothesis-6.170.14-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:1309925a71a2baacb85ed5c20291ddb4b2a4665bd33fb594697463d457374a1b", size = 1332876, upload-time = "2026-10-30T06:50:57.14Z" }, + { url = "https://files.pythonhosted.org/packages/cc/73/fd330eec15026dc30d0ec8c658517c99703774098305632efb3e523b2f38/hypothesis-6.170.14-cp312-cp312-win_amd64.whl", hash = "sha256:ac98e833bdefbcbebb9094180a7431a8946f1acc91a5b779c814aecca71c7345", size = 691003, upload-time = "2026-10-30T06:50:36.624Z" }, + { url = "https://files.pythonhosted.org/packages/88/95/9729cc49645d07211ce8158f0bb51860873b3b99fb1650c2dad78bd2e86c/hypothesis-6.170.14-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:7d5e5affa9e69b8673f15bef86421740945e74f3c31d0817141ba020c7e00705", size = 801888, upload-time = "2026-10-30T06:50:06.456Z" }, + { url = "https://files.pythonhosted.org/packages/6e/ac/687277e019d68b96d2c33b979ad8a955b5747e279891f8c335bbf58cd9bc/hypothesis-6.170.14-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e7ae9ff3220659c4721acb58d34e72227a0af7747fc903ae177dfdb0323796b4", size = 793879, upload-time = "2026-10-30T06:51:09.51Z" }, + { url = "https://files.pythonhosted.org/packages/b2/1a/b46a2959b41ff9409af8b43fd63b07c6ed51958db0e9d314db9ee6012f5a/hypothesis-6.170.14-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:857932ff6bba183af0236f659b042023ab8b4ad58725b9024bcaba941c8cd5d0", size = 1123483, upload-time = "2026-10-30T06:50:47.998Z" }, + { url = "https://files.pythonhosted.org/packages/cb/2a/6735b2edff23d045441b5917f35c01a4c139a186604847ca19bbd63e81e2/hypothesis-6.170.14-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8126e8208acadfc3c92d0aedb8011cf724fb59e5523ceaca3f4612dcc409e632", size = 1167158, upload-time = "2026-10-30T06:51:26.528Z" }, + { url = "https://files.pythonhosted.org/packages/ea/f0/b5b4273568620080db24dda2008236bd6c64599ab03210e46977b44b54d5/hypothesis-6.170.14-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d28ab5c5b08b1de11bc022d576b20f9f585f7e7758778295461f80b9222d7a18", size = 1298440, upload-time = "2026-10-30T06:50:10.195Z" }, + { url = "https://files.pythonhosted.org/packages/2a/c7/38427a9803dac9bdd06581061a84672107d821f6d91b833dec467c37f5d5/hypothesis-6.170.14-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:033aca79fd3f9a6e15402dfddbc988141d43eae38d26a5fc554f6879f7e0e5f9", size = 1332909, upload-time = "2026-10-30T06:51:17.244Z" }, + { url = "https://files.pythonhosted.org/packages/7a/4f/47c060c96d077750c8a7b83b2e50b1add2050b391e77f49126828c125264/hypothesis-6.170.14-cp313-cp313-win_amd64.whl", hash = "sha256:03831d833b161b9b7a32f0b607205293954f3a651196c80138804b9479845d71", size = 690983, upload-time = "2026-10-30T06:51:04.201Z" }, + { url = "https://files.pythonhosted.org/packages/e8/3a/7ce36c91aafb192e08d990b3a8fd632be113eeb43973a94fddaa4172c570/hypothesis-6.170.14-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:a2410c5a6dcc011338e53cd381bd32d979b6f06d799c5cf01d76b923f0a0670b", size = 802098, upload-time = "2026-10-30T06:51:48.445Z" }, + { url = "https://files.pythonhosted.org/packages/c7/dc/17bc4ba00a48976944918bb5d7207519ed206fdf98af0d0319e5ba33afd2/hypothesis-6.170.14-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:dcf6303c65e73acf134b7854cbdfec8f2108cfdc84806fa180de04d7946aab0c", size = 793921, upload-time = "2026-10-30T06:50:07.623Z" }, + { url = "https://files.pythonhosted.org/packages/1c/c3/252db9362dab21e3c3351abb89e715fa1ef598ebaf30abb9c9b1dd3b3aab/hypothesis-6.170.14-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:67d17e8b91d3913e7aa2b1179957df4e0634caea3868b5ba4fe2bf1945334134", size = 1123996, upload-time = "2026-10-30T06:51:05.628Z" }, + { url = "https://files.pythonhosted.org/packages/f8/41/573468979e9579eb6df937e6a96c1a944dfe2765c73624156571e3cc4b34/hypothesis-6.170.14-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:96f9f8a684c88ad2280129b28bc609adb6b31cd63b1b83b397b86d552391e947", size = 1167330, upload-time = "2026-10-30T06:51:23.741Z" }, + { url = "https://files.pythonhosted.org/packages/f6/1c/d1b790ec3247966ceb3d5729bb79f810e24b1684b49b6290c9ad5be05e33/hypothesis-6.170.14-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c42d504d9bc0aef71fa88877c68ce202c02242c84f05791be2785e01e3a8699", size = 1298940, upload-time = "2026-10-30T06:51:27.68Z" }, + { url = "https://files.pythonhosted.org/packages/f6/a3/2fb6f7e825c9d943c5c35e0d9ef968e416267db2ef710e6d2d3cf7f36ab0/hypothesis-6.170.14-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ad84ca98f38e9f385b0f616df04bea4a9c6849333e221ce9289e9566ccd41058", size = 1333102, upload-time = "2026-10-30T06:51:20.993Z" }, + { url = "https://files.pythonhosted.org/packages/49/09/b533e4ac06dc317314d574dd0e3b548bac9cffe05e7e7a32f7b25aa7f052/hypothesis-6.170.14-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:f569bd9745835f9864d53c853e401c2021e796fdbb385497fdae633fe950b6ba", size = 633715, upload-time = "2026-10-30T06:50:25.715Z" }, + { url = "https://files.pythonhosted.org/packages/08/c6/9c47f183e5a8a4059d7c1c39e85f7b140f4416961b5ed0dac5d35dc6e1f7/hypothesis-6.170.14-cp314-cp314-win_amd64.whl", hash = "sha256:31c6a4015c02c45ba0285ff48538f4eb2c2de7e836f313bb8832fd8d86f71d62", size = 690702, upload-time = "2026-10-30T06:50:41.998Z" }, + { url = "https://files.pythonhosted.org/packages/05/47/623b189cc8d9e360bfbb69828fb88ca4d0d4996b3975f3191b49a50f04ce/hypothesis-6.170.14-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:744c587791b2c5e38d1da2eaca617b1a82073832dfcf42975ddb03e93f187919", size = 800266, upload-time = "2026-10-30T06:50:29.462Z" }, + { url = "https://files.pythonhosted.org/packages/ea/af/7ec2697869df2c7f5d1c414b1a0a21746aa0298d41d7358e45fb7ed83992/hypothesis-6.170.14-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:d0daded6c2f3e05f6bd989059bb798e570722802e05eb3543c6f5d7a1e7e6500", size = 792500, upload-time = "2026-10-30T06:50:51.878Z" }, + { url = "https://files.pythonhosted.org/packages/e0/cf/2c7b13b5e56cd42789dbef4948f21e685c4a4db7dd6daffa253f261a9193/hypothesis-6.170.14-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:986eaec8ca5a22710b1086b88734035c7d7ffcdb4ce02341b9f7069cc1bdfc5e", size = 1122469, upload-time = "2026-10-30T06:51:31.981Z" }, + { url = "https://files.pythonhosted.org/packages/63/17/aab31d83a3195edf21ff0b05ea8fabf1fcf723517a6ba7bd1aee3b2713e3/hypothesis-6.170.14-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:44c481115770bb3df5014b0369ea1aaf7319b81b3346a744a9265fc933190c3d", size = 1166369, upload-time = "2026-10-30T06:50:54.572Z" }, + { url = "https://files.pythonhosted.org/packages/8b/ce/88ab370d5c1e06c74b87995381c28f5c1456bcd832db339bb93b6b15cae5/hypothesis-6.170.14-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:261ae20a5573cf59e5dbf931c7fb9abcb012695adccb61fdab108cbba1772ee5", size = 1297198, upload-time = "2026-10-30T06:51:22.283Z" }, + { url = "https://files.pythonhosted.org/packages/06/03/5c8d82f48d3a7adb593d1c141fdae597ef874e27dabd94ede9edd5c36c29/hypothesis-6.170.14-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:833f7e6d93c03f3c00583a126e61f631d6b91a84e157a30da2d1c786ce5658b7", size = 1332114, upload-time = "2026-10-30T06:51:34.61Z" }, + { url = "https://files.pythonhosted.org/packages/25/24/b08d06042af15e1a5fd2fe378feaa2336983f4ecc2bf645b38c6accf7d7d/hypothesis-6.170.14-cp314-cp314t-win_amd64.whl", hash = "sha256:224cad0e7ba8bfc6ef708652a3cc2148161e3157d191eb77ecc85dc5d16afb70", size = 690815, upload-time = "2026-10-30T06:51:43.881Z" }, + { url = "https://files.pythonhosted.org/packages/ee/72/bbd7a5341e23b56759937cef4fe83ea9c907f15ffccee1c2e8e2c8937546/hypothesis-6.170.14-cp315-abi3.abi3t-macosx_10_12_x86_64.whl", hash = "sha256:e76877e7c7a5cf8c4fbf98280dc6582f5d6450a73494490e8b48b7d03fb5e12f", size = 799866, upload-time = "2026-10-30T06:50:12.929Z" }, + { url = "https://files.pythonhosted.org/packages/df/f0/be3b05a4c57bbb91b0048c37a5f3a278accb1ec51e853bb87037379c3f93/hypothesis-6.170.14-cp315-abi3.abi3t-macosx_11_0_arm64.whl", hash = "sha256:5a613867bd2675101dda8fda380c3ce6be47842bed5640e1ba7b6b6fd0ffe8a3", size = 792315, upload-time = "2026-10-30T06:50:37.847Z" }, + { url = "https://files.pythonhosted.org/packages/aa/2b/f0a1509d5d4ad2e3b3c834bebd8a4b1f430d01d350710f45338342756b91/hypothesis-6.170.14-cp315-abi3.abi3t-manylinux1_i686.manylinux_2_5_i686.whl", hash = "sha256:a2249f2428ea94c47bccceb9fb8beada242c577dd4b3cd2edef5fb7160d1b71b", size = 1158319, upload-time = "2026-10-30T06:50:46.745Z" }, + { url = "https://files.pythonhosted.org/packages/6c/5c/eea759905d4b7d4d9a750b9b61a419120742143288c48c7d0ba633efc5f1/hypothesis-6.170.14-cp315-abi3.abi3t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:20df34cca1265c1683ccdcab5872cf0a023f40235a3092a96d4bd81e4fc57986", size = 1121221, upload-time = "2026-10-30T06:50:28.317Z" }, + { url = "https://files.pythonhosted.org/packages/93/b0/3c90a9e734aa0a4aef0fa9004dd7c2bcd27a6e66bc32a8690647639cba88/hypothesis-6.170.14-cp315-abi3.abi3t-manylinux2014_armv7l.manylinux_2_17_armv7l.whl", hash = "sha256:4a7f8023efbec61296fd20edb35d638ec89ff761d3f9d3dc8128ac8f204652ec", size = 1152108, upload-time = "2026-10-30T06:50:35.02Z" }, + { url = "https://files.pythonhosted.org/packages/63/fd/4b7e7f69ee248547f6fe69475197ffbb5ae9563ab49c1430f020bd1d5493/hypothesis-6.170.14-cp315-abi3.abi3t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:3faa2b45f30320fb57ce1c30f1bf9f08b1fd1cc1c606ae2a3e1efd0256fd7c77", size = 1145869, upload-time = "2026-10-30T06:51:54.841Z" }, + { url = "https://files.pythonhosted.org/packages/15/de/dc96bc0646e8543386e3c539fea154075b2d0f40e6b3fb19022c43058dac/hypothesis-6.170.14-cp315-abi3.abi3t-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:5be7a2239adde6c72001eb42cdb924d3c3856b1ee80342049961efdd6c72d064", size = 1199697, upload-time = "2026-10-30T06:51:40.612Z" }, + { url = "https://files.pythonhosted.org/packages/f5/fd/363472210e3ed2a356b7bc6ca8ca405c0dae6531ad43c8f76ca20a946884/hypothesis-6.170.14-cp315-abi3.abi3t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1a7aa883cd30c6c0c7259f34c1e3f37617c0d52fa6718e93c9926cc49b8047d6", size = 1165533, upload-time = "2026-10-30T06:50:14.401Z" }, + { url = "https://files.pythonhosted.org/packages/37/3f/989ab6da164efe574399debdf64370d153f17d4cfda51c8d7e14781dc92b/hypothesis-6.170.14-cp315-abi3.abi3t-manylinux_2_31_riscv64.whl", hash = "sha256:79360d1dd671e602d2327e26fb9179645511adcfb9ec9d28c5d6bda313920aa2", size = 1119844, upload-time = "2026-10-30T06:51:08.317Z" }, + { url = "https://files.pythonhosted.org/packages/5d/c4/7758868c29d3fbcdd0face967a2d7e3af221a1c8d57802c7ac6cb49becd9/hypothesis-6.170.14-cp315-abi3.abi3t-musllinux_1_2_aarch64.whl", hash = "sha256:e4c45885e931a55c5097ebf0a9b65459878ad5e1c69c0448ae37e03fd764ddfc", size = 1296008, upload-time = "2026-10-30T06:50:40.598Z" }, + { url = "https://files.pythonhosted.org/packages/c7/b2/dc88869b6f8508fbfb290c4a9b03328142743d041ae4a144870b904b69bd/hypothesis-6.170.14-cp315-abi3.abi3t-musllinux_1_2_armv7l.whl", hash = "sha256:a6d09601401c019366826effa98ae64393dd149b50d7d6ea202ad472784278a8", size = 1425148, upload-time = "2026-10-30T06:51:18.492Z" }, + { url = "https://files.pythonhosted.org/packages/f2/f2/f6dde97e8a1d017d9663164f48f600f65e9d2ade37c43447421f6a369cbe/hypothesis-6.170.14-cp315-abi3.abi3t-musllinux_1_2_i686.whl", hash = "sha256:05e2a645b94fd43107a3360fe0aaf6db268199c80a43cf375fac5f1f9047745f", size = 1378730, upload-time = "2026-10-30T06:51:16.063Z" }, + { url = "https://files.pythonhosted.org/packages/9b/42/e2aa93586527428b51c019a4cedf28e1d2f10dfec5f9acfa3e1a22cd2e86/hypothesis-6.170.14-cp315-abi3.abi3t-musllinux_1_2_ppc64le.whl", hash = "sha256:cb54946f70ae5008ea8cd58ce9a27b2126506732b53b83a6d117371d7c755362", size = 1274346, upload-time = "2026-10-30T06:51:14.2Z" }, + { url = "https://files.pythonhosted.org/packages/c2/22/17ee8d38c5cb0a948ba9bb6cd7be2351a021cf1ea6686aa4fac16c29c45a/hypothesis-6.170.14-cp315-abi3.abi3t-musllinux_1_2_riscv64.whl", hash = "sha256:b72805cf56012a3d96f4a1fbb869f178d448f242b4f64310956134a3c952113f", size = 1291604, upload-time = "2026-10-30T06:50:53.304Z" }, + { url = "https://files.pythonhosted.org/packages/0c/c8/c97cdaa1191519fadcc06bc118f271910d7d7972caeecb70b7003a62303c/hypothesis-6.170.14-cp315-abi3.abi3t-musllinux_1_2_x86_64.whl", hash = "sha256:a17500680ec2c3cf006c12d3832a6c8daba8a18386f93b3d1828de07b07910aa", size = 1331507, upload-time = "2026-10-30T06:51:02.784Z" }, + { url = "https://files.pythonhosted.org/packages/cf/c7/1007bdaf517b086df3c54d06de6c47f099704fd8bd6c2b5013e749833ee4/hypothesis-6.170.14-cp315-abi3.abi3t-win32.whl", hash = "sha256:b1d9baaef909100fd1a6ba5457ecffa4a184e820c7f0bd3d4589f14870f5cade", size = 684627, upload-time = "2026-10-30T06:50:49.194Z" }, + { url = "https://files.pythonhosted.org/packages/51/d1/68539ecbe240de4afc03c5fa120ecf0875690708f2a97914352ae3172d36/hypothesis-6.170.14-cp315-abi3.abi3t-win_amd64.whl", hash = "sha256:a725b854c5df088c5a579f1c323657258abe5d8e32e16cd07beb48960f31c070", size = 690617, upload-time = "2026-10-30T06:51:42.268Z" }, + { url = "https://files.pythonhosted.org/packages/66/d7/9671c708cdab1c49ff96344f6730ccd76a7e1c2bdbbb2bed93bc9d6a18b4/hypothesis-6.170.14-cp315-abi3.abi3t-win_arm64.whl", hash = "sha256:75f60ef38c010742339a9b2ec2a44cb0aed6c6c8f1455ad6a6de48f5216a72b2", size = 688595, upload-time = "2026-10-30T06:51:37.945Z" }, + { url = "https://files.pythonhosted.org/packages/ba/ae/822ac5f03f0f13e96b4a7b380ede8fa4ae2a46730f4e5c6d7eca016147bb/hypothesis-6.170.14-pp312-pypy312_pp80-macosx_10_12_x86_64.whl", hash = "sha256:c7b5dee8b87f8a78c1786eb1f96a6c0c91fcc2e10b2e9d876dcbc8e3f65dffc3", size = 801000, upload-time = "2026-10-30T06:50:44.17Z" }, + { url = "https://files.pythonhosted.org/packages/fa/b2/e4a0901adb647ca375a794cbf1a23e5e49e59c77d191d1922e3d3c4ce6b6/hypothesis-6.170.14-pp312-pypy312_pp80-macosx_11_0_arm64.whl", hash = "sha256:e99d1bc7d8e8fc17cb67a38e003cbf4a17f41f7851c1f205efd35b6f3ba93da5", size = 792896, upload-time = "2026-10-30T06:51:25.028Z" }, + { url = "https://files.pythonhosted.org/packages/d1/9c/ac50eed9becb0de590d8af7bd1b67546e58b84dda4abb89ca563eed1babe/hypothesis-6.170.14-pp312-pypy312_pp80-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d66dab3b79eda78964665ab4f9d7a269137c14cd6ec7ea4fb27002a8786452b9", size = 1122853, upload-time = "2026-10-30T06:51:33.262Z" }, + { url = "https://files.pythonhosted.org/packages/c4/7f/bc41905c23c74c3db4620a5382977b168156f357e8a421c2b9f66f764979/hypothesis-6.170.14-pp312-pypy312_pp80-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:88d829c29907e7d86f8b1d33041bb73814062a3681ec51dcb231f64874f88b14", size = 1167065, upload-time = "2026-10-30T06:51:45.442Z" }, + { url = "https://files.pythonhosted.org/packages/0f/17/b28b4348a95694562357dc0c8ecb521dd607051dddd7af158c754b895710/hypothesis-6.170.14-pp312-pypy312_pp80-win_amd64.whl", hash = "sha256:90d4e85f22c3154a59df851bb2c59c89c129fba3155125d98ebad80bf69aad26", size = 692119, upload-time = "2026-10-30T06:51:19.8Z" }, +] + [[package]] name = "iniconfig" version = "2.3.1" @@ -984,6 +1069,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9d/7a/d968e294073affff457b041c2be9868a40c1c71f4a35fcc1e45e5493067b/pytest_cov-7.1.0-py3-none-any.whl", hash = "sha256:a0461110b7865f9a271aa1b51e516c9a95de9d696734a2f71e3e78f46e1d4678", size = 22876, upload-time = "2026-03-21T20:11:14.438Z" }, ] +[[package]] +name = "sortedcontainers" +version = "2.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e8/c4/ba2f8066cceb6f23394729afe52f3bf7adec04bf9ed2c820b39e19299111/sortedcontainers-2.4.0.tar.gz", hash = "sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88", size = 30594, upload-time = "2021-05-16T22:03:42.897Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/32/46/9cb0e58b2deb7f82b84065f37f3bffeb12413f947f9388e4cac22c4621ce/sortedcontainers-2.4.0-py2.py3-none-any.whl", hash = "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0", size = 29575, upload-time = "2021-05-16T22:03:41.177Z" }, +] + [[package]] name = "typing-extensions" version = "4.16.0" From cc0203789024bb6b4b351c9e7a2441abd23cb64a Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 3 Nov 2026 02:18:29 +0100 Subject: [PATCH 023/250] Make util.syslog.netlog() work, also on Windows netlog() always raised a TypeError on Python 3, and the module imported syslog, which Windows lacks, only for two constants. It now sends the message as UTF-8 and resolves the address with getaddrinfo(), so IPv6 works, too. --- src/Exscript/util/syslog.py | 26 ++++++++++++++++---------- tests/Exscript/util/syslogTest.py | 30 ++++++++++++++++++++---------- 2 files changed, 36 insertions(+), 20 deletions(-) diff --git a/src/Exscript/util/syslog.py b/src/Exscript/util/syslog.py index 26947e13..e7698143 100644 --- a/src/Exscript/util/syslog.py +++ b/src/Exscript/util/syslog.py @@ -27,7 +27,10 @@ import os import socket import sys -import syslog as _syslog + +# The values of syslog.LOG_DEBUG and syslog.LOG_USER, which exist only on Unix. +LOG_DEBUG = 7 +LOG_USER = 1 << 3 def netlog( @@ -35,10 +38,11 @@ def netlog( source=None, host='localhost', port=514, - priority=_syslog.LOG_DEBUG, - facility=_syslog.LOG_USER, + priority=LOG_DEBUG, + facility=LOG_USER, ): """ + Sends a message to a syslog server over UDP, like the logger -n command. Python's built in syslog module does not support networking, so this is the alternative. The source argument specifies the message source that is @@ -47,20 +51,22 @@ def netlog( The priority and facility arguments are equivalent to those of Python's built in syslog module. + :type message: str + :param message: The message. :type source: str :param source: The source address. :type host: str :param host: The IP address or hostname of the receiving server. - :type port: str - :param port: The TCP port number of the receiving server. + :type port: int + :param port: The UDP port number of the receiving server. :type priority: int :param priority: The message priority. :type facility: int :param facility: The message facility. """ if not source: - source = '%s[%s]' + (sys.argv[0], os.getpid()) - data = '<%d>%s: %s' % (priority + facility, source, message) - sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - sock.sendto(data, (host, port)) - sock.close() + source = '%s[%s]' % (sys.argv[0], os.getpid()) + data = ('<%d>%s: %s' % (priority + facility, source, message)).encode('utf-8') + family, type_, proto, _, address = socket.getaddrinfo(host, port, type=socket.SOCK_DGRAM)[0] + with socket.socket(family, type_, proto) as sock: + sock.sendto(data, address) diff --git a/tests/Exscript/util/syslogTest.py b/tests/Exscript/util/syslogTest.py index 00e8c25a..b8cf57e7 100644 --- a/tests/Exscript/util/syslogTest.py +++ b/tests/Exscript/util/syslogTest.py @@ -1,20 +1,30 @@ +import os +import socket +import sys import unittest import Exscript.util.syslog +from Exscript.util.syslog import netlog -class castTest(unittest.TestCase): +class syslogTest(unittest.TestCase): CORRELATE = Exscript.util.syslog - def testNetlog(self): - pass - # NOTE: No idea how to test netlog. - # How to open a local syslog server? Mocking? - + def setUp(self): + self.server = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + self.server.bind(('127.0.0.1', 0)) + self.server.settimeout(5) + self.addCleanup(self.server.close) + self.port = self.server.getsockname()[1] -def suite(): - return unittest.TestLoader().loadTestsFromTestCase(castTest) + def testNetlog(self): + netlog('hellö', source='router1', host='127.0.0.1', port=self.port) + self.assertEqual(self.server.recv(1024), '<15>router1: hellö'.encode()) # user.debug + netlog('x', source='s', host='127.0.0.1', port=self.port, priority=3, facility=4 << 3) + self.assertEqual(self.server.recv(1024), b'<35>s: x') # auth.err -if __name__ == '__main__': - unittest.TextTestRunner(verbosity=2).run(suite()) + def testDefaultSource(self): + netlog('x', host='127.0.0.1', port=self.port) + expected = '<15>%s[%d]: x' % (sys.argv[0], os.getpid()) + self.assertEqual(self.server.recv(1024), expected.encode()) From d6d3c1ac1052085988927669af761eb6b3dd483a Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 3 Nov 2026 02:18:29 +0100 Subject: [PATCH 024/250] Accept Windows paths in pseudo:// URLs Url.from_string() split the file name at the colon of the drive letter and failed to parse the rest as a port. --- CHANGES.md | 8 ++++++++ src/Exscript/util/url.py | 5 +++-- tests/Exscript/util/urlTest.py | 2 ++ 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 60584e97..9b4059f0 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -194,6 +194,14 @@ The names below refer to functions callable from templates, not protocol methods | - | `string.strip(strings, ...)` | Removes leading/trailing whitespace or specified characters. | | `sys.exec()` | `sys.exec()` | **BREAKING:** Local command output is now text rather than bytes, decoded using Python's default subprocess encoding; line endings are retained. | +## Utilities + +- **`Exscript.util.syslog.netlog()`:** Works again; on Python 3 it always raised a + `TypeError`. It also works over IPv6, and the module can be imported on Windows, + which has no `syslog` module. +- **Pseudo Protocol on Windows:** `pseudo://` URLs accept absolute Windows paths such + as `pseudo://C:\devices\router.py`; the drive letter was taken for a port number. + ## Multi-Threading ## Demos diff --git a/src/Exscript/util/url.py b/src/Exscript/util/url.py index 2cf8038e..d822e948 100644 --- a/src/Exscript/util/url.py +++ b/src/Exscript/util/url.py @@ -221,10 +221,11 @@ def from_string(url, default_protocol='ssh'): except IndexError: pass - # Parse hostname and port number. + # Parse hostname and port number. The address of the pseudo protocol is + # a file name, which may start with a drive letter (C:). result.hostname = netloc + parsed.path result.port = _WELL_KNOWN_PORTS.get(result.protocol) - if ':' in netloc: + if ':' in netloc and result.protocol != 'pseudo': result.hostname, port = netloc.split(':') result.port = int(port) diff --git a/tests/Exscript/util/urlTest.py b/tests/Exscript/util/urlTest.py index 9ff894ae..3153fd0b 100644 --- a/tests/Exscript/util/urlTest.py +++ b/tests/Exscript/util/urlTest.py @@ -59,6 +59,8 @@ ('pseudo://filename', 'pseudo://filename'), ('pseudo:///abspath', 'pseudo:///abspath'), ('pseudo:///abs/path', 'pseudo:///abs/path'), + ('pseudo://C:\\abs\\path', 'pseudo://C:\\abs\\path'), + ('pseudo://C:/abs/path', 'pseudo://C:/abs/path'), ] From 40a3ca822eaca8abe30b80dd51ee36726a640177 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 3 Nov 2026 02:18:29 +0100 Subject: [PATCH 025/250] Make the tests pass on Windows Windows does not allow opening a NamedTemporaryFile a second time, its c_long has 32 bits, and its path separator is not "/". --- tests/Exscript/QueueTest.py | 2 +- tests/Exscript/util/fileTest.py | 33 +++++++++-------------- tests/Exscript/util/interactTest.py | 18 ++++++++----- tests/Exscript/util/mailTest.py | 10 ++++--- tests/templates/builtinvars/test.exscript | 4 +-- 5 files changed, 32 insertions(+), 35 deletions(-) diff --git a/tests/Exscript/QueueTest.py b/tests/Exscript/QueueTest.py index 8f6aafa3..a9378a28 100644 --- a/tests/Exscript/QueueTest.py +++ b/tests/Exscript/QueueTest.py @@ -258,7 +258,7 @@ def start_cb(data, job, host, conn): match_called = Value(ctypes.c_bool, False) start_called = Value(ctypes.c_bool, False) - account_hash = Value(ctypes.c_long, 0) + account_hash = Value(ctypes.c_ssize_t, 0) # c_long has only 32 bits on Windows data = { 'match-called': match_called, 'start-called': start_called, diff --git a/tests/Exscript/util/fileTest.py b/tests/Exscript/util/fileTest.py index ada99597..3d6186a8 100644 --- a/tests/Exscript/util/fileTest.py +++ b/tests/Exscript/util/fileTest.py @@ -25,28 +25,19 @@ def setUp(self): data += 'user2:' + base64.encodebytes(b'password2').decode('utf8') + '\n' data += 'user3 = ' + base64.encodebytes(b'password3').decode('utf8') + '\n' data += 'user4 : ' + base64.encodebytes(b'password4').decode('utf8') + '\n' - self.account_file = NamedTemporaryFile() - self.account_file.write(data.encode('utf8')) - self.account_file.flush() - - self.host_file = NamedTemporaryFile() - self.host_file.write('\n'.join(hosts).encode('utf8')) - self.host_file.flush() - + self.account_file = self.write_tempfile(data.encode('utf8')) + self.host_file = self.write_tempfile('\n'.join(hosts).encode('utf8')) content = '\n'.join([h + ' blah' for h in hosts]) - self.csv_host_file = NamedTemporaryFile() - self.csv_host_file.write(b'hostname test\n') - self.csv_host_file.write(content.encode('utf8')) - self.csv_host_file.flush() - - self.lib_file = NamedTemporaryFile() - self.lib_file.write(b'__lib__ = {"test": object}\n') - self.lib_file.flush() - - def tearDown(self): - self.account_file.close() - self.host_file.close() - self.csv_host_file.close() + self.csv_host_file = self.write_tempfile(b'hostname test\n' + content.encode('utf8')) + self.lib_file = self.write_tempfile(b'__lib__ = {"test": object}\n') + + def write_tempfile(self, data): + # Closed before the test opens it by name, which Windows requires. + file = NamedTemporaryFile(delete=False) + with file: + file.write(data) + self.addCleanup(os.remove, file.name) + return file def testGetAccountsFromFile(self): from Exscript.util.file import get_accounts_from_file diff --git a/tests/Exscript/util/interactTest.py b/tests/Exscript/util/interactTest.py index f88f1ca3..513ac205 100644 --- a/tests/Exscript/util/interactTest.py +++ b/tests/Exscript/util/interactTest.py @@ -1,5 +1,6 @@ +import os +import tempfile import unittest -from tempfile import NamedTemporaryFile import Exscript.util.interact from Exscript.util.interact import InputHistory @@ -9,16 +10,19 @@ class InputHistoryTest(unittest.TestCase): CORRELATE = InputHistory def setUp(self): - t = NamedTemporaryFile() - self.history = InputHistory(t.name) + # A file name in a temporary directory: Windows cannot reopen an open temporary file. + tmpdir = tempfile.TemporaryDirectory() + self.addCleanup(tmpdir.cleanup) + self.filename = os.path.join(tmpdir.name, 'history') + self.history = InputHistory(self.filename) def testConstructor(self): - t = NamedTemporaryFile() + filename = self.filename + '2' h = InputHistory() - h = InputHistory(t.name) - h = InputHistory(t.name, 'foo') + h = InputHistory(filename) + h = InputHistory(filename, 'foo') h.set('aaa', 'bbb') - with open(t.name) as fp: + with open(filename) as fp: self.assertEqual(fp.read(), '[foo]\naaa = bbb\n\n') def testGet(self): diff --git a/tests/Exscript/util/mailTest.py b/tests/Exscript/util/mailTest.py index de523db9..81f5668f 100644 --- a/tests/Exscript/util/mailTest.py +++ b/tests/Exscript/util/mailTest.py @@ -1,3 +1,4 @@ +import os import tempfile import unittest from getpass import getuser @@ -164,11 +165,12 @@ def testFromTemplateString(self): def testFromTemplate(self): from Exscript.util.mail import from_template - tmpfile = tempfile.NamedTemporaryFile() - tmpfile.write(template.encode('utf8')) - tmpfile.flush() + with tempfile.NamedTemporaryFile( + delete=False + ) as tmpfile: # Windows: close before reopening + tmpfile.write(template.encode('utf8')) + self.addCleanup(os.remove, tmpfile.name) mail = from_template(tmpfile.name, **vars) - tmpfile.close() self.checkResult(mail) def testSend(self): diff --git a/tests/templates/builtinvars/test.exscript b/tests/templates/builtinvars/test.exscript index 06d39ebb..f3ac8866 100644 --- a/tests/templates/builtinvars/test.exscript +++ b/tests/templates/builtinvars/test.exscript @@ -1,5 +1,5 @@ { fail "username: $__username__" if not __username__ matches /^\S+$/ - fail "hostname: $__hostname__" if not __hostname__ matches /^\S+\/pseudodev\.py$/ - fail "filename: $__filename__" if not __filename__ matches /^\S+\/test\.exscript$/ + fail "hostname: $__hostname__" if not __hostname__ matches /^\S+pseudodev\.py$/ + fail "filename: $__filename__" if not __filename__ matches /^\S+test\.exscript$/ } From 26d4d78cc440645401c0153ab3322b2b3cec2b86 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 3 Nov 2026 02:18:29 +0100 Subject: [PATCH 026/250] Run the Windows test job with the requested Python In PowerShell, the default shell of Windows runners, "$PYTHON" is not the environment variable, so the job ran with the runner's default Python. --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7eb0f786..562b1381 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -48,6 +48,7 @@ jobs: fetch-depth: 0 - uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3 - name: Run tests + shell: bash # also on Windows, where the default shell does not expand "$PYTHON" env: PYTHON: ${{ matrix.python }} run: uv run --frozen --python "$PYTHON" pytest --cov=Exscript --cov-report=term-missing:skip-covered From f4fe3b160e102f8a4454cbcc20b0261e5c3df54e Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 3 Nov 2026 02:35:15 +0100 Subject: [PATCH 027/250] Rewrite the generic prompt pattern without catastrophic backtracking Its optional parts could all match the same word characters, so a long run of them, as in hex dumps, could stall a session. The parts are separated by @, :, ~, / or a space, so possessive quantifiers match the same strings in linear time. --- CHANGES.md | 7 ++ src/Exscript/protocols/drivers/driver.py | 30 +++---- tests/Exscript/protocols/PromptRegexTest.py | 91 +++++++++++++++++++++ 3 files changed, 113 insertions(+), 15 deletions(-) create mode 100644 tests/Exscript/protocols/PromptRegexTest.py diff --git a/CHANGES.md b/CHANGES.md index 9b4059f0..c2f2f1f3 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -126,6 +126,13 @@ marked with **BREAKING**; a migration guide will accompany the release. ### Protocol Drivers +- **Generic Prompt Pattern in Linear Time:** The prompt pattern of the generic driver, + which other drivers inherit, backtracked catastrophically on long lines that are no + prompt: a line with a run of 100 word characters took 1.3 seconds to check, 200 + characters 25 seconds, so hex or base64 output could stall a session. The pattern + matches the same strings, verified against the old one, in linear time. A test + checks every driver pattern for this. + - **BREAKING - Graceful Logout:** `Protocol.close(force=False)` now calls the driver's new `logout()` method before the connection is closed. The base `Driver` sends `exit\r`, the custom `VRPDriver` sends `quit\r` and `SROSDriver` sends `logout\r`. diff --git a/src/Exscript/protocols/drivers/driver.py b/src/Exscript/protocols/drivers/driver.py index 40536481..182351f1 100644 --- a/src/Exscript/protocols/drivers/driver.py +++ b/src/Exscript/protocols/drivers/driver.py @@ -33,28 +33,28 @@ _ignore = r'[\x1b\x07\x00]' _nl = r'(?:\r\n|[\r\n])' _prompt_start = _nl + r'(?:' + _unprintable + r'*|' + _ignore + '*)' -_filename = r'(?:[\w\+\-\._]+)' -_path = r'(?:(?:' + _filename + r')?(?:/' + _filename + r')*/?)' -_any_path = r'(?:' + _path + r'|~' + _path + r'?)' -_host = r'(?:[\w+\-\.]+)' -_user = r'(?:[\w+\-]+)' -_user_host = r'(?:(?:' + _user + r'\@)?' + _host + r')' +# A prompt like "[user@host:~/path (config)]#", where most parts are optional. +# The runs of name characters are matched possessively (++, *+): the parts are +# separated by @, :, ~, / or a space, so a run never needs to be split. This +# keeps the time linear on long lines that are no prompt. +_name = r'[\w+\-.]' +_path = r'~?' + _name + r'*+(?:/' + _name + r'++)*+/?' _prompt_re = [ re.compile( _prompt_start + r'[\[\<]?' - + r'\w+' - + _user_host - + r'?' + + r'\w(?:[\w+\-]++@' # user@host, at least two characters before the @ + + _name + + r'++|' + + _name + + r'*+)' + r':?' - + _any_path - + r'?' + + _path + r'[: ]?' - + _any_path - + r'?' + + _path + r'(?:\(' - + _filename - + r'\))?' + + _name + + r'++\))?' + r'[\]\-]?' + r'[#>%\$\]] ?' + _unprintable diff --git a/tests/Exscript/protocols/PromptRegexTest.py b/tests/Exscript/protocols/PromptRegexTest.py new file mode 100644 index 00000000..4f7f4542 --- /dev/null +++ b/tests/Exscript/protocols/PromptRegexTest.py @@ -0,0 +1,91 @@ +import itertools +import random +import re +import string +import time +import unittest + +from Exscript.protocols.drivers import driver_map +from Exscript.protocols.drivers.driver import _prompt_re + + +def old_generic_prompt(): + """Returns a reference regex: the generic prompt regex must match the same strings.""" + unprintable = r'[^' + re.escape(string.printable) + r']' + start = r'(?:\r\n|[\r\n])(?:' + unprintable + r'*|[\x1b\x07\x00]*)' + filename = r'(?:[\w\+\-\._]+)' + path = r'(?:(?:' + filename + r')?(?:/' + filename + r')*/?)' + any_path = r'(?:' + path + r'|~' + path + r'?)' + user_host = r'(?:(?:(?:[\w+\-]+)\@)?(?:[\w+\-\.]+))' + middle = r'[\[\<]?\w+' + user_host + r'?:?' + any_path + r'?[: ]?' + any_path + r'?' + end = r'(?:\(' + filename + r'\))?[\]\-]?[#>%\$\]] ?' + unprintable + r'*\Z' + return re.compile(start + middle + end, re.I) + + +def span(regex, text): + match = regex.search(text) + return match.span() if match else None + + +class PromptRegexTest(unittest.TestCase): + """The prompt regexes of the drivers.""" + + def testGenericPromptMatchesTheSameStrings(self): + old, new = old_generic_prompt(), _prompt_re[0] + alphabet = list('ab_1é+-.@: ~/()[]<>#%$\r\nß\tX') + ['\x1b', '\x07', '\x00', '\x01'] + rng = random.Random(20261103) + for _ in range(30000): + prefix = rng.choice(['', '\r\n', '\n', 'x\r\n', '\r\n\x1b']) + text = prefix + ''.join(rng.choice(alphabet) for _ in range(rng.randint(0, 18))) + self.assertEqual(span(old, text), span(new, text), repr(text)) + for length in range(1, 5): + for chars in itertools.product(list('a.-@:/~ #]([') + ['\r\n'], repeat=length): + text = '\r\n' + ''.join(chars) + self.assertEqual(span(old, text), span(new, text), repr(text)) + + def testGenericPrompts(self): + prompts = [ + 'router#', + 'router> ', + 'user@host:~$ ', + 'user@host:~/dir/sub$ ', + '[user@host ~]$ ', + '', + 'RP/0/RSP0/CPU0:router(config)#', + 'router(config-if)#', + 'host%', + ] + for prompt in prompts: + self.assertTrue(_prompt_re[0].search('output\r\n' + prompt), prompt) + for text in 'a b c#', 'ab.cd@host#', 'a@host#', 'a:b:c:d#', 'line without prompt': + self.assertIsNone(_prompt_re[0].search('output\r\n' + text), text) + + def testAllRegexesNeedLinearTime(self): + # Lines that are no prompt but contain long runs of the characters that + # prompts consist of. A regex with catastrophic backtracking needs + # seconds to minutes for them; the others need microseconds. + lines = [ + 'a' * 120 + ' x', + 'a.' * 60 + '!', + 'a-' * 60 + '!', + '1 ' * 60 + '!', + 'a/' * 60 + '!', + ('a' * 60 + ' ') * 2 + '!', + 'ab:' * 40 + '!', + '(a' * 60 + '!', + 'a@' * 60 + '!', + '\x1b[0m' * 30 + 'a' * 60 + '!', + ] + texts = ['\r\n' + line for line in lines] + [' ' * 120 + '!', 'x\r\n' * 1000] + for name, driver in driver_map.items(): + for attr, value in vars(driver).items(): + if not isinstance(value, list): + continue + for regex in value: + if not isinstance(regex, re.Pattern): + continue + for text in texts: + start = time.perf_counter() + regex.search(text) + elapsed = time.perf_counter() - start + self.assertLess(elapsed, 0.2, f'{name}.{attr} on {text[:30]!r}') From 77fe27a81c253cfc11f9721655b5f1a1ada501a7 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 3 Nov 2026 03:27:16 +0100 Subject: [PATCH 028/250] Unify expect() for all protocol adapters Protocol does the matching and searches each character about once; the adapters only read. SSH no longer misses a match that more output follows in the same read. Login errors count only after the last send(), and a closed connection raises ConnectionClosedException. --- CHANGES.md | 43 ++++- src/Exscript/protocols/dummy.py | 38 +--- src/Exscript/protocols/exception.py | 9 + src/Exscript/protocols/netconf.py | 32 ++-- src/Exscript/protocols/protocol.py | 156 +++++++++++++-- src/Exscript/protocols/ssh2.py | 80 ++------ src/Exscript/protocols/telnet.py | 44 +---- src/Exscript/protocols/telnetlib.py | 159 +++------------- tests/Exscript/KnownIssuesTest.py | 63 +----- tests/Exscript/protocols/ExpectTest.py | 180 ++++++++++++++++++ tests/Exscript/protocols/telnetlibTest.py | 43 +++-- .../linux/alpine-3.22-bash-ssh.golden.json | 2 +- 12 files changed, 470 insertions(+), 379 deletions(-) create mode 100644 tests/Exscript/protocols/ExpectTest.py diff --git a/CHANGES.md b/CHANGES.md index c2f2f1f3..d2bdf592 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -97,6 +97,32 @@ marked with **BREAKING**; a migration guide will accompany the release. - `Exscript.util.url.Url.from_string(url, default_protocol='ssh')` - `exscript` CLI: `-p`/`--protocol` +### Waiting for Responses + +- **One Implementation for All Protocols:** `expect()`, `waitfor()`, `expect_prompt()` + and `execute()` work the same for SSH, Telnet, NETCONF and the dummy protocol, and + the adapters read up to 64 KiB at a time: + - patterns that end with `$` or `\Z`, like most driver prompts, are searched in the + last 1024 characters of the received data (before: 150); + - other patterns are searched in all data received since the last match. Before, + SSH searched only the last 150 characters after each read, so it missed a match + that was followed by more output in the same read and waited until the timeout. + Telnet found it, because it read 64 bytes at a time; + - a large response is searched once instead of after every read: `execute()` with a + 4 MiB response takes 0.6 seconds of CPU time over Telnet instead of 4.3. +- **BREAKING - `ConnectionClosedException`:** If the host closes the connection while + Exscript waits for a response, all protocols raise the new + `Exscript.protocols.exception.ConnectionClosedException`. Before, SSH raised a + `ProtocolException`, and Telnet an `EOFError`, or a `TimeoutException` if unmatched + data was left. The new exception is a `ProtocolException` and an `EOFError`, so only + code that expects a `TimeoutException` for a closed connection must change. +- **Login Errors After Sending Only:** `login()`, `app_authenticate()` and + `app_authorize()` raise `LoginFailure` only if the driver's login error patterns + match data that arrived after Exscript sent something, e.g. the password. Before, + a banner such as "Access denied for unauthorized users" failed the login before any + credentials were sent. A login error is detected with the next prompt, when the host + closes the connection, or at the timeout. + ### Telnet - **Telnet Client Based on CPython 3.12:** `Exscript.protocols.telnetlib` is now based @@ -115,14 +141,15 @@ marked with **BREAKING**; a migration guide will accompany the release. - passwords are no longer logged at debug level 5. - **BREAKING - `Exscript.protocols.telnetlib` Is Internal:** The module contains only what the Telnet adapter uses. Removed: the methods `read_until()`, `read_some()`, - `read_eager()`, `read_very_eager()`, `read_lazy()`, `read_very_lazy()`, `interact()` - and `mt_interact()` with their helpers, `sock_avail()`, `get_socket()`, `fileno()` - and `set_receive_callback()`; the function `test()`; the option constants other - than those Exscript negotiates; and the `stdout` and `stderr` arguments. The - remaining constants are integers instead of bytes, and the options of the - constructor are keyword-only. Use `Exscript.protocols.Telnet`. If you need a drop-in - replacement for the telnetlib module that Python 3.13 removed, use the package - `standard-telnetlib`. + `read_eager()`, `read_very_eager()`, `read_lazy()`, `read_very_lazy()`, `read_all()`, + `expect()`, `waitfor()`, `interact()` and `mt_interact()` with their helpers, + `sock_avail()`, `get_socket()`, `fileno()` and `set_receive_callback()`; the + attribute `cancel_expect`; the function `test()`; the option constants other than + those Exscript negotiates; and the `stdout` and `stderr` arguments. The new method + `receive(timeout)` reads once and passes the text to the callback. The remaining + constants are integers instead of bytes, and the options of the constructor are + keyword-only. Use `Exscript.protocols.Telnet`. If you need a drop-in replacement for + the telnetlib module that Python 3.13 removed, use the package `standard-telnetlib`. ### Protocol Drivers diff --git a/src/Exscript/protocols/dummy.py b/src/Exscript/protocols/dummy.py index fd6c5269..cd32f9e6 100644 --- a/src/Exscript/protocols/dummy.py +++ b/src/Exscript/protocols/dummy.py @@ -62,25 +62,6 @@ def __init__(self, device=None, **kwargs): def is_dummy(self): return True - def _expect_any(self, prompt_list, flush=True): - self._doinit() - - # Cancelled by a callback during self._say(). - if self._expect_canceled: - return -2, None, self.response - - # Look for a match in the buffer. - for i, prompt in enumerate(prompt_list): - matches = prompt.search(str(self._buffer)) - if matches is not None: - self.response = self._buffer.head(matches.start()) - if flush: - self._buffer.pop(matches.end()) - return i, matches, self.response - - # "Timeout". - return -1, None, self.response - def _say(self, string): self._receive_cb(string) self._buffer.append(string) @@ -108,20 +89,11 @@ def _send(self, data): self._say(response) def _domatch(self, prompt, flush): - # Wait for a prompt. - result, match, self.response = self._expect_any(prompt, flush) - - if match: - self._dbg(2, "Got a prompt, match was: %s" % repr(match.group())) - self._dbg(4, "Response (flush=%s): %s" % (repr(flush), repr(self.response))) - - if result == -1: - self._raise_on_response_timeout() - if result == -2: - self._raise_on_canceled_expect() - if self.response is None: - self._raise_on_response_eof() - return result, match + self._doinit() + return super()._domatch(prompt, flush) + + def _fill_buffer(self, timeout): + return None # the device answers only in _send(), so nothing more arrives def _logout(self): pass diff --git a/src/Exscript/protocols/exception.py b/src/Exscript/protocols/exception.py index d984d545..77fef248 100644 --- a/src/Exscript/protocols/exception.py +++ b/src/Exscript/protocols/exception.py @@ -42,6 +42,15 @@ class TimeoutException(ProtocolException): pass +class ConnectionClosedException(ProtocolException, EOFError): + """ + An exception that is thrown if the connected host closed the + connection while a prompt was expected. It is also an EOFError. + """ + + pass + + class ExpectCancelledException(ProtocolException): """ An exception that is thrown if Protocol.cancel_expect() diff --git a/src/Exscript/protocols/netconf.py b/src/Exscript/protocols/netconf.py index a771af0b..3a5f977e 100644 --- a/src/Exscript/protocols/netconf.py +++ b/src/Exscript/protocols/netconf.py @@ -357,22 +357,22 @@ def reassemble_chunks(response): return assembled def _domatch(self, prompt, flush): - # This mimics the behavior of CLI-driven prompt matching to keep API - # parity with the Telnet and SSH2 adapter. - deadline = time.monotonic() + self.get_timeout() - while not self._expect_canceled: - n, match, search_window = self._domatch_buffer(prompt) - if not match: - # This is just a safety-net as the buffer is always filled by - # the replication wrapper and ncclient's blocking dispatch. - if time.monotonic() > deadline: - self._raise_on_response_timeout() - time.sleep(0.05) - continue - self.response = self._domatch_extract(match, len(search_window), flush) - self.response = self.reassemble_chunks(self.response) - return n, match - self._raise_on_canceled_expect() + n, match = super()._domatch(prompt, flush) + self.response = self.reassemble_chunks(self.response) + return n, match + + def _fill_buffer(self, timeout): + # ncclient's thread reads the channel and appends to the buffer (see the + # recv wrapper in _init_ncclient()), so this only waits for new data. + size = self._buffer.size() + deadline = None if timeout is None else time.monotonic() + timeout + while self._buffer.size() == size: + if not self._replicate: # the channel was closed + return False + if deadline is not None and time.monotonic() >= deadline: + return None + time.sleep(0.01) + return True def _send(self, data): self._ncc_response = None diff --git a/src/Exscript/protocols/protocol.py b/src/Exscript/protocols/protocol.py index 29d0383b..223f18cc 100644 --- a/src/Exscript/protocols/protocol.py +++ b/src/Exscript/protocols/protocol.py @@ -48,6 +48,7 @@ from ..util.url import _WELL_KNOWN_PORTS from .drivers import Driver, driver_map from .exception import ( + ConnectionClosedException, DriverReplacedException, ExpectCancelledException, InvalidCommandException, @@ -77,6 +78,29 @@ ) _skey_re = re.compile(r'(?:s\/key|otp-md4) (\d+) (\S+)(?=\s|[\r\n])') +# Prompts end the received data, so patterns that end with $ or \Z are searched +# only in the end of the buffer. +_PROMPT_WINDOW = 1024 +# The other patterns are searched in the data that arrived since the last +# search, plus this many characters before it, so a match may span two reads. +_LOOKBEHIND = 150 + + +def _ends_with_anchor(regex): + """Whether the regular expression ends with an unescaped $ or \\Z.""" + pattern = regex.pattern.rstrip(')') + if pattern.endswith('$') and not regex.flags & re.MULTILINE: + stem = pattern[:-1] + elif pattern.endswith('Z'): + stem = pattern[:-1] + if not stem.endswith('\\'): + return False + stem = stem[:-1] + else: + return False + return (len(stem) - len(stem.rstrip('\\'))) % 2 == 0 # the anchor itself is not escaped + + # Needed on module level as each host connection has its own thread. _connection_monitor: dict[str, float] = {} _connection_monitor_lock = threading.RLock() @@ -440,6 +464,8 @@ def _init(self): self._driver_replaced = False self._expect_canceled = False self._last_account = None + # The buffer position at the last send(); None before the first one. + self._sent_at = None def _reset(self): # Reset all per-session state settings when close() is called. @@ -764,12 +790,13 @@ def get_connect_timeout(self): def set_timeout(self, timeout): """ - Defines the maximum time that the adapter waits before a call to - :meth:`expect()`, :meth:`waitfor()` or :meth:`expect_prompt()` fails - with a raised TimeoutException. + Defines how long :meth:`expect()`, :meth:`waitfor()` or + :meth:`expect_prompt()` wait for data from the remote host before they + fail with a raised TimeoutException. The time starts again whenever + data arrives, so a long output that keeps arriving does not time out. :type timeout: int - :param timeout: The maximum response time in seconds. + :param timeout: The maximum time without data, in seconds. """ self.timeout = int(timeout) @@ -1071,6 +1098,10 @@ def is_protocol_authenticated(self): def _app_authenticate(self, account, password, flush=True, bailout=False): """ Generic implementation for app-level authentications or authorizations. + + A login error counts only if the host sent it after the last call of + send(), e.g. after the password: banners and the message of the day + may contain words like "denied" or "failed". """ user = account.get_name() @@ -1080,7 +1111,6 @@ def _app_authenticate(self, account, password, flush=True, bailout=False): # structure to allow for mapping the match index back to the # prompt type. prompts = ( - ('login-error', self.get_login_error_prompt()), ('username', self.get_username_prompt()), ('skey', [_skey_re]), ('password', self.get_password_prompt()), @@ -1097,23 +1127,22 @@ def _app_authenticate(self, account, password, flush=True, bailout=False): try: index, match = self._waitfor(prompt_list) except TimeoutException: + self._raise_on_login_error() self._raise_on_response_timeout() except DriverReplacedException: msg = "Handling DriverReplacedException, retrying to wait for a prompt." self._dbg(4, msg) continue - except ExpectCancelledException: - raise - except EOFError: + except ConnectionClosedException: + self._raise_on_login_error() raise - # Login error detected. + # Login error detected, e.g. followed by a new user name prompt. + self._raise_on_login_error() section, prompt = prompt_map[index] - if section == 'login-error': - raise LoginFailure("Login failed") # User name prompt. - elif section == 'username': + if section == 'username': self._dbg(2, "Username prompt received.") self.expect(prompt) # consume the prompt from the buffer self.send(user + '\r') @@ -1311,6 +1340,7 @@ def send(self, data): self._dbg(4, 'Sending: ') else: self._dbg(4, 'Sending: %s' % repr(data)) + self._sent_at = self._buffer.size() # before: Dummy answers within _send() self._send(data) @contextlib.contextmanager @@ -1356,7 +1386,92 @@ def execute(self, command, consume=True): def _domatch(self, prompt, flush): """ - Should be overwritten. That's the place where self.response should be set. + Waits until one of the given regular expressions matches the received + data, and sets self.response to the data before the match. With flush, + removes the data up to the end of the match from the buffer. + + Raises TimeoutException if no data arrives for self.timeout seconds, + and ConnectionClosedException if the host closed the connection. + + :type prompt: list(re.Pattern) + :param prompt: The regular expressions. + :type flush: bool + :param flush: Whether to remove the response and the match from the + buffer. + :rtype: int, re.Match + :return: The index of the regular expression that matched, and the match. + :raises TimeoutException: If no data arrives for self.timeout seconds. + :raises ConnectionClosedException: If the host closed the connection. + """ + searched = 0 + while True: + if self._expect_canceled: + self._raise_on_canceled_expect() + n, match, start, end = self._search(prompt, searched) + if match is not None: + break + searched = self._buffer.size() + received = self._fill_buffer(self.timeout) + if received is None: + self._raise_on_response_timeout() + if not received: + self._raise_on_response_eof() + self._dbg(2, 'Got a prompt, match was: %s' % repr(match.group())) + self.response = self._buffer.head(start) + if flush: + self._buffer.pop(end) + if self._sent_at is not None: + self._sent_at = max(0, self._sent_at - end) + self._dbg(4, 'Response (flush=%s): %s' % (repr(flush), repr(self.response))) + return n, match + + def _search(self, prompt, searched): + """ + Searches the buffer for the first of the given regular expressions + that matches. A regular expression that ends with $ or \\Z is searched + in the last _PROMPT_WINDOW characters, after the driver's cleanup of + the response; the others in the data from the given position on, plus + _LOOKBEHIND characters before it. + + :type prompt: list(re.Pattern) + :param prompt: The regular expressions. + :type searched: int + :param searched: The size of the buffer at the last search. + :rtype: tuple(int, re.Match|None, int, int) + :return: The index of the regular expression, the match, and the start + and end of the match in the buffer; -1, None, 0, 0 if none matched. + """ + size = self._buffer.size() + tail = new = None + for n, regex in enumerate(prompt): + if _ends_with_anchor(regex): + if tail is None: + raw = self._buffer.tail(_PROMPT_WINDOW) + tail, incomplete = self.get_driver().clean_response_for_re_match(raw) + tail_offset = size - len(incomplete) - len(tail) + match = regex.search(tail) + offset = tail_offset + else: + if new is None: + new_offset = max(0, searched - _LOOKBEHIND) + new = self._buffer.tail(size - new_offset) + match = regex.search(new) + offset = new_offset + if match is not None: + return n, match, offset + match.start(), offset + match.end() + return -1, None, 0, 0 + + def _fill_buffer(self, timeout): + """ + Should be overwritten. Waits for data from the remote host, and passes + the received text to _receive_cb() and to the buffer. + + :type timeout: float|None + :param timeout: The maximum time to wait, in seconds; None waits + without limit. + :rtype: bool|None + :return: True if data was received, None if the time ran out, False + if the host closed the connection. """ raise NotImplementedError() @@ -1538,6 +1653,19 @@ def _raise_on_response_with_errors(self, line): line = line[:200] + '...' raise InvalidCommandException('Error in the response from the device: ' + line) + def _raise_on_login_error(self): + """ + Raises LoginFailure if one of the login error patterns matches the + data that the host sent after the last call of send(). + """ + if self._sent_at is None: + return + size = self._buffer.size() + text = self._buffer.tail(size - min(self._sent_at, size)) + for regex in self.get_login_error_prompt(): + if regex.search(text): + raise LoginFailure('Login failed') + def _raise_on_response_timeout(self): buffer = str(self._buffer)[-200:] if buffer: @@ -1550,7 +1678,7 @@ def _raise_on_response_eof(self): if buffer: buffer = '...' + buffer msg = 'EOF while waiting for a response/prompt. Buffer: %s' - raise ProtocolException(msg % repr(buffer)) + raise ConnectionClosedException(msg % repr(buffer)) def _call_key_handlers(self, key_handlers, data): if key_handlers is not None: diff --git a/src/Exscript/protocols/ssh2.py b/src/Exscript/protocols/ssh2.py index e0f03fb0..f495616c 100644 --- a/src/Exscript/protocols/ssh2.py +++ b/src/Exscript/protocols/ssh2.py @@ -24,12 +24,11 @@ SSH version 2 support, based on paramiko. """ +import codecs import logging import os -import select import socket import threading -import time from binascii import hexlify import paramiko @@ -50,6 +49,7 @@ _logger = logging.getLogger(__name__) _logger.addHandler(logging.NullHandler()) _known_hosts_lock = threading.Lock() +_READ_SIZE = 65536 def _prefer_key_type(options, keytype): @@ -379,6 +379,8 @@ def _paramiko_channel(self): msg = 'Failed to open a channel with pty/shell. (%s)' % e self._dbg(1, msg) raise LoginFailure(msg) + # Decodes characters that span two reads, too. + self._decoder = codecs.getincrementaldecoder(self.encoding)() def _connect(self): self._transport = self._paramiko_connect() @@ -429,65 +431,19 @@ def _send(self, data): self._dbg(1, 'Error while writing to connection') raise - def _wait_for_data(self): - end = time.time() + self.timeout - while True: - readable, writeable, excp = select.select([self._channel], [], [], 1) - if readable: - return True - if time.time() > end: - return False - - def _fill_buffer(self): - # Wait for a response of the device. - if not self._wait_for_data(): - self._raise_on_response_timeout() - - # Read the response. - data = self._channel.recv(1024).decode(self.encoding) - if not data: - return False - self._receive_cb(data) - self._buffer.append(data) - return True - - def _domatch_buffer(self, prompt): - # Check whether the buffered data we received so far matches the prompt. - driver = self.get_driver() - search_window_size = 150 - search_window = self._buffer.tail(search_window_size) - search_window, incomplete_tail = driver.clean_response_for_re_match(search_window) - n, match = -1, None - for n, regex in enumerate(prompt): - match = regex.search(search_window) - if match is not None: - break - if match: - self._dbg(2, "Got a prompt, match was: %s" % repr(match.group())) - return n, match, search_window - - def _domatch_extract(self, match, search_window_length, flush): - # This extracts the response from the buffer. The end position is - # where the matched prompt starts. So the response is everything - # in the buffer up to this end position. - end = self._buffer.size() - search_window_length + match.start() - response = self._buffer.head(end) - if flush: - prompt_length = match.end() - match.start() - self._buffer.pop(end + prompt_length) - self._dbg(4, "Response (flush=%s): %s" % (repr(flush), repr(response))) - return response - - def _domatch(self, prompt, flush): - while not self._expect_canceled: - n, match, search_window = self._domatch_buffer(prompt) - if not match: - if not self._fill_buffer(): # reads the channel - self._raise_on_response_eof() - continue - self.response = self._domatch_extract(match, len(search_window), flush) - return n, match - self._raise_on_canceled_expect() + def _fill_buffer(self, timeout): + self._channel.settimeout(timeout) + try: + data = self._channel.recv(_READ_SIZE) + except TimeoutError: + return None + finally: + self._channel.settimeout(None) # sendall() blocks without limit, as before + text = self._decoder.decode(data, final=not data) + if text: + self._receive_cb(text) + self._buffer.append(text) + return bool(data) def _set_terminal_size(self, rows, cols): self._channel.resize_pty(cols, rows) @@ -500,7 +456,7 @@ def _close(self, force=False): if not force: try: if self._channel.recv_ready(): - self._fill_buffer() + self._fill_buffer(0) except Exception: pass self._channel.close() diff --git a/src/Exscript/protocols/telnet.py b/src/Exscript/protocols/telnet.py index c9896f84..0077bbb9 100644 --- a/src/Exscript/protocols/telnet.py +++ b/src/Exscript/protocols/telnet.py @@ -62,7 +62,6 @@ def _connect(self): termtype=self.termtype, receive_callback=self._telnetlib_received, ) - self._tn.cancel_expect = self._expect_canceled # sync states if self.debug >= 5: self._tn.set_debuglevel(1) return True @@ -75,42 +74,8 @@ def _send(self, data): self._dbg(1, 'Error while writing to connection') raise - def _domatch(self, prompt, flush): - if flush: - func = self._tn.expect - else: - func = self._tn.waitfor - - # Wait for a prompt. - cleanup_callback = self.get_driver().clean_response_for_re_match - try: - result, match, self.response = func(prompt, self.timeout, cleanup=cleanup_callback) - except Exception: - self._dbg(1, 'Error while waiting for ' + repr(prompt)) - raise - - if match: - self._dbg(2, "Got a prompt, match was: %s" % repr(match.group())) - self._dbg(4, "Response (flush=%s): %s" % (repr(flush), repr(self.response))) - if flush: - # Only to mimic the telnetlib cooked-queue behavior to keep the - # buffer clean for the buffer monitoring feature. - prompt_length = match.end() - match.start() - self._buffer.pop(len(self.response) + prompt_length) - - if result == -1: - self._raise_on_response_timeout() - if result == -2: - self._raise_on_canceled_expect() - if self.response is None: - self._raise_on_response_eof() - - return result, match - - def cancel_expect(self): - super().cancel_expect() - if self._tn: - self._tn.cancel_expect = True + def _fill_buffer(self, timeout): + return self._tn.receive(timeout) # the callback appends the text to the buffer def _set_terminal_size(self, rows, cols): self._tn.set_window_size(rows, cols) @@ -122,8 +87,9 @@ def _close(self, force=False): if self._tn: if not force: try: - self._tn.sock.settimeout(2) - self._tn.read_all() + # Receive the rest until the host closes the connection. + while self._fill_buffer(2): + pass except Exception: pass self._tn.close() diff --git a/src/Exscript/protocols/telnetlib.py b/src/Exscript/protocols/telnetlib.py index 4b1798a9..e6f1c61e 100644 --- a/src/Exscript/protocols/telnetlib.py +++ b/src/Exscript/protocols/telnetlib.py @@ -14,10 +14,9 @@ # a callback, and NUL bytes are removed from it (the original also removed # XON); the terminal type (TTYPE) and window size (NAWS) options are # negotiated and the echo option is accepted, all other options are refused; -# commands are parsed in whole reads instead of byte by byte; the client waits -# for any of several regular expressions in a search window that a cleanup -# function can process; the read_*() variants, interact(), the option -# negotiation callback and the deprecation warning were removed. +# commands are parsed in whole reads instead of byte by byte; receive() waits +# for one read with a timeout; expect(), the read_*() variants, interact(), +# the option negotiation callback and the deprecation warning were removed. """ The Telnet client (RFC 854) of Exscript's Telnet adapter, derived from the telnetlib module of CPython. @@ -27,12 +26,8 @@ import codecs import logging -import re -import select import socket import struct -from io import StringIO -from time import monotonic __all__ = ['Telnet'] @@ -58,9 +53,7 @@ _NEGOTIATION = {DO: 'DO', DONT: 'DONT', WILL: 'WILL', WONT: 'WONT'} _IAC = bytes([IAC]) -# _waitfor() searches the end of the received text after every read, so its -# search window covers all data only with small reads. -_READ_SIZE = 64 +_READ_SIZE = 65536 def _command(*codes): @@ -111,11 +104,9 @@ def __init__( self.window_size = termsize self.termtype = termtype self.data_callback = receive_callback - self.cancel_expect = False self.can_naws = False self.eof = False self.rawq = b'' # received bytes that end with an incomplete command - self.cookedq = StringIO() self._decoder = codecs.getincrementaldecoder(encoding)() self._sb = None # the data of the subnegotiation in progress if host: @@ -264,10 +255,8 @@ def process_rawq(self): self.rawq = b'' self._sb = None text = self._decoder.decode(bytes(cooked).replace(b'\0', b''), self.eof) - if text: - self.cookedq.write(text) - if self.data_callback is not None: - self.data_callback(text) + if text and self.data_callback is not None: + self.data_callback(text) def fill_rawq(self): """ @@ -279,123 +268,27 @@ def fill_rawq(self): self.eof = not data self.rawq += data - def _read_cooked(self): - text = self.cookedq.getvalue() - self.cookedq.seek(0) - self.cookedq.truncate() - return text - - def read_all(self): - """ - Reads until the connection is closed. - - :rtype: str - :return: The received text that expect() did not consume. - """ - self.process_rawq() - while not self.eof: - self.fill_rawq() - self.process_rawq() - return self._read_cooked() - - def _wait_for_data(self, timeout): - # Waits in steps of one second; a single select() with the full timeout - # did not always return when data arrived. - end = monotonic() + timeout - while True: - readable, _, _ = select.select([self.sock], [], [], 1) - if readable: - return True - if monotonic() > end: - return False - - def _waitfor(self, relist, timeout=None, flush=False, cleanup=None): - relist = [regex if hasattr(regex, 'search') else re.compile(regex) for regex in relist] - search_window_size = 150 - head_lookback_size = 10 - self.msg('Expecting: %s', [regex.pattern for regex in relist]) - incomplete_tail = '' - clean_sw_size = search_window_size - while True: - self.process_rawq() - if self.cancel_expect: - self.cancel_expect = False - self.msg('Cancelling expect()') - return -2, None, '' - qlen = self.cookedq.tell() - if cleanup: - while True: - pos = qlen - clean_sw_size - len(incomplete_tail) - head_lookback_size - self.cookedq.seek(max(0, pos)) - search_window = self.cookedq.read() - search_window, incomplete_tail = cleanup(search_window) - search_window_len = len(search_window) - if clean_sw_size > qlen or search_window_len >= search_window_size: - search_window = search_window[-search_window_size:] - if search_window_len > search_window_size: - clean_sw_size -= search_window_size - break - clean_sw_size += search_window_size - else: - self.cookedq.seek(max(0, qlen - search_window_size)) - search_window = self.cookedq.read() - for index, regex in enumerate(relist): - match = regex.search(search_window) - if match is None: - continue - self.msg('Match found: %r', match.group()) - self.cookedq.seek(0) - text = self.cookedq.read(qlen - (match.end() - match.start())) - if flush: - self.cookedq.seek(0) - self.cookedq.truncate() - self.cookedq.write(search_window[match.end() :]) - else: - self.cookedq.seek(qlen) - return index, match, text - if self.eof: - break - if timeout is not None and not self._wait_for_data(timeout): - break - self.fill_rawq() - text = self._read_cooked() - if not text and self.eof: - raise EOFError('telnet connection closed') - return -1, None, text - - def waitfor(self, relist, timeout=None, cleanup=None): - """ - Reads until one of the regular expressions matches the end of the - received text, and keeps the text. - - Returns a tuple of the index of the first regular expression that - matched, the match object, and the text up to the match. When nothing - matched until the timeout, returns (-1, None, text) with the received - text; when expect was cancelled, returns (-2, None, ''). Raises - EOFError if the connection was closed and no text was received. - - :type relist: list(str|re.Pattern) - :param relist: The regular expressions. - :type timeout: float|None - :param timeout: Seconds without received data until the timeout. - :type cleanup: callable|None - :param cleanup: Returns the text to search and an incomplete tail. - :rtype: tuple(int, re.Match|None, str) - :return: The index, the match, and the text. - """ - return self._waitfor(relist, timeout, flush=False, cleanup=cleanup) - - def expect(self, relist, timeout=None, cleanup=None): + def receive(self, timeout): """ - Like waitfor(), but removes the text up to the end of the match. + Waits for data, reads it once and passes the received text to the + callback. - :type relist: list(str|re.Pattern) - :param relist: The regular expressions. :type timeout: float|None - :param timeout: Seconds without received data until the timeout. - :type cleanup: callable|None - :param cleanup: Returns the text to search and an incomplete tail. - :rtype: tuple(int, re.Match|None, str) - :return: The index, the match, and the text. + :param timeout: The maximum time to wait, in seconds; None waits + without limit. + :rtype: bool|None + :return: True if data was received, None if the time ran out, False + if the connection was closed. """ - return self._waitfor(relist, timeout, flush=True, cleanup=cleanup) + if self.eof: + return False + self.sock.settimeout(timeout) + try: + self.fill_rawq() + except (TimeoutError, BlockingIOError): # BlockingIOError: timeout 0 + return None + finally: + if self.sock: + self.sock.settimeout(self.connect_timeout) # for sendall() + self.process_rawq() + return not self.eof diff --git a/tests/Exscript/KnownIssuesTest.py b/tests/Exscript/KnownIssuesTest.py index ead24ceb..b4b7198a 100644 --- a/tests/Exscript/KnownIssuesTest.py +++ b/tests/Exscript/KnownIssuesTest.py @@ -7,75 +7,16 @@ and move the test next to the tests of the fixed module. """ -import os -import socket import sys import threading import time import unittest import uuid -from Exscript import Account, PrivateKey +from Exscript import Account from Exscript.emulators import VirtualDevice -from Exscript.protocols import SSH2, Dummy, Telnet +from Exscript.protocols import SSH2, Dummy from Exscript.protocols.drivers import driver_map, isam -from Exscript.servers import SSHd, Telnetd - -HOST = '127.0.0.1' -KEYFILE = os.path.join(os.path.dirname(__file__), 'protocols', 'id_rsa') - - -def free_port(): - """Returns a currently unused TCP port on the loopback interface.""" - with socket.socket() as sock: - sock.bind((HOST, 0)) - return sock.getsockname()[1] - - -class ServerMixin: - """Starts an Exscript test server for the test and stops it afterwards.""" - - def start_daemon(self, daemon_cls, device, **kwargs): - port = free_port() - daemon = daemon_cls(HOST, port, device, **kwargs) - daemon.start() - time.sleep(0.2) - self.addCleanup(daemon.join) # cleanups run LIFO: exit() first - self.addCleanup(daemon.exit) - return port - - -class ExpectTest(ServerMixin, unittest.TestCase): - """expect() must find a pattern regardless of how much data follows it.""" - - def setUp(self): - self.device = VirtualDevice(HOST, login_type=VirtualDevice.LOGIN_TYPE_NONE, strict=False) - self.device.add_command('show big', 'Please confirm: MARKER ' + 'x' * 2000) - - def expectMarker(self, conn, port=None): - self.addCleanup(conn.close, force=True) - conn.connect(HOST if port else 'dummy', port) - conn.protocol_authenticate(Account('user', 'password', needs_lock=False)) - conn.send('show big\r') - conn.expect('MARKER') - - def testDummy(self): - self.expectMarker(Dummy(device=self.device)) - - def testTelnet(self): - port = self.start_daemon(Telnetd, self.device) - self.expectMarker(Telnet(timeout=1, connect_rate_limit=0), port) - - @unittest.expectedFailure # SSH2 only searches the last 150 characters after each read - def testSSH2(self): - port = self.start_daemon(SSHd, self.device, key=PrivateKey.from_file(KEYFILE)) - conn = SSH2( - timeout=1, - connect_rate_limit=0, - verify_fingerprint=False, - preferred_authentications=['password'], - ) - self.expectMarker(conn, port) class DriverTest(unittest.TestCase): diff --git a/tests/Exscript/protocols/ExpectTest.py b/tests/Exscript/protocols/ExpectTest.py new file mode 100644 index 00000000..92bc7ca0 --- /dev/null +++ b/tests/Exscript/protocols/ExpectTest.py @@ -0,0 +1,180 @@ +import os +import re +import socket +import time +import unittest + +from Exscript import Account, PrivateKey +from Exscript.emulators import VirtualDevice +from Exscript.protocols import SSH2, Dummy, Telnet +from Exscript.protocols.exception import ( + ConnectionClosedException, + LoginFailure, + ProtocolException, + TimeoutException, +) +from Exscript.protocols.protocol import Protocol, _ends_with_anchor +from Exscript.servers import SSHd, Telnetd + +HOST = '127.0.0.1' +KEYFILE = os.path.join(os.path.dirname(__file__), 'id_rsa') + + +class Scripted(Protocol): + """ + Receives the given chunks, one per read; None is a read that times out. + After the last chunk, the host closes the connection. Records what was sent. + """ + + def __init__(self, chunks, **kwargs): + super().__init__(connect_rate_limit=0, **kwargs) + self.chunks = list(chunks) + self.sent = [] + self.timeouts = [] + + def _connect(self): + return True + + def _send(self, data): + self.sent.append(data) + self._send_cb(data) + + def _fill_buffer(self, timeout): + self.timeouts.append(timeout) + if not self.chunks: + return False + chunk = self.chunks.pop(0) + if chunk is None: + return None + self._receive_cb(chunk) + self._buffer.append(chunk) + return True + + def _close(self, force=False): + pass + + +def connect(chunks, **kwargs): + conn = Scripted(chunks, **kwargs) + conn.connect('router', 23) + return conn + + +class ExpectTest(unittest.TestCase): + """The expect engine of Protocol, used by all adapters.""" + + def testMatchInTheMiddleOfALargeRead(self): + conn = connect(['x' * 100000 + ' MARKER ' + 'y' * 100000 + '\r\nrouter# ']) + conn.expect('MARKER') + self.assertEqual(conn.response, 'x' * 100000 + ' ') + conn.expect_prompt() + self.assertEqual(conn.response, ' ' + 'y' * 100000) + + def testMatchAcrossReads(self): + conn = connect(['abcMAR', 'KERdef']) + conn.expect('MARKER') + self.assertEqual(conn.response, 'abc') + + def testWaitforKeepsTheData(self): + conn = connect(['abc MARKER def']) + conn.waitfor('MARKER') + conn.waitfor('MARKER') + conn.expect('def') + self.assertEqual(conn.response, 'abc MARKER ') + + def testTimeoutRestartsWithEveryRead(self): + conn = connect(['line\r\n'] * 5 + [None], timeout=7) + self.assertRaises(TimeoutException, conn.expect_prompt) + self.assertEqual(conn.timeouts, [7] * 6) + + def testConnectionClosed(self): + conn = connect(['partial output']) + with self.assertRaises(ConnectionClosedException) as cm: + conn.expect_prompt() + self.assertIsInstance(cm.exception, ProtocolException) + self.assertIsInstance(cm.exception, EOFError) # callers may catch EOFError + + def testEndsWithAnchor(self): + for pattern in r'#$', r'#\Z', r'(?:#|>)$', r'a\\$', r'a\\\Z': + self.assertTrue(_ends_with_anchor(re.compile(pattern)), pattern) + for pattern in r'#', r'\$', r'a\\\$', r'\\Z', r'[$]', r'(a$|b)': + self.assertFalse(_ends_with_anchor(re.compile(pattern)), pattern) + self.assertFalse(_ends_with_anchor(re.compile(r'#$', re.M))) # $ ends every line + + +class LoginErrorTest(unittest.TestCase): + """A login error counts only if the host sent it after the last send().""" + + account = Account('admin', 'secret', needs_lock=False) + + def connect(self, chunks): + return connect(chunks, driver='generic') # no OS guessing, no enable command + + def testKeywordsInTheBannerAreNoError(self): + banner = 'Unauthorized access is denied!\r\nLast failed login: today\r\n' + conn = self.connect([banner + 'User: ', 'Password: ', '\r\nWelcome\r\nrouter> ']) + conn.login(self.account) + self.assertEqual(conn.sent, ['admin\r', 'secret\r']) + + def testKeywordsWithoutCredentialsAreNoError(self): + conn = self.connect(['Access denied for guests\r\nrouter> ']) # as after SSH authentication + conn.login(self.account) + self.assertEqual(conn.sent, []) + + def testWrongPassword(self): + conn = self.connect(['User: ', 'Password: ', '\r\n% Login invalid\r\n\r\nUser: ']) + self.assertRaises(LoginFailure, conn.login, self.account) + self.assertEqual(conn.sent, ['admin\r', 'secret\r']) + + def testWrongPasswordAndDisconnect(self): + conn = self.connect(['Password: ', '\r\nAccess denied\r\n']) + self.assertRaises(LoginFailure, conn.login, self.account) + + def testFailedAuthorization(self): + conn = self.connect(['\r\nrouter> ', 'enable\r\n% Authorization failed.\r\nrouter> ']) + conn.login(self.account) + conn.send('enable\r') + self.assertRaises(LoginFailure, conn.app_authorize, self.account) + + +class ServerExpectTest(unittest.TestCase): + """expect() finds a pattern regardless of how much data follows it.""" + + def setUp(self): + self.device = VirtualDevice(HOST, login_type=VirtualDevice.LOGIN_TYPE_NONE, strict=False) + self.device.add_command('show big', 'Please confirm: MARKER ' + 'x' * 2000) + + def start_daemon(self, daemon_cls, **kwargs): + with socket.socket() as sock: + sock.bind((HOST, 0)) + port = sock.getsockname()[1] + daemon = daemon_cls(HOST, port, self.device, **kwargs) + daemon.start() + time.sleep(0.2) + self.addCleanup(daemon.join) # cleanups run LIFO: exit() first + self.addCleanup(daemon.exit) + return port + + def expectMarker(self, conn, port=None): + self.addCleanup(conn.close, force=True) + conn.connect(HOST if port else 'dummy', port) + conn.protocol_authenticate(Account('user', 'password', needs_lock=False)) + conn.send('show big\r') + conn.expect('MARKER') + + def testDummy(self): + self.expectMarker(Dummy(device=self.device)) + + def testTelnet(self): + port = self.start_daemon(Telnetd) + self.expectMarker(Telnet(timeout=1, connect_rate_limit=0), port) + + def testSSH2(self): + port = self.start_daemon(SSHd, key=PrivateKey.from_file(KEYFILE)) + conn = SSH2( + timeout=1, + connect_rate_limit=0, + verify_fingerprint=False, + preferred_authentications=['password'], + ) + self.expectMarker(conn, port) diff --git a/tests/Exscript/protocols/telnetlibTest.py b/tests/Exscript/protocols/telnetlibTest.py index eaab54b2..4c47c21e 100644 --- a/tests/Exscript/protocols/telnetlibTest.py +++ b/tests/Exscript/protocols/telnetlibTest.py @@ -24,17 +24,19 @@ def recv(self, size): def sendall(self, data): self.sent.append(bytes(data)) + def settimeout(self, timeout): + pass + def process(chunks, encoding='latin-1'): - """Returns the cooked text, the text passed to the callback, and the sent bytes.""" + """Returns the text passed to the callback and the sent bytes.""" tn = Telnet(encoding=encoding, termtype='vt100', termsize=(24, 80)) tn.sock = FakeSock(chunks) received = [] tn.data_callback = received.append - while not tn.eof: - tn.fill_rawq() - tn.process_rawq() - return tn.cookedq.getvalue(), ''.join(received), b''.join(tn.sock.sent) + while tn.receive(None): + pass + return ''.join(received), b''.join(tn.sock.sent) def iac(*codes): @@ -56,9 +58,8 @@ class telnetlibTest(unittest.TestCase): def testData(self): stream = b'a\x00b\x11c' + iac(IAC) + b'd' + iac(NOP) + iac(0) + b'e' - text, received, sent = process([stream]) + text, sent = process([stream]) self.assertEqual(text, 'ab\x11c\xffde') # NUL is removed, XON is kept - self.assertEqual(received, text) self.assertEqual(sent, b'') def testNegotiation(self): @@ -78,7 +79,7 @@ def testNegotiation(self): iac(SB, BINARY, 1) + iac(SE): b'', } for stream, reply in replies.items(): - self.assertEqual(process([b'x' + stream + b'y']), ('xy', 'xy', reply), stream) + self.assertEqual(process([b'x' + stream + b'y']), ('xy', reply), stream) def testSubnegotiation(self): # An escaped IAC inside the subnegotiation does not end it. @@ -86,11 +87,11 @@ def testSubnegotiation(self): self.assertEqual(process([stream])[0], 'x') # An unterminated subnegotiation ends at the next command. stream = iac(SB, TTYPE, 1) + iac(DO, TTYPE) + b'x' - self.assertEqual(process([stream]), ('x', 'x', iac(WILL, TTYPE))) + self.assertEqual(process([stream]), ('x', iac(WILL, TTYPE))) def testIncompleteCommandAtEof(self): for stream in iac(), iac(DO), iac(SB, TTYPE, 1), iac(SB, TTYPE, 1, IAC): - self.assertEqual(process([b'x' + stream]), ('x', 'x', b'')) + self.assertEqual(process([b'x' + stream]), ('x', b'')) def testEveryReadSplit(self): stream = ( @@ -113,8 +114,26 @@ def testSplitDoesNotMatter(self, parts, sizes): def testMultibyteCharacterSplit(self): stream = 'Grüße 20 €'.encode() for cut in range(1, len(stream)): - text, received, _ = process([stream[:cut], stream[cut:]], encoding='utf-8') - self.assertEqual((text, received), ('Grüße 20 €', 'Grüße 20 €'), cut) + text, _ = process([stream[:cut], stream[cut:]], encoding='utf-8') + self.assertEqual(text, 'Grüße 20 €', cut) + + def testReceive(self): + local, remote = socket.socketpair() + self.addCleanup(remote.close) + tn = Telnet(connect_timeout=7) + tn.sock = local + received = [] + tn.data_callback = received.append + self.assertIsNone(tn.receive(0)) # no data yet + self.assertIsNone(tn.receive(0.05)) + remote.sendall(b'hello') + self.assertTrue(tn.receive(5)) + self.assertEqual(received, ['hello']) + self.assertEqual(local.gettimeout(), 7) # restored for sendall() + remote.close() + self.assertFalse(tn.receive(5)) + self.assertFalse(tn.receive(5)) + tn.close() def testWrite(self): tn = Telnet() diff --git a/tests/recordings/linux/alpine-3.22-bash-ssh.golden.json b/tests/recordings/linux/alpine-3.22-bash-ssh.golden.json index 76348ceb..f51d2fb4 100644 --- a/tests/recordings/linux/alpine-3.22-bash-ssh.golden.json +++ b/tests/recordings/linux/alpine-3.22-bash-ssh.golden.json @@ -61,7 +61,7 @@ }, { "command": "exit", - "error": "ProtocolException", + "error": "ConnectionClosedException", "lines": 0, "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "first": "", From b6893dc5e8f3898539733e461c63ba8c94f15c70 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 3 Nov 2026 03:56:11 +0100 Subject: [PATCH 029/250] Settle generic prompts and skip escape sequences A match of the generic prompt pattern counts only if no more data arrives within 20 ms, so route codes like "S>" no longer end the response. Prompts anchored at the end are also tried without terminal escape sequences, which made logins to bash time out. --- CHANGES.md | 10 +++ src/Exscript/protocols/protocol.py | 70 ++++++++++++++--- tests/Exscript/protocols/ExpectTest.py | 78 +++++++++++++++++-- tests/Exscript/protocols/ReplayTest.py | 6 +- .../alpine-3.22-bash-ssh-vt100.golden.json | 69 +++++++++++++++- 5 files changed, 211 insertions(+), 22 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index d2bdf592..e86de855 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -122,6 +122,16 @@ marked with **BREAKING**; a migration guide will accompany the release. a banner such as "Access denied for unauthorized users" failed the login before any credentials were sent. A login error is detected with the next prompt, when the host closes the connection, or at the timeout. +- **Prompts After Escape Sequences:** A prompt after terminal escape sequences is + found now, e.g. after bash's `ESC[?2004h` (bracketed paste mode, with a `termtype` + other than the default `dumb`) or a window title; before, `login()` timed out. + Patterns are matched against the received text first, so patterns that contain + escape sequences match as before. +- **Settle Time for the Generic Prompt:** The prompt pattern of the generic driver, + which the `adtran`, `aix` and `shell` drivers use too, also matches output lines such + as FRR's route codes (`S>`). Its match now counts only if no more data arrives + within 20 ms, so each prompt over these drivers takes 20 ms longer. The prompts of + the other drivers, and prompts set with `set_prompt()`, count at once. ### Telnet diff --git a/src/Exscript/protocols/protocol.py b/src/Exscript/protocols/protocol.py index 223f18cc..a04cf4a5 100644 --- a/src/Exscript/protocols/protocol.py +++ b/src/Exscript/protocols/protocol.py @@ -47,6 +47,7 @@ from ..util.tty import get_terminal_size from ..util.url import _WELL_KNOWN_PORTS from .drivers import Driver, driver_map +from .drivers.driver import _prompt_re as _generic_prompt_re from .exception import ( ConnectionClosedException, DriverReplacedException, @@ -84,6 +85,45 @@ # The other patterns are searched in the data that arrived since the last # search, plus this many characters before it, so a match may span two reads. _LOOKBEHIND = 150 +# The generic prompt pattern also matches output lines such as the "S>" route +# codes of FRR, so its match counts only if no more data follows within this +# many seconds. +_SETTLE_TIME = 0.02 +_generic_prompt = _generic_prompt_re[0] +# Terminal escape sequences around a prompt: CSI (e.g. ESC[?2004h, bash's +# bracketed paste mode), OSC (e.g. the window title), and character sets. +_terminal_escape_re = re.compile( + r'\x1b\[[\x30-\x3f]*[\x20-\x2f]*[\x40-\x7e]|\x1b\][^\x07\x1b]*(?:\x07|\x1b\\)|\x1b[()][\x20-\x7e]' +) + + +def _strip_escapes(text): + """ + Removes terminal escape sequences. Returns the remaining text and, for + each removed sequence, its position in the remaining text and the number + of characters removed up to and including it. + """ + parts, cuts, removed, last = [], [], 0, 0 + for match in _terminal_escape_re.finditer(text): + parts.append(text[last : match.start()]) + cuts.append((match.start() - removed, removed + match.end() - match.start())) + removed += match.end() - match.start() + last = match.end() + parts.append(text[last:]) + return ''.join(parts), cuts + + +def _unstrip(cuts, pos, after): + """ + Maps a position in the text from _strip_escapes() back to the original + text. Sequences at the position itself are skipped if after is true. + """ + removed = 0 + for cut, total in cuts: + if cut > pos or (cut == pos and not after): + break + removed = total + return pos + removed def _ends_with_anchor(regex): @@ -1387,11 +1427,9 @@ def execute(self, command, consume=True): def _domatch(self, prompt, flush): """ Waits until one of the given regular expressions matches the received - data, and sets self.response to the data before the match. With flush, - removes the data up to the end of the match from the buffer. - - Raises TimeoutException if no data arrives for self.timeout seconds, - and ConnectionClosedException if the host closed the connection. + data, and sets self.response to the data before the match. A match of + the generic prompt pattern counts only if no more data arrives within + _SETTLE_TIME seconds. :type prompt: list(re.Pattern) :param prompt: The regular expressions. @@ -1408,9 +1446,11 @@ def _domatch(self, prompt, flush): if self._expect_canceled: self._raise_on_canceled_expect() n, match, start, end = self._search(prompt, searched) - if match is not None: - break searched = self._buffer.size() + if match is not None: + if prompt[n] is not _generic_prompt or not self._fill_buffer(_SETTLE_TIME): + break + continue # more data followed, so it was no prompt received = self._fill_buffer(self.timeout) if received is None: self._raise_on_response_timeout() @@ -1430,8 +1470,9 @@ def _search(self, prompt, searched): Searches the buffer for the first of the given regular expressions that matches. A regular expression that ends with $ or \\Z is searched in the last _PROMPT_WINDOW characters, after the driver's cleanup of - the response; the others in the data from the given position on, plus - _LOOKBEHIND characters before it. + the response, and if none matches, without terminal escape sequences; + the others in the data from the given position on, plus _LOOKBEHIND + characters before it. :type prompt: list(re.Pattern) :param prompt: The regular expressions. @@ -1443,8 +1484,9 @@ def _search(self, prompt, searched): """ size = self._buffer.size() tail = new = None + anchored = [_ends_with_anchor(regex) for regex in prompt] for n, regex in enumerate(prompt): - if _ends_with_anchor(regex): + if anchored[n]: if tail is None: raw = self._buffer.tail(_PROMPT_WINDOW) tail, incomplete = self.get_driver().clean_response_for_re_match(raw) @@ -1459,6 +1501,14 @@ def _search(self, prompt, searched): offset = new_offset if match is not None: return n, match, offset + match.start(), offset + match.end() + # Only then without the escape sequences, because patterns that contain + # escape sequences match only the raw text. + if tail is not None and '\x1b' in tail: + text, cuts = _strip_escapes(tail) + for n, regex in enumerate(prompt): + if anchored[n] and (match := regex.search(text)) is not None: + start = tail_offset + _unstrip(cuts, match.start(), after=False) + return n, match, start, tail_offset + _unstrip(cuts, match.end(), after=True) return -1, None, 0, 0 def _fill_buffer(self, timeout): diff --git a/tests/Exscript/protocols/ExpectTest.py b/tests/Exscript/protocols/ExpectTest.py index 92bc7ca0..9e37202a 100644 --- a/tests/Exscript/protocols/ExpectTest.py +++ b/tests/Exscript/protocols/ExpectTest.py @@ -13,7 +13,13 @@ ProtocolException, TimeoutException, ) -from Exscript.protocols.protocol import Protocol, _ends_with_anchor +from Exscript.protocols.protocol import ( + _SETTLE_TIME, + Protocol, + _ends_with_anchor, + _strip_escapes, + _unstrip, +) from Exscript.servers import SSHd, Telnetd HOST = '127.0.0.1' @@ -23,12 +29,15 @@ class Scripted(Protocol): """ Receives the given chunks, one per read; None is a read that times out. - After the last chunk, the host closes the connection. Records what was sent. + The host pauses before each item, so a read for the settle time gets + nothing; a tuple of chunks arrives without pauses. After the last chunk, + the host closes the connection. Records what was sent. """ def __init__(self, chunks, **kwargs): super().__init__(connect_rate_limit=0, **kwargs) self.chunks = list(chunks) + self.burst = [] self.sent = [] self.timeouts = [] @@ -41,11 +50,16 @@ def _send(self, data): def _fill_buffer(self, timeout): self.timeouts.append(timeout) - if not self.chunks: - return False - chunk = self.chunks.pop(0) - if chunk is None: - return None + if not self.burst: + if not self.chunks: + return False + if timeout == _SETTLE_TIME: + return None + item = self.chunks.pop(0) + if item is None: + return None + self.burst = list(item) if isinstance(item, tuple) else [item] + chunk = self.burst.pop(0) self._receive_cb(chunk) self._buffer.append(chunk) return True @@ -102,6 +116,56 @@ def testEndsWithAnchor(self): self.assertFalse(_ends_with_anchor(re.compile(r'#$', re.M))) # $ ends every line +class PromptTest(unittest.TestCase): + """The settle time of the generic prompt pattern, and escape sequences.""" + + def testPromptLikeOutputLine(self): + # FRR's route code "S>" ends a read, but more output follows at once. + reads = ('show ip route\r\nS>', '* 10.0.0.0/8 via 192.0.2.1\r\nrouter# ') + conn = connect([reads], driver='generic') + conn.expect_prompt() + self.assertEqual(conn.response, 'show ip route\r\nS>* 10.0.0.0/8 via 192.0.2.1') + + def testPromptBeforeAPause(self): + conn = connect(['show version\r\nrouter# ', 'later'], driver='generic') + conn.expect_prompt() + self.assertEqual(conn.response, 'show version') + self.assertEqual(conn.timeouts, [conn.timeout, _SETTLE_TIME]) + self.assertEqual(conn.chunks, ['later']) + + def testKnownDriverPromptCountsAtOnce(self): + conn = connect(['show version\r\nrouter#'], driver='ios') + conn.expect_prompt() + self.assertEqual(conn.timeouts, [conn.timeout]) + + def testPromptAfterBracketedPasteMode(self): + conn = connect(['ls\r\nfile\r\n\x1b[?2004hrouter1:~$ '], driver='generic') + conn.expect_prompt() + self.assertEqual(conn.response, 'ls\r\nfile') + self.assertEqual(conn._buffer.size(), 0) + + def testPromptAfterWindowTitle(self): + conn = connect(['\r\n\x1b]0;admin@router1: ~\x07admin@router1:~$ \x1b(B'], driver='generic') + conn.expect_prompt() + self.assertEqual(conn.response, '') + self.assertEqual(conn._buffer.size(), 0) + + def testPatternWithEscapeSequences(self): + # Matched against the raw data first. + conn = connect(['out\r\n\x1b[32muser\x1b[0m$ ']) + conn.set_prompt(r'\x1b\[32m\w+\x1b\[0m\$ $') + conn.expect_prompt() + self.assertEqual(conn.response, 'out\r\n') + + def testUnstrip(self): + text, cuts = _strip_escapes('a\x1b[1mb\x1b[0m\x1b[K') + self.assertEqual(text, 'ab') + self.assertEqual(_unstrip(cuts, 1, after=False), 1) + self.assertEqual(_unstrip(cuts, 1, after=True), 5) + self.assertEqual(_unstrip(cuts, 2, after=False), 6) + self.assertEqual(_unstrip(cuts, 2, after=True), 13) + + class LoginErrorTest(unittest.TestCase): """A login error counts only if the host sent it after the last send().""" diff --git a/tests/Exscript/protocols/ReplayTest.py b/tests/Exscript/protocols/ReplayTest.py index eb55591d..b31ef004 100644 --- a/tests/Exscript/protocols/ReplayTest.py +++ b/tests/Exscript/protocols/ReplayTest.py @@ -23,13 +23,13 @@ # Recordings that Exscript does not handle like the recorded device yet, and why. KNOWN_ISSUES = { 'linux/alpine-3.22-bash-ssh': 'bash prompts are not guessed as shell', - 'linux/alpine-3.22-bash-ssh-vt100': 'escape sequences before the prompt make login() time out', + 'linux/alpine-3.22-bash-ssh-vt100': 'bash prompts are not guessed as shell', 'linux/busybox-1.37-ash-telnet': '"router3 login:" is guessed as junos, whose prompt does not match', 'netopeer2/netopeer2-netconf': 'the unanchored isam prompt regex matches NETCONF replies', } -# Recordings whose results depend on read timing until the issue is fixed; their tests are skipped. +# Recordings whose results depend on the read sizes until the issue is fixed; their tests are skipped. TIMING_ISSUES = { - 'frr/frr-10.4-vtysh-ssh': 'route codes like "S>" at the end of a read match the prompt', + 'frr/frr-10.4-vtysh-ssh': 'guessed as isam, whose unanchored prompt pattern matches route codes like "S>"', } _message_id = re.compile(r'message-id="[^"]*"') _results = {} diff --git a/tests/recordings/linux/alpine-3.22-bash-ssh-vt100.golden.json b/tests/recordings/linux/alpine-3.22-bash-ssh-vt100.golden.json index 5976248f..ad6cbcfa 100644 --- a/tests/recordings/linux/alpine-3.22-bash-ssh-vt100.golden.json +++ b/tests/recordings/linux/alpine-3.22-bash-ssh-vt100.golden.json @@ -1,6 +1,71 @@ { - "login": "TimeoutException", + "login": "ok", "driver": "generic", "os": "unknown", - "commands": [] + "commands": [ + { + "command": "uname -sr", + "error": null, + "lines": 3, + "sha256": "34f8f2e43f3d72cd066d0d41cd9acbd097fa410f53f516986b642547c384d3dc", + "first": "uname -sr", + "last": "Linux 6.18.54.1-microsoft-standard-WSL2" + }, + { + "command": "cat /etc/os-release", + "error": null, + "lines": 8, + "sha256": "6d8ff4dc6995d994b946ebc2a67aceb2ccbe103ef9c84f13b0270d02dfc5493a", + "first": "cat /etc/os-release", + "last": "BUG_REPORT_URL=\"https://gitlab.alpinelinux.org/alpine/aports/-/issues\"" + }, + { + "command": "ls -l --color=always /", + "error": null, + "lines": 20, + "sha256": "1df8396c9ed91b04d313d83680f5873732d55abcfbc14b22b8f5405348a41e06", + "first": "ls -l --color=always /", + "last": "drwxr-xr-x 11 root root 4096 Oct 21 19:59 \u001b[1;34mvar" + }, + { + "command": "printf 'umlauts: \\303\\244\\303\\266\\303\\274, euro sign: \\342\\202\\254\\n'", + "error": null, + "lines": 4, + "sha256": "001ec9ad040d15730b091884100886f40f6b73ecfbe4589e3910c2244556aac3", + "first": "printf 'umlauts: \\303\\244\\303\\266\\303\\274, euro sign: \\342\\202\\254\\n' ", + "last": "umlauts: \u00c3\u00a4\u00c3\u00b6\u00c3\u00bc, euro sign: \u00e2\u0082\u00ac" + }, + { + "command": "echo the quick brown fox jumps over the lazy dog, the quick brown fox jumps over the lazy dog, and again", + "error": null, + "lines": 3, + "sha256": "8d09d26cd1c2af3ee2b83e9f0f136b8dc95edc9932a8d86859a1c4e02bad7b5b", + "first": "echo the quick brown fox jumps over the lazy dog, the quick brown fox jumps over the lazy dog, and again", + "last": "the quick brown fox jumps over the lazy dog, the quick brown fox jumps over the lazy dog, and again" + }, + { + "command": "seq 1 20000", + "error": null, + "lines": 20002, + "sha256": "bfb51050cec8f4a1a99e80b82e34436879b3a5e107ecee48c27df96fff2cea8d", + "first": "seq 1 20000", + "last": "20000" + }, + { + "command": "ls /nonexistent", + "error": null, + "lines": 3, + "sha256": "be50c8167af6e8a99d8e58d29d7e11779233e693421adac99a08a9d2c479a8ff", + "first": "ls /nonexistent", + "last": "ls: /nonexistent: No such file or directory" + }, + { + "command": "exit", + "error": "ConnectionClosedException", + "lines": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "first": "", + "last": "" + } + ] } From 171fcd4e4b6a214417f65610889d815db61afd3d Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 3 Nov 2026 04:08:03 +0100 Subject: [PATCH 030/250] Speed up the isam prompt pattern The OS guesser runs it on all received data, and its character runs backtracked on every line; possessive quantifiers match the same strings. execute() with a 4 MiB response takes 0.28 s of CPU time instead of 0.62 s. --- CHANGES.md | 3 ++- src/Exscript/protocols/drivers/isam.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index e86de855..993e5bb2 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -168,7 +168,8 @@ marked with **BREAKING**; a migration guide will accompany the release. prompt: a line with a run of 100 word characters took 1.3 seconds to check, 200 characters 25 seconds, so hex or base64 output could stall a session. The pattern matches the same strings, verified against the old one, in linear time. A test - checks every driver pattern for this. + checks every driver pattern for this. The isam driver's prompt pattern, which the + OS guesser runs on all received data, matches the same strings 3.5 times faster. - **BREAKING - Graceful Logout:** `Protocol.close(force=False)` now calls the driver's new `logout()` method before the connection is closed. The base `Driver` sends diff --git a/src/Exscript/protocols/drivers/isam.py b/src/Exscript/protocols/drivers/isam.py index e927d5ca..4c86c8ca 100644 --- a/src/Exscript/protocols/drivers/isam.py +++ b/src/Exscript/protocols/drivers/isam.py @@ -30,7 +30,7 @@ _user_re = [re.compile(r'login: ?', re.I)] _password_re = [re.compile(r'(?:\r\n|[\r\n])password: ?', re.I)] -_prompt_re = [re.compile(r'(?:\r\n|[\r\n])[\- +\d+\w+\.]+(?:\([^\)]+\))?[>#] ?')] +_prompt_re = [re.compile(r'(?:\r\n|[\r\n])[\- +\d+\w+\.]++(?:\([^\)]++\))?[>#] ?')] _error_re = [ re.compile(r'%Error'), re.compile(r'invalid token', re.I), From 99ded2e28822eadc15455c8b412bb27cd07ff8ce Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 3 Nov 2026 04:21:45 +0100 Subject: [PATCH 031/250] Fix monitors after expect() and stop copying the buffer pop() did not move the position of a monitor's last match, so after expect() a monitor missed matches such as the next "--More--". append() copied the whole buffer on every read; it now copies only the end that it searches. --- CHANGES.md | 5 +++++ src/Exscript/protocols/protocol.py | 2 ++ src/Exscript/util/buffer.py | 20 +++++++++++++++----- tests/Exscript/util/bufferTest.py | 17 +++++++++++++++++ 4 files changed, 39 insertions(+), 5 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 993e5bb2..7dcfc69e 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -132,6 +132,11 @@ marked with **BREAKING**; a migration guide will accompany the release. as FRR's route codes (`S>`). Its match now counts only if no more data arrives within 20 ms, so each prompt over these drivers takes 20 ms longer. The prompts of the other drivers, and prompts set with `set_prompt()`, count at once. +- **Monitors:** The callbacks of `add_monitor()` no longer miss matches after + `expect()` consumed the data before them. Monitors search only the end of the + received data instead of a copy of all of it for each read: 8 MiB of output in + 1 KiB reads take 0.015 seconds instead of 9.4. The positions in the match object + that the callback gets refer to the searched text, not to the whole buffer. ### Telnet diff --git a/src/Exscript/protocols/protocol.py b/src/Exscript/protocols/protocol.py index a04cf4a5..59fd38d3 100644 --- a/src/Exscript/protocols/protocol.py +++ b/src/Exscript/protocols/protocol.py @@ -1668,6 +1668,8 @@ def add_monitor(self, pattern, callback, limit=80): Arguments passed to the callback are the protocol instance, the index of the match, and the match object of the regular expression. + The positions in the match object refer to the searched end of the + buffer, not to the whole buffer. :type pattern: str|re.RegexObject|list(str|re.RegexObject) :param pattern: One or more regular expressions. diff --git a/src/Exscript/util/buffer.py b/src/Exscript/util/buffer.py index 129e6d27..1d683d99 100644 --- a/src/Exscript/util/buffer.py +++ b/src/Exscript/util/buffer.py @@ -28,6 +28,10 @@ from .cast import to_regexs +# Monitors search only the tail of the buffer, but still see this many +# characters before it, e.g. for \b or a lookbehind. +_CONTEXT = 256 + class MonitoredBuffer: """ @@ -104,6 +108,8 @@ def pop(self, bytes): self.io.seek(0) self.io.write(tail) self.io.truncate() + for item in self.monitors: + item[2] = max(0, item[2] - len(head)) return head def append(self, data): @@ -122,14 +128,17 @@ def append(self, data): # If it does, we need to disable that monitor until the matching # data is no longer in the buffer. We accomplish this by keeping # track of the position of the last matching byte. - buf = str(self) + size = self.size() + start = min(max(item[2], size - item[3]) for item in self.monitors) + offset = max(0, start - _CONTEXT) + text = self.tail(size - offset) for item in self.monitors: regex_list, callback, bytepos, limit = item - bytepos = max(bytepos, len(buf) - limit) + bytepos = max(bytepos, size - limit) for i, regex in enumerate(regex_list): - match = regex.search(buf, bytepos) + match = regex.search(text, bytepos - offset) if match is not None: - item[2] = match.end() + item[2] = offset + match.end() callback(i, match) def clear(self): @@ -147,7 +156,8 @@ def add_monitor(self, pattern, callback, limit=80): buffer. Arguments passed to the callback are the index of the match, and - the match object of the regular expression. + the match object of the regular expression. The positions in the + match object refer to the searched end of the buffer. :type pattern: str|re.RegexObject|list(str|re.RegexObject) :param pattern: One or more regular expressions. diff --git a/tests/Exscript/util/bufferTest.py b/tests/Exscript/util/bufferTest.py index 6c4ff3fe..4e130846 100644 --- a/tests/Exscript/util/bufferTest.py +++ b/tests/Exscript/util/bufferTest.py @@ -123,6 +123,23 @@ def monitor_cb(thedata, *args, **kwargs): self.assertEqual(data.get('args')[1].group(0), 'abc') self.assertEqual(data.get('kwargs'), {}) + def testMonitorAfterPop(self): + b = MonitoredBuffer() + matches = [] + b.add_monitor('abc', lambda index, match: matches.append(match.group())) + b.append('xxabc') + b.pop(5) # e.g. expect() consumed the data + b.append('abc') + self.assertEqual(matches, ['abc', 'abc']) + + def testMonitorSeesContextBeforeTheTail(self): + b = MonitoredBuffer() + matches = [] + b.add_monitor(r'(?<=foo)bar\b', lambda index, match: matches.append(match.group()), limit=3) + b.append('x' * 1000 + 'foo') + b.append('bar') + self.assertEqual(matches, ['bar']) + def suite(): return unittest.TestLoader().loadTestsFromTestCase(bufferTest) From 56255d8a497848eaf49bde772d11a8e401be314d Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 3 Nov 2026 04:21:45 +0100 Subject: [PATCH 032/250] Reassemble NETCONF chunks in linear time reassemble_chunks() copied the rest of the reply for every chunk; it now slices by index and joins once. A 16 MiB reply in 4 KiB chunks takes 0.013 s instead of 5.2 s. --- CHANGES.md | 2 ++ src/Exscript/protocols/netconf.py | 13 +++++++------ tests/Exscript/protocols/NetconfTest.py | 22 ++++++++++++++++++++++ 3 files changed, 31 insertions(+), 6 deletions(-) create mode 100644 tests/Exscript/protocols/NetconfTest.py diff --git a/CHANGES.md b/CHANGES.md index 7dcfc69e..9ad8ae1e 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -137,6 +137,8 @@ marked with **BREAKING**; a migration guide will accompany the release. received data instead of a copy of all of it for each read: 8 MiB of output in 1 KiB reads take 0.015 seconds instead of 9.4. The positions in the match object that the callback gets refer to the searched text, not to the whole buffer. +- **NETCONF 1.1 Chunks:** `Netconf.reassemble_chunks()` takes linear time: a 16 MiB + reply in 4 KiB chunks takes 0.013 seconds instead of 5.2. ### Telnet diff --git a/src/Exscript/protocols/netconf.py b/src/Exscript/protocols/netconf.py index 3a5f977e..078cf7f8 100644 --- a/src/Exscript/protocols/netconf.py +++ b/src/Exscript/protocols/netconf.py @@ -336,13 +336,14 @@ def reassemble_chunks(response): match = _chunk_header_re.match(response) if not match: return response - assembled = '' + parts = [] while match: - size = int(match.group(1)) - response = response[match.end() :] # consume the chunk header - assembled += response[:size] # extract exactly size chars of content - response = response[size:] # consume the chunk content - match = _chunk_header_re.match(response) + start = match.end() # after the chunk header + end = start + int(match.group(1)) # exactly size chars of content + parts.append(response[start:end]) + match = _chunk_header_re.match(response, end) + assembled = ''.join(parts) + response = response[end:] if response: # Internally this would be already consumed as the prompt. match = _end_of_chunks_re.match(response) diff --git a/tests/Exscript/protocols/NetconfTest.py b/tests/Exscript/protocols/NetconfTest.py new file mode 100644 index 00000000..99540b8e --- /dev/null +++ b/tests/Exscript/protocols/NetconfTest.py @@ -0,0 +1,22 @@ +"""Tests for the parts of the NETCONF adapter that need no server.""" + +import unittest + +from Exscript.protocols.netconf import Netconf + + +class ReassembleChunksTest(unittest.TestCase): + def testChunks(self): + # The second chunk contains what looks like a chunk header. + self.assertEqual(Netconf.reassemble_chunks('\n#5\n\n#3\n\n#1\n##\n'), '\n#1') + + def testWithoutChunks(self): + self.assertEqual(Netconf.reassemble_chunks(''), '') + + def testTrailingBytes(self): + self.assertRaises(ValueError, Netconf.reassemble_chunks, '\n#2\nok\n##\nx') + + def testManyChunks(self): + chunk = 'x' * 4096 + framed = ''.join('\n#4096\n' + chunk for _ in range(1024)) + '\n##\n' + self.assertEqual(Netconf.reassemble_chunks(framed), chunk * 1024) From 99f82a222cc167647389f14101fc986ae5aeb1f8 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 3 Nov 2026 04:43:27 +0100 Subject: [PATCH 033/250] Space connects per host and reconnect only after failed attempts The rate limiter slept while it held a process-wide lock, which blocked the connects to all other hosts. The ios driver reconnected before every SSH authentication method, also before those that send nothing without keys. --- CHANGES.md | 12 ++++++ src/Exscript/protocols/protocol.py | 33 ++++++++--------- src/Exscript/protocols/ssh2.py | 17 ++++++--- tests/Exscript/KnownIssuesTest.py | 40 +------------------- tests/Exscript/protocols/ProtocolTest.py | 47 +++++++++++++++++++++++- tests/Exscript/protocols/SSH2Test.py | 44 ++++++++++++++++++++-- 6 files changed, 128 insertions(+), 65 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 9ad8ae1e..7373c5d3 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -97,6 +97,18 @@ marked with **BREAKING**; a migration guide will accompany the release. - `Exscript.util.url.Url.from_string(url, default_protocol='ssh')` - `exscript` CLI: `-p`/`--protocol` +### Connecting + +- **Connect Rate Limit Per Host:** `connect_rate_limit` no longer delays connects to + other hosts. The limiter slept while it held a process-wide lock, so one waiting + connect blocked all others, by default for up to 1.1 seconds. Concurrent connects + to the same host are now spaced, too; before, they could start together. +- **Fewer SSH Reconnects:** Drivers that reconnect between authentication methods + (`ios`) now reconnect only after a method sent a request that failed. Before, they + reconnected before every method, also before the public key methods when there was + no key: a password login to an IOS device that also offers public keys and + keyboard-interactive took 5 SSH connections, now 3. + ### Waiting for Responses - **One Implementation for All Protocols:** `expect()`, `waitfor()`, `expect_prompt()` diff --git a/src/Exscript/protocols/protocol.py b/src/Exscript/protocols/protocol.py index 59fd38d3..44f747ac 100644 --- a/src/Exscript/protocols/protocol.py +++ b/src/Exscript/protocols/protocol.py @@ -942,18 +942,16 @@ def guess_os(self): def _enforce_connect_rate_limit(self, rate_limit=0): """ - Enforce a connection rate-limit to prevent remote connection resets. - The limit represents the maximum number of connects per minute. However, - an offset of 0.1s is added per login to avoid exactly hitting the limit. - Uses `self.connect_rate_limit`, if the given rate-limit is zero. + Waits until the connect keeps the rate limit: connects to the same + host, also concurrent ones, are spaced by 60 / rate_limit + 0.1 + seconds, while connects to other hosts are not delayed. :type rate_limit: int :param rate_limit: The connection rate-limit (connects per minute); 0 uses connect_rate_limit. """ key = self.host.lower() - wait = 0.0 - offset = 0.1 + interval = 0.0 precision = 3 if not rate_limit: @@ -961,19 +959,20 @@ def _enforce_connect_rate_limit(self, rate_limit=0): if rate_limit > 0: self._dbg(1, "Connection rate limit: %s per minute" % rate_limit) - wait = round((60 / rate_limit) + offset, precision) - self._dbg(2, "Maximum connection delay: %s seconds" % wait) + interval = round((60 / rate_limit) + 0.1, precision) + self._dbg(2, "Maximum connection delay: %s seconds" % interval) + # Reserve the next free time for this host under the lock, but wait + # outside of it. with self._connection_monitor_lock: - now = time.time() - if wait: - # Calculate the remaining wait time since last connection attempt. - last = self._connection_monitor.get(key, 0.0) - wait = round(wait - min((now - last), wait), precision) - if wait: - self._dbg(2, "Remaining connection delay: %s seconds" % wait) - time.sleep(wait) - self._connection_monitor[key] = now + now = time.monotonic() + last = self._connection_monitor.get(key) + start = now if last is None or not interval else max(now, last + interval) + self._connection_monitor[key] = start if last is None else max(start, last) + delay = round(start - now, precision) + if delay > 0: + self._dbg(2, "Remaining connection delay: %s seconds" % delay) + time.sleep(delay) def _connect(self): """ diff --git a/src/Exscript/protocols/ssh2.py b/src/Exscript/protocols/ssh2.py index f495616c..eea412b1 100644 --- a/src/Exscript/protocols/ssh2.py +++ b/src/Exscript/protocols/ssh2.py @@ -336,12 +336,14 @@ def _paramiko_auth(self, username, password): # Finally try all supported login methods. errors = [] + attempted = True # auth_none() above was an attempt for method in auth_methods: - # Some OSes (e.g. JunOS ERX OS, Huawei) do not accept further login - # attempts after failing one. So in this hack, we - # re-connect after each attempt... - if self.get_driver().reconnect_between_auth_methods or not self._transport.active: - if self.get_driver().reconnect_between_auth_methods: + # Some devices (e.g. Cisco WS-C3750 with IOS) do not accept further + # login attempts after failing one. So in this hack, we re-connect + # after each attempt that reached the server, auth_none() included. + reconnect = self.get_driver().reconnect_between_auth_methods and attempted + if reconnect or not self._transport.active: + if reconnect: self._dbg(1, 'Forced reconnect by driver.') else: self._dbg(1, 'Forced reconnect by inactive client.') @@ -349,8 +351,12 @@ def _paramiko_auth(self, username, password): # Some devices (e.g. Cisco ASR9010) seem to have tight rate limits # for reconnects, so connect() waits some time before the reconnect. self.connect(self.host, self.port) + attempted = False self._dbg(2, 'Authenticating with %s().' % method.__name__) + # paramiko creates a new auth handler for each request that it sends; + # without keys, the public key methods send none. + handler = self._transport.auth_handler try: method(username, password) return @@ -366,6 +372,7 @@ def _paramiko_auth(self, username, password): msg = 'SSHException with %s(): %s' % (method.__name__, str(e)) self._dbg(1, msg) errors.append(msg) + attempted = attempted or self._transport.auth_handler is not handler raise LoginFailure('Login failed: ' + '; '.join(errors)) def _paramiko_channel(self): diff --git a/tests/Exscript/KnownIssuesTest.py b/tests/Exscript/KnownIssuesTest.py index b4b7198a..6a7c5f77 100644 --- a/tests/Exscript/KnownIssuesTest.py +++ b/tests/Exscript/KnownIssuesTest.py @@ -8,14 +8,11 @@ """ import sys -import threading -import time import unittest -import uuid from Exscript import Account from Exscript.emulators import VirtualDevice -from Exscript.protocols import SSH2, Dummy +from Exscript.protocols import Dummy from Exscript.protocols.drivers import driver_map, isam @@ -41,41 +38,6 @@ def testLastLoginLineDoesNotBreakLogin(self): self.assertEqual(conn.get_response().strip(), 'Linux') -class ConnectRateLimitTest(unittest.TestCase): - def enforce(self, host, starts=None): - conn = SSH2(connect_rate_limit=60) # 1.1 s spacing per host - conn.host = host - conn._enforce_connect_rate_limit() - if starts is not None: - starts.append(time.monotonic()) - - @unittest.expectedFailure # the limiter sleeps while holding a process-wide lock - def testOtherHostsAreNotDelayed(self): - host = uuid.uuid4().hex - self.enforce(host + '-a') - waiting = threading.Thread(target=self.enforce, args=(host + '-a',)) - waiting.start() - time.sleep(0.05) - start = time.monotonic() - self.enforce(host + '-b') - elapsed = time.monotonic() - start - waiting.join() - self.assertLess(elapsed, 0.5) - - @unittest.expectedFailure # concurrent connects to one host may start together - def testConcurrentConnectsToOneHostAreSpaced(self): - host = uuid.uuid4().hex - starts = [] - self.enforce(host, starts) - threads = [threading.Thread(target=self.enforce, args=(host, starts)) for _ in range(2)] - for thread in threads: - thread.start() - for thread in threads: - thread.join() - starts.sort() - self.assertGreaterEqual(min(b - a for a, b in zip(starts, starts[1:])), 1.0) - - def suite(): return unittest.TestLoader().loadTestsFromModule(sys.modules[__name__]) diff --git a/tests/Exscript/protocols/ProtocolTest.py b/tests/Exscript/protocols/ProtocolTest.py index 84f9f1eb..ba84764e 100644 --- a/tests/Exscript/protocols/ProtocolTest.py +++ b/tests/Exscript/protocols/ProtocolTest.py @@ -1,9 +1,11 @@ import os import re import tempfile +import threading import time import unittest import unittest.mock +import uuid from functools import partial from Exscript import Account, PrivateKey @@ -905,8 +907,51 @@ def testReconnect(self): self.assertGreater(second_log_size, first_log_size) +class ConnectRateLimitTest(unittest.TestCase): + def enforce(self, host, rate_limit=60): # 60: 1.1 s spacing per host + conn = Protocol(connect_rate_limit=rate_limit) + conn.host = host + conn._enforce_connect_rate_limit() + + def testOtherHostsAreNotDelayed(self): + host = uuid.uuid4().hex + self.enforce(host + '-a') + waiting = threading.Thread(target=self.enforce, args=(host + '-a',)) + waiting.start() + time.sleep(0.05) + start = time.monotonic() + self.enforce(host + '-b') + elapsed = time.monotonic() - start + waiting.join() + self.assertLess(elapsed, 0.5) + + def testConnectsToOneHostAreSpaced(self): + host = uuid.uuid4().hex + with unittest.mock.patch('Exscript.protocols.protocol.time.sleep') as sleep: + for _ in range(3): + self.enforce(host) + delays = [call.args[0] for call in sleep.call_args_list] + self.assertEqual(len(delays), 2) + self.assertAlmostEqual(delays[0], 1.1, delta=0.05) + self.assertAlmostEqual(delays[1], 2.2, delta=0.05) # after the slot reserved for the second + + def testNoLimitDoesNotWait(self): + host = uuid.uuid4().hex + with unittest.mock.patch('Exscript.protocols.protocol.time.sleep') as sleep: + self.enforce(host) + self.enforce(host) # reserves a slot 1.1 s ahead + self.enforce(host, rate_limit=0) + self.assertEqual(sleep.call_count, 1) + + def suite(): - return unittest.TestLoader().loadTestsFromTestCase(ProtocolTest) + loader = unittest.TestLoader() + return unittest.TestSuite( + [ + loader.loadTestsFromTestCase(ProtocolTest), + loader.loadTestsFromTestCase(ConnectRateLimitTest), + ] + ) if __name__ == '__main__': diff --git a/tests/Exscript/protocols/SSH2Test.py b/tests/Exscript/protocols/SSH2Test.py index ae3cddea..78e54f5f 100644 --- a/tests/Exscript/protocols/SSH2Test.py +++ b/tests/Exscript/protocols/SSH2Test.py @@ -45,10 +45,11 @@ def testAuthenticate(self): class KeyServer: - """Completes the SSH key exchange with its host keys; nothing else.""" + """Completes the SSH key exchange with its host keys; nothing else unless the interface allows it.""" - def __init__(self, test, *keys): + def __init__(self, test, *keys, interface=paramiko.ServerInterface): self.keys = keys + self.interface = interface self._transports = [] self._closing = threading.Event() self._listener = socket.create_server((HOST, 0)) @@ -70,7 +71,7 @@ def _accept(self): self._transports.append(transport) # With an event, the key exchange runs in the transport's thread, and the # transport stays open until the client or close() closes it. - transport.start_server(event=threading.Event(), server=paramiko.ServerInterface()) + transport.start_server(event=threading.Event(), server=self.interface()) def close(self): self._closing.set() @@ -173,12 +174,49 @@ def testInvalidPolicy(self): self.assertRaises(ValueError, SSH2, host_key_policy='yes') +class _PasswordOnly(paramiko.ServerInterface): + """Offers three methods, but accepts only the password.""" + + def get_allowed_auths(self, username): + return 'publickey,keyboard-interactive,password' + + def check_auth_password(self, username, password): + return paramiko.AUTH_SUCCESSFUL + + +class ReconnectTest(unittest.TestCase): + """Drivers like ios reconnect between methods, but only after a request that failed.""" + + def handshakes(self, driver): + server = KeyServer(self, paramiko.RSAKey(filename=keyfile), interface=_PasswordOnly) + conn = SSH2(timeout=1, connect_rate_limit=0, verify_fingerprint=False, driver=driver) + self.addCleanup(conn.close, force=True) + home = tempfile.TemporaryDirectory() # no key files + self.addCleanup(home.cleanup) + with ( + mock.patch.dict(os.environ, {'HOME': home.name, 'USERPROFILE': home.name}), + mock.patch('paramiko.Agent') as agent, + ): + agent.return_value.get_keys.return_value = [] + conn.connect(HOST, server.port) + conn._paramiko_auth('admin', 'password') + return len(server._transports) + + def testReconnectAfterFailedRequests(self): + # auth_none(), then keyboard-interactive fail; the public key methods send nothing. + self.assertEqual(self.handshakes('ios'), 3) + + def testNoReconnectByDefault(self): + self.assertEqual(self.handshakes('generic'), 1) + + def suite(): loader = unittest.TestLoader() return unittest.TestSuite( [ loader.loadTestsFromTestCase(SSH2Test), loader.loadTestsFromTestCase(HostKeyTest), + loader.loadTestsFromTestCase(ReconnectTest), ] ) From 2d4af862ca797c9b7beb0789e5728b3c02e25ae2 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 3 Nov 2026 05:07:28 +0100 Subject: [PATCH 034/250] Close reliably, try all SSH addresses and fix interact() close() left the connection open when the logout failed; it now always closes and raises the logout's error afterwards. interact() reads and sends through the adapter, so Telnet commands and multibyte characters work, and an OSError in select() no longer raises TypeError. --- CHANGES.md | 14 ++++ src/Exscript/protocols/protocol.py | 102 +++++++++++-------------- src/Exscript/protocols/ssh2.py | 17 +---- tests/Exscript/protocols/ExpectTest.py | 94 +++++++++++++++++++---- tests/Exscript/protocols/SSH2Test.py | 15 ++++ 5 files changed, 156 insertions(+), 86 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 7373c5d3..dc4f545c 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -108,6 +108,20 @@ marked with **BREAKING**; a migration guide will accompany the release. reconnected before every method, also before the public key methods when there was no key: a password login to an IOS device that also offers public keys and keyboard-interactive took 5 SSH connections, now 3. +- **All Addresses for SSH:** SSH and NETCONF connections try all addresses of the + host, e.g. IPv4 if IPv6 fails, as Telnet does; before, they tried only the first. +- **Reliable `close()`:** `close()` closes the connection and resets the object even + if the logout fails, and raises the error of the logout afterwards. Errors while + closing, e.g. because the host closed the connection first, are only logged; + before, `close(force=True)` could raise an `EOFError`. + +### Interactive Shell + +- **`interact()` Fixes:** Typed characters of several bytes, e.g. umlauts on a UTF-8 + terminal, no longer crash the session. Telnet commands from the host are processed + instead of shown, and typed text is sent like with `send()`, so Telnet escapes the + byte 255. The previous `SIGWINCH` handler is restored when the session ends. On + Windows, the thread that receives no longer keeps the process alive. ### Waiting for Responses diff --git a/src/Exscript/protocols/protocol.py b/src/Exscript/protocols/protocol.py index 44f747ac..3bc52aa2 100644 --- a/src/Exscript/protocols/protocol.py +++ b/src/Exscript/protocols/protocol.py @@ -24,14 +24,13 @@ An abstract base class for all protocols. """ +import codecs import contextlib -import errno import logging import os import re import select import signal -import socket import sys import threading import time @@ -145,12 +144,10 @@ def _ends_with_anchor(regex): _connection_monitor: dict[str, float] = {} _connection_monitor_lock = threading.RLock() -# NOTE: Encoding/Decoding non-ASCII byte-streams might not work properly. -# Right now the default encoding is latin-1 and most devices stick to 7 bit -# ASCII, even in Netconf XML/RPC (usually UTF-8). ASCII is a subset of UTF-8. -# This will start to break if a device sends or expects true UTF-8 encoded -# data, which is not ASCII. Then byte-stream decoding of stdin and the channel -# should be modernized to use an incremental decoder (see "codecs" package). +# NOTE: The default encoding is latin-1: it decodes any byte, and most devices +# send 7-bit ASCII, even in Netconf XML/RPC (usually UTF-8). For devices that +# send UTF-8, pass encoding='utf-8'; the adapters decode incrementally, so a +# character split across two reads is decoded correctly. class Protocol: @@ -1747,6 +1744,8 @@ def _set_terminal_size(self, rows, cols): raise NotImplementedError() def _open_posix_shell(self, channel, key_handlers, handle_window_size): + # Typed characters may take several bytes. + decoder = codecs.getincrementaldecoder(sys.stdin.encoding or 'utf-8')(errors='replace') # We need to make sure to use an unbuffered stdin, else multi-byte # chars (such as arrow keys) won't work properly. with os.fdopen(sys.stdin.fileno(), 'rb', 0) as stdin: @@ -1760,41 +1759,28 @@ def handle_sigwinch(signum, frame): self._dbg(2, "Trying to update terminal size: %dx%d" % (rows, cols)) self._set_terminal_size(rows, cols) - signal.signal(signal.SIGWINCH, handle_sigwinch) + old_handler = signal.signal(signal.SIGWINCH, handle_sigwinch) handle_sigwinch(None, None) # Read from stdin and write to the network, endlessly. try: tty.setraw(sys.stdin.fileno()) tty.setcbreak(sys.stdin.fileno()) - channel.settimeout(0.0) while True: - try: - r, w, e = select.select([channel, stdin], [], []) - except OSError as e: - code, message = e - if code == errno.EINTR: - # This may happen when SIGWINCH is called - # during the select; we just retry then. - continue - raise - - if channel in r: - try: - data = channel.recv(1024).decode(self.encoding) - except TimeoutError: - pass - if not data: - self._dbg(1, 'EOF from remote host') - break - self._receive_cb(data) - self._buffer.append(data) + # select() retries by itself if SIGWINCH interrupts it. + r, w, e = select.select([channel, stdin], [], []) + if channel in r and self._fill_buffer(0) is False: + self._dbg(1, 'EOF from remote host') + break if stdin in r: - data = stdin.read(1).decode(sys.stdin.encoding) + byte = stdin.read(1) self._buffer.clear() - if len(data) == 0: + if not byte: break + data = decoder.decode(byte) + if not data: + continue # more bytes of the character follow # Temporarily revert stdin behavior while callbacks are # active. @@ -1804,33 +1790,25 @@ def handle_sigwinch(signum, frame): termios.tcsetattr(stdin, termios.TCSADRAIN, curtty) if not is_handled: - channel.send(data.encode(self.encoding)) - self._send_cb(data) + self._send(data) if self.send_data is not None: self.send_data.write(data) finally: termios.tcsetattr(stdin, termios.TCSADRAIN, oldtty) + if handle_window_size: + signal.signal(signal.SIGWINCH, old_handler or signal.SIG_DFL) def _open_windows_shell(self, channel, key_handlers, handle_window_size): - import threading - - def writeall(sock): - while True: - # NOTE: Windows stdin is always cooked, remote may send double-echos. - # With remote PTY/Shell connections we might see our echos from - # our cooked stdin and also from the remote channel. Needs to be - # analyzed on next possible testing with a windows box. - data = sock.recv(1024).decode(self.encoding) - if not data: - self._dbg(1, 'EOF from remote host') - break - self._receive_cb(data) - self._buffer.append(data) + def receive(): + # NOTE: Windows stdin is always cooked, remote may send double-echos. + # With remote PTY/Shell connections we might see our echos from + # our cooked stdin and also from the remote channel. Needs to be + # analyzed on next possible testing with a windows box. + while self._fill_buffer(None) is not False: + pass + self._dbg(1, 'EOF from remote host') - # NOTE: Thread in windows might need a daemon=True to avoid hanging on exit. - # Needs to be analyzed on next possible testing with a windows box. - writer = threading.Thread(name="channel.recv", target=writeall, args=(channel,)) - writer.start() + threading.Thread(name="channel.recv", target=receive, daemon=True).start() try: while True: @@ -1843,8 +1821,7 @@ def writeall(sock): if not data: break if not self._call_key_handlers(key_handlers, data): - channel.send(data.encode(self.encoding)) - self._send_cb(data) + self._send(data) if self.send_data is not None: self.send_data.write(data) except EOFError: @@ -2030,7 +2007,9 @@ def close(self, force=False): With `force=False` the driver-specific logout() method is called before the connection is closed. Dummy and Netconf adapters have a special - static logout sequence. + static logout sequence. The connection is closed even if the logout + fails, and the error of the logout is raised afterwards; errors while + closing are only logged, because the host may have closed first. .. WARNING:: Ensure the remote host has already left any previously used submode @@ -2042,8 +2021,15 @@ def close(self, force=False): """ msg = 'Closing connection to %s:%s (force=%s).' self._dbg(1, msg % (self.host, self.port, repr(force))) - if self.connected: + if not self.connected: + self._reset() + return + try: if not force: self._logout() - self._close(force=force) - self._reset() + finally: + try: + self._close(force=force) + except Exception as e: # e.g. the host closed the connection first + self._dbg(1, 'Error while closing the connection: %s' % repr(e)) + self._reset() diff --git a/src/Exscript/protocols/ssh2.py b/src/Exscript/protocols/ssh2.py index eea412b1..483f3090 100644 --- a/src/Exscript/protocols/ssh2.py +++ b/src/Exscript/protocols/ssh2.py @@ -152,21 +152,8 @@ def _append_known_host(self, name, key): fp.write(line.encode()) def _paramiko_connect(self): - # Find supported address families. - addrinfo = socket.getaddrinfo(self.host, self.port) - for family, socktype, proto, canonname, sockaddr in addrinfo: - af = family - addr = sockaddr - if socktype == socket.SOCK_STREAM: - break - - # Open a socket. - self._sock = socket.socket(af, socket.SOCK_STREAM) - try: - self._sock.settimeout(self.connect_timeout or None) - except: - pass - self._sock.connect(addr) + # Tries all addresses of the host, e.g. IPv4 after IPv6. + self._sock = socket.create_connection((self.host, self.port), self.connect_timeout or None) # Init the paramiko protocol. t = paramiko.Transport(self._sock, disabled_algorithms=self.disabled_algorithms) diff --git a/tests/Exscript/protocols/ExpectTest.py b/tests/Exscript/protocols/ExpectTest.py index 9e37202a..67fe3cf3 100644 --- a/tests/Exscript/protocols/ExpectTest.py +++ b/tests/Exscript/protocols/ExpectTest.py @@ -1,8 +1,10 @@ import os import re import socket +import threading import time import unittest +from unittest import mock from Exscript import Account, PrivateKey from Exscript.emulators import VirtualDevice @@ -201,6 +203,36 @@ def testFailedAuthorization(self): self.assertRaises(LoginFailure, conn.app_authorize, self.account) +class CloseTest(unittest.TestCase): + def testForceNeverRaises(self): + conn = connect([]) + conn._close = mock.Mock(side_effect=EOFError) # paramiko, if the host closed first + conn.close(force=True) + conn._close.assert_called_once_with(force=True) + self.assertFalse(conn.connected) + + def testFailedLogoutStillCloses(self): + conn = connect([]) + conn._logout = mock.Mock(side_effect=TimeoutException('no response to exit')) + conn._close = mock.Mock() + self.assertRaises(TimeoutException, conn.close) + conn._close.assert_called_once_with(force=False) + self.assertFalse(conn.connected) + + +def start_daemon(test, daemon_cls, device, **kwargs): + """Starts a test server for the device; it stops when the test ends.""" + with socket.socket() as sock: + sock.bind((HOST, 0)) + port = sock.getsockname()[1] + daemon = daemon_cls(HOST, port, device, **kwargs) + daemon.start() + time.sleep(0.2) + test.addCleanup(daemon.join) # cleanups run LIFO: exit() first + test.addCleanup(daemon.exit) + return daemon, port + + class ServerExpectTest(unittest.TestCase): """expect() finds a pattern regardless of how much data follows it.""" @@ -208,17 +240,6 @@ def setUp(self): self.device = VirtualDevice(HOST, login_type=VirtualDevice.LOGIN_TYPE_NONE, strict=False) self.device.add_command('show big', 'Please confirm: MARKER ' + 'x' * 2000) - def start_daemon(self, daemon_cls, **kwargs): - with socket.socket() as sock: - sock.bind((HOST, 0)) - port = sock.getsockname()[1] - daemon = daemon_cls(HOST, port, self.device, **kwargs) - daemon.start() - time.sleep(0.2) - self.addCleanup(daemon.join) # cleanups run LIFO: exit() first - self.addCleanup(daemon.exit) - return port - def expectMarker(self, conn, port=None): self.addCleanup(conn.close, force=True) conn.connect(HOST if port else 'dummy', port) @@ -230,11 +251,11 @@ def testDummy(self): self.expectMarker(Dummy(device=self.device)) def testTelnet(self): - port = self.start_daemon(Telnetd) + port = start_daemon(self, Telnetd, self.device)[1] self.expectMarker(Telnet(timeout=1, connect_rate_limit=0), port) def testSSH2(self): - port = self.start_daemon(SSHd, key=PrivateKey.from_file(KEYFILE)) + port = start_daemon(self, SSHd, self.device, key=PrivateKey.from_file(KEYFILE))[1] conn = SSH2( timeout=1, connect_rate_limit=0, @@ -242,3 +263,50 @@ def testSSH2(self): preferred_authentications=['password'], ) self.expectMarker(conn, port) + + +@unittest.skipUnless(hasattr(os, 'openpty'), 'interact() reads from a POSIX terminal') +class InteractTest(unittest.TestCase): + """interact() passes typed keys and received text, also multibyte characters.""" + + def setUp(self): + self.device = VirtualDevice(HOST, login_type=VirtualDevice.LOGIN_TYPE_NONE, strict=False) + self.device.add_command(r'say \S+', lambda command: command.split()[1] + '!') + + def interact(self, daemon_cls, conn, **kwargs): + daemon, port = start_daemon(self, daemon_cls, self.device, **kwargs) + self.device.add_command('exit', daemon.exit_command) + self.addCleanup(conn.close, force=True) + conn.connect(HOST, port) + conn.protocol_authenticate(Account('user', 'password', needs_lock=False)) + received = [] + conn.data_received_event.connect(received.append) + + master, slave = os.openpty() # interact() closes the slave + self.addCleanup(os.close, master) + + def type_keys(): + time.sleep(0.3) + os.write(master, 'say grüße\r'.encode()) + time.sleep(0.3) + os.write(master, b'exit\r') + + typist = threading.Thread(target=type_keys) + typist.start() + with mock.patch('sys.stdin', mock.Mock(encoding='utf-8', fileno=lambda: slave)): + conn.interact(handle_window_size=False) + typist.join() + self.assertIn('grüße!', ''.join(received)) + + def testTelnet(self): + self.interact(Telnetd, Telnet(timeout=1, connect_rate_limit=0, encoding='utf-8')) + + def testSSH2(self): + conn = SSH2( + timeout=1, + connect_rate_limit=0, + verify_fingerprint=False, + preferred_authentications=['password'], + encoding='utf-8', + ) + self.interact(SSHd, conn, key=PrivateKey.from_file(KEYFILE)) diff --git a/tests/Exscript/protocols/SSH2Test.py b/tests/Exscript/protocols/SSH2Test.py index 78e54f5f..a28cc86a 100644 --- a/tests/Exscript/protocols/SSH2Test.py +++ b/tests/Exscript/protocols/SSH2Test.py @@ -210,6 +210,20 @@ def testNoReconnectByDefault(self): self.assertEqual(self.handshakes('generic'), 1) +class ConnectTest(unittest.TestCase): + def testTriesAllAddresses(self): + server = KeyServer(self, paramiko.RSAKey(filename=keyfile)) + addresses = [ + (socket.AF_INET6, socket.SOCK_STREAM, 6, '', ('::1', server.port, 0, 0)), # refused + (socket.AF_INET, socket.SOCK_STREAM, 6, '', (HOST, server.port)), + ] + conn = SSH2(timeout=1, connect_rate_limit=0, verify_fingerprint=False) + self.addCleanup(conn.close, force=True) + with mock.patch('socket.getaddrinfo', return_value=addresses): + conn.connect('router.example', server.port) + self.assertTrue(conn.connected) + + def suite(): loader = unittest.TestLoader() return unittest.TestSuite( @@ -217,6 +231,7 @@ def suite(): loader.loadTestsFromTestCase(SSH2Test), loader.loadTestsFromTestCase(HostKeyTest), loader.loadTestsFromTestCase(ReconnectTest), + loader.loadTestsFromTestCase(ConnectTest), ] ) From dc22af4a0ec8be311a3b3dd18e0d21527dc229ca Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 3 Nov 2026 05:17:48 +0100 Subject: [PATCH 035/250] Read and write text files as UTF-8 on every platform Before Python 3.15, open() without an encoding uses the locale's encoding, e.g. cp1252 on Windows, which misread UTF-8 templates and could fail to write device output to logs. util.daemonize opens its log files in binary mode, as unbuffered text mode raises ValueError. --- CHANGES.md | 5 +++ pyproject.toml | 6 ++- scripts/exscript | 2 +- src/Exscript/logger.py | 6 +-- src/Exscript/parselib/lexer.py | 2 +- src/Exscript/protocols/protocol.py | 6 ++- src/Exscript/queue.py | 50 ++++++++++++----------- src/Exscript/stdlib/file.py | 8 ++-- src/Exscript/util/daemonize.py | 6 +-- src/Exscript/util/interact.py | 2 +- src/Exscript/util/mail.py | 4 +- src/Exscript/util/pidutil.py | 2 +- src/Exscript/util/template.py | 6 +-- src/Exscript/util/tty.py | 2 +- tests/Exscript/FileLoggerTest.py | 4 +- tests/Exscript/QueueTest.py | 2 +- tests/Exscript/TemplateTest.py | 2 +- tests/Exscript/protocols/OsGuesserTest.py | 2 +- tests/Exscript/protocols/ProtocolTest.py | 4 +- tests/Exscript/protocols/SSH2Test.py | 4 +- tests/Exscript/util/interactTest.py | 2 +- tests/Exscript/util/xmlTest.py | 2 +- 22 files changed, 71 insertions(+), 58 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index dc4f545c..d4fe7a81 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -11,6 +11,11 @@ marked with **BREAKING**; a migration guide will accompany the release. - **License Labels:** Exscript remains MIT-licensed. `Exscript.protocols.telnetlib` is derived from CPython's telnetlib (3.12), so it carries the PSF notice, the license text is in `LICENSES/PSF-2.0.txt`, and the package declares `MIT AND PSF-2.0`. +- **UTF-8 Files:** Exscript reads and writes text files, e.g. templates, host lists + and log files, as UTF-8 on every platform, as Python 3.15 does by default. Before, + it used the encoding of the locale, e.g. cp1252 on Windows, where writing device + output to a log file could fail. `util.daemonize` no longer fails with + `ValueError: can't have unbuffered text I/O`. - **Python 2.7 Support:** - **Python 3.11+ Support:** - **Dependencies:** diff --git a/pyproject.toml b/pyproject.toml index 9f927335..237666c2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -86,7 +86,11 @@ extend-exclude = ["doc"] quote-style = "preserve" [tool.ruff.lint] -select = ["E", "W", "F", "I", "UP", "B", "S"] +# PLW1514 (open() without an encoding) is a preview rule; only explicitly selected +# preview rules are enabled. +preview = true +explicit-preview-rules = true +select = ["E", "W", "F", "I", "UP", "B", "S", "PLW1514"] ignore = [ "UP031", # printf-style formatting is used throughout # Baseline of existing findings: fix a rule everywhere, then remove it here. diff --git a/scripts/exscript b/scripts/exscript index f48b576e..236662b1 100755 --- a/scripts/exscript +++ b/scripts/exscript @@ -143,7 +143,7 @@ def run(): if not args.filename: parser.error("Missing template filename") try: - with open(args.filename) as fp: + with open(args.filename, encoding="utf-8") as fp: script_content = fp.read() except OSError as e: parser.error(str(e)) diff --git a/src/Exscript/logger.py b/src/Exscript/logger.py index c9ae3cf1..2b4bdbfc 100644 --- a/src/Exscript/logger.py +++ b/src/Exscript/logger.py @@ -200,10 +200,10 @@ def __init__(self, name, filename, mode='a', delete=False): def __str__(self): data = '' if os.path.isfile(self.filename): - with open(self.filename) as thefile: + with open(self.filename, encoding="utf-8") as thefile: data += thefile.read() if os.path.isfile(self.errorname): - with open(self.errorname) as thefile: + with open(self.errorname, encoding="utf-8") as thefile: data += thefile.read() return data @@ -211,7 +211,7 @@ def _write_file(self, filename, *data): if not self.do_log: return try: - with open(filename, self.mode) as thefile: + with open(filename, self.mode, encoding='utf-8') as thefile: thefile.write(' '.join(data)) except Exception as e: print('Error writing to %s: %s' % (filename, e)) diff --git a/src/Exscript/parselib/lexer.py b/src/Exscript/parselib/lexer.py index 76b1647d..04616dbd 100644 --- a/src/Exscript/parselib/lexer.py +++ b/src/Exscript/parselib/lexer.py @@ -240,5 +240,5 @@ def parse(self, string, filename=None): return compiled def parse_file(self, filename): - with open(filename) as fp: + with open(filename, encoding='utf-8') as fp: return self.parse(fp.read(), filename) diff --git a/src/Exscript/protocols/protocol.py b/src/Exscript/protocols/protocol.py index 3bc52aa2..b4716f3f 100644 --- a/src/Exscript/protocols/protocol.py +++ b/src/Exscript/protocols/protocol.py @@ -430,7 +430,9 @@ def logfile(self, value) -> None: except Exception: pass self._logfile = value - self._log = open(self._logfile, 'a') if value and self.connected else None + self._log = ( + open(self._logfile, 'a', encoding='utf-8') if value and self.connected else None + ) @property def os_guesser(self): @@ -1003,7 +1005,7 @@ def connect(self, hostname=None, port=None): # Connecting... self._dbg(1, "Attempting to connect: '%s:%s'" % (self.host, self.port)) if self.logfile and not self._log: - self._log = open(self.logfile, 'a') + self._log = open(self.logfile, 'a', encoding='utf-8') self._enforce_connect_rate_limit() self._connected = bool(self._connect()) self._dbg(2, "Finished to connect: '%s:%s'" % (self.host, self.port)) diff --git a/src/Exscript/queue.py b/src/Exscript/queue.py index c132cb8e..e273d7df 100644 --- a/src/Exscript/queue.py +++ b/src/Exscript/queue.py @@ -220,7 +220,6 @@ def __init__( - S = status bar - L = live conversation - - D = debug messages - E = errors - ! = errors with tracebacks - F = fatal errors with tracebacks @@ -231,8 +230,11 @@ def __init__( - verbose = 0: stdout = None, stderr = EF - verbose = 1, max_threads = 1: stdout = L, stderr = EF - verbose = 1, max_threads = n: stdout = S, stderr = EF - - verbose >= 2, max_threads = 1: stdout = DL, stderr = !F - - verbose >= 2, max_threads = n: stdout = DS, stderr = !F + - verbose >= 2, max_threads = 1: stdout = L, stderr = !F + - verbose >= 2, max_threads = n: stdout = S, stderr = !F + + From verbosity 2 on, the queue also logs debug messages to the Python + logger ``Exscript.queue``. :type domain: str :param domain: The default domain of the contacted hosts. @@ -263,7 +265,7 @@ def __init__( self.host_driver = host_driver self.exc_cb = exc_cb self.lock = threading.RLock() - self.devnull = open(os.devnull, 'w') + self.devnull = open(os.devnull, 'w', encoding='utf-8') self.channel_map = {'fatal_errors': self.stderr} self.completed = 0 self.total = 0 @@ -454,34 +456,34 @@ def get_max_threads(self): def add_account_pool(self, pool, match=None): """ Adds a new account pool. If the given match argument is - None, the pool the default pool. Otherwise, the match argument is + None, the pool is the default pool. Otherwise, the match argument is a callback function that is invoked to decide whether or not the given pool should be used for a host. When Exscript logs into a host, the account is chosen in the following order: - # Exscript checks whether an account was attached to the - :class:`Host` object using :class:`Host.set_account()`), and uses that. + #. Exscript checks whether an account was attached to the + :class:`Host` object using :class:`Host.set_account()`, and uses that. - # If the :class:`Host` has no account attached, Exscript walks - through all pools that were passed to :class:`Queue.add_account_pool()`. - For each pool, it passes the :class:`Host` to the function in the - given match argument. If the return value is True, the account - pool is used to acquire an account. - (Accounts within each pool are taken in a round-robin - fashion.) + #. If the :class:`Host` has no account attached, Exscript walks + through all pools that were passed to :class:`Queue.add_account_pool()`. + For each pool, it passes the :class:`Host` to the function in the + given match argument. If the return value is True, the account + pool is used to acquire an account. + (Accounts within each pool are taken in a round-robin + fashion.) - # If no matching account pool is found, an account is taken - from the default account pool. + #. If no matching account pool is found, an account is taken + from the default account pool. - # Finally, if all that fails and the default account pool - contains no accounts, an error is raised. + #. Finally, if all that fails and the default account pool + contains no accounts, an error is raised. Example usage:: - def do_nothing(conn): - conn.autoinit() + def log_in(job, host, conn): + conn.login() def use_this_pool(host): return host.get_name().startswith('foo') @@ -497,7 +499,7 @@ def use_this_pool(host): queue.add_account_pool(other_pool, use_this_pool) host = Host('localhost') - queue.run(host, do_nothing) + queue.run(host, log_in) In the example code, the host has no account attached. As a result, the queue checks whether use_this_pool() returns True. Because the @@ -524,7 +526,7 @@ def add_account(self, account): def is_completed(self): """ Returns True if the task is completed, False otherwise. - In other words, this methods returns True if the queue is empty. + In other words, this method returns True if the queue is empty. :rtype: bool :return: Whether all tasks are completed. @@ -627,8 +629,8 @@ def run(self, hosts, function, attempts=1): """ Add the given function to a queue, and call it once for each host according to the threading options. - Use decorators.bind() if you also want to pass additional - arguments to the callback function. + Use :func:`Exscript.util.decorator.bind` if you also want to pass + additional arguments to the callback function. Returns an object that represents the queued task, and that may be passed to is_completed() to check the status. diff --git a/src/Exscript/stdlib/file.py b/src/Exscript/stdlib/file.py index 796fc0ae..b9b22c8b 100644 --- a/src/Exscript/stdlib/file.py +++ b/src/Exscript/stdlib/file.py @@ -29,7 +29,7 @@ def chmod(scope, filename, mode): """ Changes the permissions of the given file (or list of files) to the given mode. You probably want to use an octal representation - for the integer, e.g. "chmod(myfile, 0644)". + for the integer, e.g. ``file.chmod(myfile, 0644)``. :type filename: string :param filename: A filename. @@ -49,7 +49,7 @@ def clear(scope, filename): :type filename: string :param filename: A filename. """ - with open(filename[0], 'w'): + with open(filename[0], 'w', encoding='utf-8'): pass return True @@ -98,7 +98,7 @@ def read(scope, filename): :rtype: string :return: The content of the file. """ - with open(filename[0]) as fp: + with open(filename[0], encoding='utf-8') as fp: lines = fp.readlines() scope.define(__response__=lines) return lines @@ -131,6 +131,6 @@ def write(scope, filename, lines, mode=['a']): :type mode: string :param mode: Any of the above listed modes. """ - with open(filename[0], mode[0]) as fp: + with open(filename[0], mode[0], encoding='utf-8') as fp: fp.writelines(['%s\n' % line.rstrip() for line in lines]) return True diff --git a/src/Exscript/util/daemonize.py b/src/Exscript/util/daemonize.py index abc7966d..3886a2fa 100644 --- a/src/Exscript/util/daemonize.py +++ b/src/Exscript/util/daemonize.py @@ -29,9 +29,9 @@ def _redirect_output(filename): - out_log = open(filename, 'a+', 0) - err_log = open(filename, 'a+', 0) - dev_null = open(os.devnull) + out_log = open(filename, 'ab', 0) + err_log = open(filename, 'ab', 0) + dev_null = open(os.devnull, 'rb') os.close(sys.stdin.fileno()) os.close(sys.stdout.fileno()) os.close(sys.stderr.fileno()) diff --git a/src/Exscript/util/interact.py b/src/Exscript/util/interact.py index 15eec777..fa7d9eba 100644 --- a/src/Exscript/util/interact.py +++ b/src/Exscript/util/interact.py @@ -107,7 +107,7 @@ def set(self, key, value): return None self.parser.set(self.section, key, value) - with open(self.filename, mode="w") as fp: + with open(self.filename, encoding='utf-8', mode="w") as fp: self.parser.write(fp) return value diff --git a/src/Exscript/util/mail.py b/src/Exscript/util/mail.py index 6760f745..8c78aa0f 100644 --- a/src/Exscript/util/mail.py +++ b/src/Exscript/util/mail.py @@ -418,7 +418,7 @@ def from_template(filename, **kwargs): :rtype: Mail :return: The resulting mail. """ - with open(filename) as fp: + with open(filename, encoding='utf-8') as fp: return from_template_string(fp.read(), **kwargs) @@ -432,7 +432,7 @@ def _get_mime_object(filename): maintype, subtype = ctype.split('/', 1) if maintype == 'text': - with open(filename) as fp: + with open(filename, encoding='utf-8') as fp: msg = MIMEText(fp.read(), _subtype=subtype) elif maintype == 'image': with open(filename, 'rb') as fp: diff --git a/src/Exscript/util/pidutil.py b/src/Exscript/util/pidutil.py index 112a61ea..c5af403e 100644 --- a/src/Exscript/util/pidutil.py +++ b/src/Exscript/util/pidutil.py @@ -47,7 +47,7 @@ def read(path): # Try to read the pid from the pidfile. _logger.info("Checking pidfile '%s'", path) try: - return int(open(path).read()) + return int(open(path, encoding="utf-8").read()) except OSError as xxx_todo_changeme: (code, text) = xxx_todo_changeme.args if code == errno.ENOENT: # no such file or directory diff --git a/src/Exscript/util/template.py b/src/Exscript/util/template.py index 07139013..526eceee 100644 --- a/src/Exscript/util/template.py +++ b/src/Exscript/util/template.py @@ -113,7 +113,7 @@ def test_file(filename, secure=True, **kwargs): :param kwargs: Variables to define in the template. """ parser_args = {'secure': secure, 'no_escaping': False} - with open(filename) as fp: + with open(filename, encoding='utf-8') as fp: _compile(None, filename, fp.read(), parser_args, **kwargs) @@ -184,7 +184,7 @@ def eval_file(conn, filename, strip_command=True, no_escaping=False, *, secure=T :param kwargs: Variables to define in the template. """ parser_args = {'strip_command': strip_command, 'no_escaping': no_escaping, 'secure': secure} - with open(filename) as fp: + with open(filename, encoding='utf-8') as fp: return _run(conn, filename, fp.read(), parser_args, **kwargs) @@ -239,5 +239,5 @@ def paste_file(conn, filename, no_escaping=False, *, secure=True, **kwargs): :param kwargs: Variables to define in the template. """ parser_args = {'no_prompt': True, 'no_escaping': no_escaping, 'secure': secure} - with open(filename) as fp: + with open(filename, encoding='utf-8') as fp: return _run(conn, filename, fp.read(), parser_args, **kwargs) diff --git a/src/Exscript/util/tty.py b/src/Exscript/util/tty.py index dc8c2c70..f4324ba0 100644 --- a/src/Exscript/util/tty.py +++ b/src/Exscript/util/tty.py @@ -114,7 +114,7 @@ def get_terminal_size(default_rows=25, default_cols=80): os.close(fd) # Try `stty size` - with open(os.devnull, 'w') as devnull: + with open(os.devnull, 'w', encoding='utf-8') as devnull: try: process = Popen(['stty', 'size'], stderr=devnull, stdout=PIPE, close_fds=True) except (OSError, ValueError): diff --git a/tests/Exscript/FileLoggerTest.py b/tests/Exscript/FileLoggerTest.py index ea226d6f..cf7a9d56 100644 --- a/tests/Exscript/FileLoggerTest.py +++ b/tests/Exscript/FileLoggerTest.py @@ -65,7 +65,7 @@ def testAddLog2(self): self.logger.add_log(id(self.job), self.job.name, 2) self.assertTrue(os.path.isfile(self.logfile)) self.assertFalse(os.path.exists(self.errfile)) - with open(self.logfile) as fp: + with open(self.logfile, encoding='utf-8') as fp: content = fp.read() self.assertEqual(content, '') @@ -75,7 +75,7 @@ def testLog2(self): self.logger.log(id(self.job), 'hello world') self.assertTrue(os.path.isfile(self.logfile)) self.assertFalse(os.path.exists(self.errfile)) - with open(self.logfile) as fp: + with open(self.logfile, encoding='utf-8') as fp: content = fp.read() self.assertEqual(content, 'hello world') diff --git a/tests/Exscript/QueueTest.py b/tests/Exscript/QueueTest.py index a9378a28..3ed972e4 100644 --- a/tests/Exscript/QueueTest.py +++ b/tests/Exscript/QueueTest.py @@ -429,7 +429,7 @@ def testQueueLogging(self): self.assertIn('dummy1.log', logfiles) self.assertIn('dummy2.log', logfiles) for file in logfiles: - with open(os.path.join(self.tempdir, file)) as fp: + with open(os.path.join(self.tempdir, file), encoding='utf-8') as fp: content = fp.read() diff --git a/tests/Exscript/TemplateTest.py b/tests/Exscript/TemplateTest.py index 2526a9e2..f9ef19aa 100644 --- a/tests/Exscript/TemplateTest.py +++ b/tests/Exscript/TemplateTest.py @@ -45,7 +45,7 @@ def dummy_cb(job, host, conn, template_test): print(log.data) raise log.data = log.data.replace('\r\n', '\n') - with open(expected, newline='') as fp: + with open(expected, encoding='utf-8', newline='') as fp: expected_data = fp.read() # open(expected, 'w').write(log.data) # open('output', 'w').write(log.data) diff --git a/tests/Exscript/protocols/OsGuesserTest.py b/tests/Exscript/protocols/OsGuesserTest.py index 40d366e0..433f77b8 100644 --- a/tests/Exscript/protocols/OsGuesserTest.py +++ b/tests/Exscript/protocols/OsGuesserTest.py @@ -71,7 +71,7 @@ def testCheckReceivedData(self): if not drivers.driver_map[osname].supports_os_guesser(): continue file = os.path.join(banner_dir, file) - with open(file) as fp: + with open(file, encoding='utf-8') as fp: banner = fp.read().rstrip('\n') osg = OsGuesser() for char in banner: diff --git a/tests/Exscript/protocols/ProtocolTest.py b/tests/Exscript/protocols/ProtocolTest.py index ba84764e..1fb98305 100644 --- a/tests/Exscript/protocols/ProtocolTest.py +++ b/tests/Exscript/protocols/ProtocolTest.py @@ -881,7 +881,7 @@ def testReconnect(self): self.assertEqual(first_response, self.protocol.response) # First session data must have been flushed to disk. - with open(logpath) as f: + with open(logpath, encoding='utf-8') as f: first_log_size = len(f.read()) self.assertGreater(first_log_size, 0) @@ -902,7 +902,7 @@ def testReconnect(self): self.assertEqual(logpath, self.protocol.logfile) # Second session appended to the same logfile. - with open(logpath) as f: + with open(logpath, encoding='utf-8') as f: second_log_size = len(f.read()) self.assertGreater(second_log_size, first_log_size) diff --git a/tests/Exscript/protocols/SSH2Test.py b/tests/Exscript/protocols/SSH2Test.py index a28cc86a..d32742f2 100644 --- a/tests/Exscript/protocols/SSH2Test.py +++ b/tests/Exscript/protocols/SSH2Test.py @@ -95,7 +95,7 @@ def setUp(self): self.known_hosts = os.path.join(tmpdir.name, 'known_hosts') def writeKnownHosts(self, *entries): - with open(self.known_hosts, 'w') as fp: + with open(self.known_hosts, 'w', encoding='utf-8') as fp: for name, host_key in entries: fp.write(f'{name} {host_key.get_name()} {host_key.get_base64()}\n') @@ -150,7 +150,7 @@ def testWarnPolicyPinsTheKey(self): self.connect(conn) def testAutoAddPolicyAppendsToKnownHosts(self): - with open(self.known_hosts, 'w') as fp: + with open(self.known_hosts, 'w', encoding='utf-8') as fp: fp.write(f'192.0.2.1 {self.other_key.get_name()} {self.other_key.get_base64()}') self.connect(host_key_policy='auto-add') known = paramiko.HostKeys(self.known_hosts) diff --git a/tests/Exscript/util/interactTest.py b/tests/Exscript/util/interactTest.py index 513ac205..242ae9de 100644 --- a/tests/Exscript/util/interactTest.py +++ b/tests/Exscript/util/interactTest.py @@ -22,7 +22,7 @@ def testConstructor(self): h = InputHistory(filename) h = InputHistory(filename, 'foo') h.set('aaa', 'bbb') - with open(filename) as fp: + with open(filename, encoding='utf-8') as fp: self.assertEqual(fp.read(), '[foo]\naaa = bbb\n\n') def testGet(self): diff --git a/tests/Exscript/util/xmlTest.py b/tests/Exscript/util/xmlTest.py index 9d9e871c..6f817383 100644 --- a/tests/Exscript/util/xmlTest.py +++ b/tests/Exscript/util/xmlTest.py @@ -19,7 +19,7 @@ def testValidateXml(self): self.assertFalse(validate_xml('\udcff', silent=True)) def testExternalEntitiesAreNotResolved(self): - with tempfile.NamedTemporaryFile('w', suffix='.txt', delete=False) as fp: + with tempfile.NamedTemporaryFile('w', encoding='utf-8', suffix='.txt', delete=False) as fp: fp.write('XXE-MARKER') self.addCleanup(os.remove, fp.name) url = pathlib.Path(fp.name).as_uri() From 8eb85abf327b8c3cd606fed2f7b441856f9e4ffb Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 3 Nov 2026 05:41:25 +0100 Subject: [PATCH 036/250] Scope NETCONF pretty-printing and lock the shared buffer Each login patched ncclient's to_xml(), so every ncclient user in the process sent pretty-printed XML; a ContextVar now limits it to Exscript's own calls. MonitoredBuffer takes a lock, as ncclient's thread appends to the buffer that the caller searches. --- CHANGES.md | 5 +++ src/Exscript/protocols/netconf.py | 56 ++++++++++++++++--------- src/Exscript/util/buffer.py | 22 ++++++++++ tests/Exscript/protocols/NetconfTest.py | 50 ++++++++++++++++++++++ 4 files changed, 113 insertions(+), 20 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index d4fe7a81..60e7c5d4 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -170,6 +170,11 @@ marked with **BREAKING**; a migration guide will accompany the release. that the callback gets refer to the searched text, not to the whole buffer. - **NETCONF 1.1 Chunks:** `Netconf.reassemble_chunks()` takes linear time: a 16 MiB reply in 4 KiB chunks takes 0.013 seconds instead of 5.2. +- **NETCONF Threads:** ncclient's thread appends the received data to the buffer that + the caller searches. The buffer is locked now; before, a search could move the + position at which the thread wrote. Exscript still sends its `` and RPCs + pretty-printed, but no longer makes other users of ncclient in the same process do + so (changes against the 2.9 development versions). ### Telnet diff --git a/src/Exscript/protocols/netconf.py b/src/Exscript/protocols/netconf.py index 078cf7f8..fa821af8 100644 --- a/src/Exscript/protocols/netconf.py +++ b/src/Exscript/protocols/netconf.py @@ -24,12 +24,16 @@ SSH2 Netconf support, based on ncclient and paramiko. """ +import contextlib +import contextvars import functools import inspect import logging import re import time +import ncclient.operations.rpc +import ncclient.transport.session from lxml.etree import XMLSyntaxError from ncclient import NCClientError from ncclient.manager import Manager, make_device_handler @@ -54,6 +58,32 @@ _chunk_header_re = re.compile(r'\n#(\d+)\n') _end_of_chunks_re = re.compile(r'\n##\n') +# Exscript sends its and RPCs pretty-printed, so that they are readable +# in logs. ncclient serializes them with the to_xml() of these two modules, in +# the thread that sends; the replacement pretty-prints only within _pretty(), +# so other users of ncclient in the process are not affected. +_pretty_print = contextvars.ContextVar('exscript_netconf_pretty_print', default=False) + + +@functools.wraps(to_xml) +def _to_xml(*args, **kwargs): + if _pretty_print.get(): + kwargs['pretty_print'] = True + return to_xml(*args, **kwargs) + + +ncclient.operations.rpc.to_xml = _to_xml +ncclient.transport.session.to_xml = _to_xml + + +@contextlib.contextmanager +def _pretty(): + token = _pretty_print.set(True) + try: + yield + finally: + _pretty_print.reset(token) + class Netconf(SSH2): """ @@ -251,26 +281,10 @@ def send_wrapper(*args, **kwargs): self._channel.send = send_wrapper - # Setup the RPC relevant to_xml wrapper so that ncclient's RPCs are - # always send (and logged by Exscript) in a pretty formatted style. - # This is quite hacky, but the only way to achieve it on the fly - # without additional XML parsing and reformatting. - import ncclient.operations.retrieve as retrieve_module - import ncclient.operations.rpc as rpc_module - import ncclient.transport.session as session_module - - @functools.wraps(to_xml) - def to_xml_wrapper(*args, **kwargs): - kwargs['pretty_print'] = True - return to_xml(*args, **kwargs) - - retrieve_module.to_xml = to_xml_wrapper # for retrieved RPC replies - rpc_module.to_xml = to_xml_wrapper # for all kinds of sent RPCs - session_module.to_xml = to_xml_wrapper # where client hello is build - # Starts IO thread, sends client , consumes server . self._buffer.clear() # keeps the first self.response clean - session._post_connect(self.get_timeout()) + with _pretty(): + session._post_connect(self.get_timeout()) # Selects the driver/device-specific XML parser for all subsequent RPCs. session.parser = device_handler.get_xml_parser(session) @@ -380,7 +394,8 @@ def _send(self, data): self._validate_rpc_body(data) try: element = _parse(data) - self._ncc_response = self._ncc_manager.dispatch(element) + with _pretty(): + self._ncc_response = self._ncc_manager.dispatch(element) # wrappers handle the self.data_sent_event except TimeoutExpiredError: self._raise_on_response_timeout() @@ -435,7 +450,8 @@ def _interact(self, key_handlers=None, handle_window_size=True): def _logout(self): try: - self._ncc_manager.close_session() + with _pretty(): + self._ncc_manager.close_session() except Exception: pass diff --git a/src/Exscript/util/buffer.py b/src/Exscript/util/buffer.py index 1d683d99..a419ba90 100644 --- a/src/Exscript/util/buffer.py +++ b/src/Exscript/util/buffer.py @@ -24,6 +24,8 @@ A buffer object. """ +import functools +import threading from io import StringIO from .cast import to_regexs @@ -33,10 +35,21 @@ _CONTEXT = 256 +def _locked(method): + @functools.wraps(method) + def wrapper(self, *args, **kwargs): + with self._lock: + return method(self, *args, **kwargs) + + return wrapper + + class MonitoredBuffer: """ A specialized string buffer that allows for monitoring the content using regular expression-triggered callbacks. + The methods are thread-safe: the NETCONF adapter, for example, appends + in ncclient's thread while the caller searches. """ def __init__(self, io=None): @@ -53,14 +66,17 @@ def __init__(self, io=None): else: self.io = io self.monitors = [] + self._lock = threading.RLock() self.clear() + @_locked def __str__(self): """ Returns the content of the buffer. """ return self.io.getvalue() + @_locked def size(self): """ Returns the size of the buffer. @@ -70,6 +86,7 @@ def size(self): """ return self.io.tell() + @_locked def head(self, bytes): """ Returns the number of given bytes from the head of the buffer. @@ -84,6 +101,7 @@ def head(self, bytes): self.io.seek(oldpos) return head + @_locked def tail(self, bytes): """ Returns the number of given bytes from the tail of the buffer. @@ -95,6 +113,7 @@ def tail(self, bytes): self.io.seek(max(0, self.size() - bytes)) return self.io.read() + @_locked def pop(self, bytes): """ Like :class:`head()`, but also removes the head from the buffer. @@ -112,6 +131,7 @@ def pop(self, bytes): item[2] = max(0, item[2] - len(head)) return head + @_locked def append(self, data): """ Appends the given data to the buffer, and triggers all connected @@ -141,6 +161,7 @@ def append(self, data): item[2] = offset + match.end() callback(i, match) + @_locked def clear(self): """ Removes all data from the buffer. @@ -150,6 +171,7 @@ def clear(self): for item in self.monitors: item[2] = 0 + @_locked def add_monitor(self, pattern, callback, limit=80): """ Calls the given function whenever the given pattern matches the diff --git a/tests/Exscript/protocols/NetconfTest.py b/tests/Exscript/protocols/NetconfTest.py index 99540b8e..aeb404d2 100644 --- a/tests/Exscript/protocols/NetconfTest.py +++ b/tests/Exscript/protocols/NetconfTest.py @@ -1,9 +1,22 @@ """Tests for the parts of the NETCONF adapter that need no server.""" +import inspect +import threading import unittest +import ncclient.operations.rpc +import ncclient.transport.session +from ncclient.manager import make_device_handler +from ncclient.operations.rpc import RPC, RPCReply +from ncclient.transport.session import HelloHandler +from ncclient.transport.ssh import SSHSession +from ncclient.xml_ import NCElement, new_ele + +from Exscript.protocols import netconf from Exscript.protocols.netconf import Netconf +BASE = 'urn:ietf:params:netconf:base:1.0' + class ReassembleChunksTest(unittest.TestCase): def testChunks(self): @@ -20,3 +33,40 @@ def testManyChunks(self): chunk = 'x' * 4096 framed = ''.join('\n#4096\n' + chunk for _ in range(1024)) + '\n##\n' self.assertEqual(Netconf.reassemble_chunks(framed), chunk * 1024) + + +class NcclientContractTest(unittest.TestCase): + """The ncclient internals that the adapter uses; an update of ncclient must keep them.""" + + def setUp(self): + self.handler = make_device_handler({'name': 'default'}) + + def testSessionAttributes(self): + # _init_ncclient() sets these on a session that it did not connect. + session = SSHSession(self.handler) + names = '_host', '_transport', '_channel', '_channel_id', '_channel_name', '_connected' + for name in names: + self.assertTrue(hasattr(session, name), name) + self.assertIsInstance(session._closing, threading.Event) + self.assertIn('timeout', inspect.signature(session._post_connect).parameters) + session.parser = self.handler.get_xml_parser(session) + + def testReplyOfATransformedResult(self): + # _raise_on_response_with_errors() reads the reply behind an NCElement. + reply = RPCReply('' % BASE) + reply.parse() + self.assertIs(NCElement(reply, lambda root: root)._NCElement__result, reply) + + def testPrettyPrintOnlyForExscript(self): + self.assertIs(ncclient.operations.rpc.to_xml, netconf._to_xml) + self.assertIs(ncclient.transport.session.to_xml, netconf._to_xml) + rpc = RPC.__new__(RPC) # _wrap() needs only these two attributes + rpc._id, rpc._device_handler = '1', self.handler + for serialize in ( + lambda: HelloHandler.build([BASE], self.handler), + lambda: rpc._wrap(new_ele('get')), + ): + self.assertNotIn('\n <', serialize()) # other ncclient users + with netconf._pretty(): + self.assertIn('\n <', serialize()) + self.assertNotIn('\n <', serialize()) From af3f704338f031ff195a21c11591167f3f25fcc8 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 3 Nov 2026 05:43:17 +0100 Subject: [PATCH 037/250] Run the benchmarks in CI The job publishes the results and fails if one exceeds four times its target, so slower CI machines pass, but quadratic regressions do not. --- .github/workflows/ci.yml | 26 ++++++++++++++++++++++++++ tools/bench_bulk.py | 37 ++++++++++++++++++++++++++++++++++--- 2 files changed, 60 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 562b1381..9b57e79a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -69,6 +69,32 @@ jobs: name: dist path: dist/ + benchmarks: + name: Benchmarks + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + fetch-depth: 0 + - uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3 + - name: Run the benchmarks (fails on 4x the targets) + run: uv run --frozen --python 3.14 python tools/bench_bulk.py --json bench.json --check 2>&1 | tee bench.txt + - name: Add the results to the summary + if: always() + run: | + { + echo '### Benchmarks (4 MiB, median of 3 runs)' + echo '```' + cat bench.txt + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + if: always() + with: + name: benchmarks + path: bench.json + api-diff: name: Public API changes (report only) if: github.event_name == 'pull_request' diff --git a/tools/bench_bulk.py b/tools/bench_bulk.py index 556050da..b49b9c7b 100755 --- a/tools/bench_bulk.py +++ b/tools/bench_bulk.py @@ -3,16 +3,18 @@ Connection-layer benchmarks on loopback (no real devices needed). Measures ``execute()`` with a large response over Telnet and SSH, NETCONF 1.1 -chunk reassembly, and the SSH handshakes needed for one login when the IOS -driver is detected. Run from the repository root:: +chunk reassembly, the SSH handshakes needed for one login when the IOS driver +is detected, and the time of ``import Exscript``. With --check, it fails if a +result exceeds its limit in LIMITS. Run from the repository root:: - python tools/bench_bulk.py [--mib 4] [--runs 3] [--json results.json] + python tools/bench_bulk.py [--mib 4] [--runs 3] [--json results.json] [--check] """ import argparse import json import socket import statistics +import subprocess import sys import threading import time @@ -141,6 +143,25 @@ def bench_ios_login(host_key): return elapsed, server.connections, driver +def bench_import(): + """Returns the seconds that ``import Exscript`` takes in a new interpreter.""" + # The arguments are constant: this interpreter, importing Exscript. + command = [sys.executable, '-X', 'importtime', '-c', 'import Exscript'] + output = subprocess.run(command, capture_output=True, text=True, check=True).stderr # noqa: S603 + return int(output.strip().splitlines()[-1].split('|')[1]) / 1e6 # cumulative microseconds + + +# Limits for --check, per MiB where they scale: loose enough that slower CI +# machines pass, but quadratic regressions fail. +LIMITS = { + 'telnet_execute_cpu_s': lambda mib: 0.5 * mib, + 'ssh_execute_cpu_s': lambda mib: 0.3 * mib, + 'netconf_reassembly_s': lambda mib: 0.025 * mib, + 'ios_login_handshakes': lambda mib: 3, + 'import_s': lambda mib: 0.5, +} + + def main(): parser = argparse.ArgumentParser(description=__doc__.split('\n\n')[0]) parser.add_argument('--mib', type=int, default=4, help='response size in MiB (default: 4)') @@ -148,6 +169,7 @@ def main(): '--runs', type=int, default=3, help='runs per measurement, median is reported' ) parser.add_argument('--json', help='also write the results to this file') + parser.add_argument('--check', action='store_true', help='fail if a result exceeds its limit') args = parser.parse_args() size = args.mib * 1024 * 1024 payload = LINE * (size // len(LINE) + 1) @@ -175,12 +197,21 @@ def main(): results['ios_login_s'] = round(elapsed, 2) results['ios_login_handshakes'] = handshakes results['ios_login_driver'] = driver + results['import_s'] = round(statistics.median(bench_import() for _ in range(args.runs)), 3) for key, value in results.items(): print('%-28s %s' % (key, value)) if args.json: with open(args.json, 'w', encoding='utf-8') as fp: json.dump(results, fp, indent=1) + if args.check: + exceeded = [ + '%s: %s > %s' % (key, results[key], limit(args.mib)) + for key, limit in LIMITS.items() + if results[key] > limit(args.mib) + ] + if exceeded: + sys.exit('Limits exceeded:\n ' + '\n '.join(exceeded)) if __name__ == '__main__': From 7e5300450c03639581edd1cc99242a1852d0e511 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 3 Nov 2026 06:12:50 +0100 Subject: [PATCH 038/250] Load lxml and ncclient only when NETCONF needs them Importing lxml enables the GIL on free-threaded Python, and Exscript imported it with the package. util.xml also creates a parser per call, as lxml lets only one thread at a time use a parser. --- .github/workflows/ci.yml | 5 ++- CHANGES.md | 5 +++ src/Exscript/interpreter/execute.py | 4 +- src/Exscript/protocols/netconf.py | 54 ++++++++++++++++--------- src/Exscript/util/xml.py | 28 +++++++------ tests/Exscript/protocols/NetconfTest.py | 17 +++++++- 6 files changed, 77 insertions(+), 36 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9b57e79a..e8ef90f2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,7 +38,9 @@ jobs: python: ["3.12", "3.13", "3.14", "3.15"] experimental: [false] include: - - { os: ubuntu-latest, python: "3.14t", experimental: true } + # lxml has not declared support for free-threading, so importing it enables the GIL; + # PYTHON_GIL=0 keeps it off, so that this job tests without the GIL. + - { os: ubuntu-latest, python: "3.14t", experimental: true, gil: "0" } - { os: windows-latest, python: "3.14", experimental: true } - { os: macos-latest, python: "3.14", experimental: true } steps: @@ -51,6 +53,7 @@ jobs: shell: bash # also on Windows, where the default shell does not expand "$PYTHON" env: PYTHON: ${{ matrix.python }} + PYTHON_GIL: ${{ matrix.gil }} # empty, i.e. ignored, except for free-threading run: uv run --frozen --python "$PYTHON" pytest --cov=Exscript --cov-report=term-missing:skip-covered build: diff --git a/CHANGES.md b/CHANGES.md index 60e7c5d4..9474e94f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -175,6 +175,11 @@ marked with **BREAKING**; a migration guide will accompany the release. position at which the thread wrote. Exscript still sends its `` and RPCs pretty-printed, but no longer makes other users of ncclient in the same process do so (changes against the 2.9 development versions). +- **Free-Threaded Python:** `import Exscript` no longer imports lxml and ncclient; + NETCONF sessions and templates with RPCs load them when they need them. lxml + enables the GIL of free-threaded Python (3.14t), so before, Exscript always ran with + the GIL there. XML is parsed with a new parser for each call, so threads no longer + wait for each other. ### Telnet diff --git a/src/Exscript/interpreter/execute.py b/src/Exscript/interpreter/execute.py index 6543986d..960e7f97 100644 --- a/src/Exscript/interpreter/execute.py +++ b/src/Exscript/interpreter/execute.py @@ -23,8 +23,6 @@ import re import textwrap -from lxml.etree import XMLSyntaxError - from ..parselib import Token from ..util.xml import validate_xml from .string import String, varstring_re @@ -71,6 +69,8 @@ def __init__(self, lexer, parser, parent, command, escape_offset=0): # to create an XML tree as valid as possible. The XML body will be # tested once more at runtime by the protocol adapter (evaluated vars). if type(parent).__name__ == 'RPCBody': + from lxml.etree import XMLSyntaxError # only templates with RPCs need lxml + # The start mark points to the first non-whitespace character of the # RPC body. We will need the original start position of the command. original_start = self.start - len( diff --git a/src/Exscript/protocols/netconf.py b/src/Exscript/protocols/netconf.py index fa821af8..5652161e 100644 --- a/src/Exscript/protocols/netconf.py +++ b/src/Exscript/protocols/netconf.py @@ -32,15 +32,6 @@ import re import time -import ncclient.operations.rpc -import ncclient.transport.session -from lxml.etree import XMLSyntaxError -from ncclient import NCClientError -from ncclient.manager import Manager, make_device_handler -from ncclient.operations.errors import TimeoutExpiredError -from ncclient.operations.rpc import RaiseMode, RPCReply -from ncclient.transport.ssh import SSHSession -from ncclient.xml_ import NCElement, to_xml from paramiko.ssh_exception import SSHException from ..util.xml import XMLValidationError, _parse, validate_xml @@ -51,6 +42,9 @@ ) from .ssh2 import SSH2 +# ncclient and lxml are imported when the first NETCONF session starts, not +# with Exscript: lxml enables the GIL of free-threaded Python. + _logger = logging.getLogger(__name__) _logger.addHandler(logging.NullHandler()) @@ -59,21 +53,28 @@ _end_of_chunks_re = re.compile(r'\n##\n') # Exscript sends its and RPCs pretty-printed, so that they are readable -# in logs. ncclient serializes them with the to_xml() of these two modules, in -# the thread that sends; the replacement pretty-prints only within _pretty(), -# so other users of ncclient in the process are not affected. +# in logs. ncclient serializes them with the to_xml() of two of its modules, in +# the thread that sends; _install_to_xml() replaces it with a function that +# pretty-prints only within _pretty(), so other users of ncclient in the +# process are not affected. _pretty_print = contextvars.ContextVar('exscript_netconf_pretty_print', default=False) -@functools.wraps(to_xml) -def _to_xml(*args, **kwargs): - if _pretty_print.get(): - kwargs['pretty_print'] = True - return to_xml(*args, **kwargs) +@functools.cache +def _install_to_xml(): + import ncclient.operations.rpc + import ncclient.transport.session + from ncclient.xml_ import to_xml + @functools.wraps(to_xml) + def _to_xml(*args, **kwargs): + if _pretty_print.get(): + kwargs['pretty_print'] = True + return to_xml(*args, **kwargs) -ncclient.operations.rpc.to_xml = _to_xml -ncclient.transport.session.to_xml = _to_xml + ncclient.operations.rpc.to_xml = _to_xml + ncclient.transport.session.to_xml = _to_xml + return _to_xml @contextlib.contextmanager @@ -168,6 +169,9 @@ def get_response_element(self): :rtype: lxml.etree.Element | None """ + from ncclient.operations.rpc import RPCReply + from ncclient.xml_ import NCElement + if self.response is None: return None # send() was called without expect_prompt() if isinstance(self._ncc_response, RPCReply): @@ -192,6 +196,8 @@ def _paramiko_channel(self): # This is a bit different from the SSH2 adapter. Here we need to init # the Netconf subsystem. We integrate ncclient's device params in the # drivers to support customization. + from ncclient.manager import make_device_handler + driver = self.get_driver() device_handler = make_device_handler(driver.netconf_device_params) subsystems = device_handler.get_ssh_subsystem_names() @@ -219,6 +225,11 @@ def _init_ncclient(self): # complete the exchange and start ncclient's IO thread. if self._ncc_manager: return + from ncclient.manager import Manager + from ncclient.operations.rpc import RaiseMode + from ncclient.transport.ssh import SSHSession + + _install_to_xml() self._dbg(1, 'Initialize ncclient and hand over channel ownership.') # Build a bare SSHSession — no connect() call, we inject everything. @@ -390,6 +401,9 @@ def _fill_buffer(self, timeout): return True def _send(self, data): + from ncclient import NCClientError + from ncclient.operations.errors import TimeoutExpiredError + self._ncc_response = None self._validate_rpc_body(data) try: @@ -408,6 +422,8 @@ def _send(self, data): self._dbg(4, 'Received response type: %s' % type(self._ncc_response).__name__) def _validate_rpc_body(self, data): + from lxml.etree import XMLSyntaxError + _wrapper_start_tag = '\n' _wrapper_end_tag = '\n' data = f'{_wrapper_start_tag}{data}{_wrapper_end_tag}' diff --git a/src/Exscript/util/xml.py b/src/Exscript/util/xml.py index 3a335f7f..2eb670a3 100644 --- a/src/Exscript/util/xml.py +++ b/src/Exscript/util/xml.py @@ -22,23 +22,25 @@ # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ Working with XML. + +lxml is imported on first use, not with Exscript: it enables the GIL of +free-threaded Python. """ -from lxml import etree -# Entities are never resolved and DTDs and other resources are never loaded, -# which rules out XXE attacks; also with huge_tree, which only lifts size limits. -_parsers = { - huge_tree: etree.XMLParser( +def _parse(string, huge_tree=False): + """ + Parses an XML string and returns the root element, safe from XXE + attacks: the parser never resolves entities or loads DTDs or other + resources (huge_tree only lifts size limits). Each call creates its own + parser, because lxml lets only one thread at a time use a parser. + """ + from lxml import etree + + parser = etree.XMLParser( resolve_entities=False, load_dtd=False, no_network=True, huge_tree=huge_tree ) - for huge_tree in (False, True) -} - - -def _parse(string, huge_tree=False): - """Parses an XML string with the safe parser and returns the root element.""" - return etree.fromstring(string.encode('utf-8'), _parsers[huge_tree]) + return etree.fromstring(string.encode('utf-8'), parser) class XMLValidationError(Exception): @@ -65,6 +67,8 @@ def validate_xml(string, *, silent=False): :rtype: bool :return: Whether the string is valid XML in UTF-8. """ + from lxml import etree + try: _parse(string) except UnicodeEncodeError: diff --git a/tests/Exscript/protocols/NetconfTest.py b/tests/Exscript/protocols/NetconfTest.py index aeb404d2..e61ee7c5 100644 --- a/tests/Exscript/protocols/NetconfTest.py +++ b/tests/Exscript/protocols/NetconfTest.py @@ -1,6 +1,8 @@ """Tests for the parts of the NETCONF adapter that need no server.""" import inspect +import subprocess +import sys import threading import unittest @@ -58,8 +60,9 @@ def testReplyOfATransformedResult(self): self.assertIs(NCElement(reply, lambda root: root)._NCElement__result, reply) def testPrettyPrintOnlyForExscript(self): - self.assertIs(ncclient.operations.rpc.to_xml, netconf._to_xml) - self.assertIs(ncclient.transport.session.to_xml, netconf._to_xml) + to_xml = netconf._install_to_xml() + self.assertIs(ncclient.operations.rpc.to_xml, to_xml) + self.assertIs(ncclient.transport.session.to_xml, to_xml) rpc = RPC.__new__(RPC) # _wrap() needs only these two attributes rpc._id, rpc._device_handler = '1', self.handler for serialize in ( @@ -70,3 +73,13 @@ def testPrettyPrintOnlyForExscript(self): with netconf._pretty(): self.assertIn('\n <', serialize()) self.assertNotIn('\n <', serialize()) + + +class LazyImportTest(unittest.TestCase): + def testImportLoadsNoLxml(self): + # lxml enables the GIL of free-threaded Python, so only NETCONF and RPC templates load it. + code = 'import sys, Exscript.protocols; print([m for m in ("lxml", "ncclient") if m in sys.modules])' + result = subprocess.run( # noqa: S603 (constant arguments) + [sys.executable, '-c', code], capture_output=True, text=True, check=True + ) + self.assertEqual(result.stdout.strip(), '[]') From d94d879d987961e08c71f4253d17fe39b57976ca Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 3 Nov 2026 16:41:46 +0100 Subject: [PATCH 039/250] Give automated sessions a fixed, wide terminal Sessions got the size of the local terminal, or 80x25 without one, so the output depended on where a script ran, and long output was wrapped or paged. They now get 511x1000 like netmiko's; the new options term_width and term_height choose another size. --- CHANGES.md | 9 ++++++ src/Exscript/host.py | 4 +++ src/Exscript/protocols/protocol.py | 15 +++++++++ src/Exscript/protocols/ssh2.py | 4 +-- src/Exscript/protocols/telnet.py | 4 +-- tests/Exscript/protocols/SSH2Test.py | 40 ++++++++++++++++++++++- tests/Exscript/protocols/TelnetTest.py | 44 +++++++++++++++++++++++++- 7 files changed, 112 insertions(+), 8 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 9474e94f..ef262cf3 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -115,6 +115,15 @@ marked with **BREAKING**; a migration guide will accompany the release. keyboard-interactive took 5 SSH connections, now 3. - **All Addresses for SSH:** SSH and NETCONF connections try all addresses of the host, e.g. IPv4 if IPv6 fails, as Telnet does; before, they tried only the first. +- **BREAKING - Terminal Size:** SSH and Telnet sessions get a terminal of 511 columns + and 1000 lines; the new options `term_width` and `term_height` (also `Host` + options) choose another size. Before, Exscript used the size of the terminal that + ran the script, or 80×25 without one (cron, services), so the same script could + get differently wrapped output. Where the driver does not set the width or turn + off paging itself, e.g. on Linux hosts, NX-OS, EOS, VRP or unknown devices, long + lines are no longer wrapped or cut at 80 characters, and pagers stop only after + 1000 lines. `interact()` still uses the size of the local terminal. For the + previous behavior without a terminal, pass `term_width=80, term_height=25`. - **Reliable `close()`:** `close()` closes the connection and resets the object even if the logout fails, and raises the error of the logout afterwards. Errors while closing, e.g. because the host closed the connection first, are only logged; diff --git a/src/Exscript/host.py b/src/Exscript/host.py index 9df512fb..4aef7e61 100644 --- a/src/Exscript/host.py +++ b/src/Exscript/host.py @@ -235,6 +235,8 @@ def set_option(self, name, value): host_key_policy: str # SSH2 only known_hosts: str # SSH2 only preferred_authentications: list[str]|None # SSH2 only + term_height: int + term_width: int termtype: str timeout: int verify_fingerprint: bool # SSH2 only @@ -257,6 +259,8 @@ def set_option(self, name, value): 'host_key_policy', 'known_hosts', 'preferred_authentications', + 'term_height', + 'term_width', 'termtype', 'timeout', 'verify_fingerprint', diff --git a/src/Exscript/protocols/protocol.py b/src/Exscript/protocols/protocol.py index b4716f3f..ec619d92 100644 --- a/src/Exscript/protocols/protocol.py +++ b/src/Exscript/protocols/protocol.py @@ -236,6 +236,8 @@ def __init__( *, host_key_policy='reject', # SSH2 only known_hosts=None, # SSH2 only + term_width=511, + term_height=1000, ): """ Constructor. @@ -264,6 +266,17 @@ def __init__( device is dumped. :keyword termtype: The terminal type to request from the remote host, e.g. 'vt100'. + :type term_width: int + :keyword term_width: The width of the terminal on the remote host, in + characters, for SSH and Telnet. Hosts wrap or cut longer lines + unless the driver changes the width, e.g. on Linux or NX-OS. The + default, 511, is the widest that many devices accept. The size + does not depend on the terminal that the script runs in; + interact() changes it to the size of that terminal. + :type term_height: int + :keyword term_height: The height of the terminal on the remote host, + in lines. Hosts may wait for a key press after this many lines + unless the driver turns paging off. :type disabled_algorithms: dict[str, list[str]]|None :keyword disabled_algorithms: Dictionary to disable SSH algorithms via 'paramiko.Transport(disabled_algorithms=...)' constructor. @@ -330,6 +343,8 @@ def __init__( self.stdout = stdout self.stderr = stderr self.termtype = termtype + self.term_width = term_width + self.term_height = term_height self.timeout = timeout # SSH2: diff --git a/src/Exscript/protocols/ssh2.py b/src/Exscript/protocols/ssh2.py index 483f3090..ddf299a4 100644 --- a/src/Exscript/protocols/ssh2.py +++ b/src/Exscript/protocols/ssh2.py @@ -42,7 +42,6 @@ from ..account import Account from ..key import PrivateKey from ..util.crypt import otp -from ..util.tty import get_terminal_size from .exception import LoginFailure from .protocol import Protocol, _skey_re @@ -363,10 +362,9 @@ def _paramiko_auth(self, username, password): raise LoginFailure('Login failed: ' + '; '.join(errors)) def _paramiko_channel(self): - rows, cols = get_terminal_size() try: self._channel = self._transport.open_session() - self._channel.get_pty(self.termtype, cols, rows) + self._channel.get_pty(self.termtype, self.term_width, self.term_height) self._channel.invoke_shell() self._channel.set_name("pty-shell-%s" % self._channel.get_id()) except SSHException as e: diff --git a/src/Exscript/protocols/telnet.py b/src/Exscript/protocols/telnet.py index 0077bbb9..ab69222d 100644 --- a/src/Exscript/protocols/telnet.py +++ b/src/Exscript/protocols/telnet.py @@ -26,7 +26,6 @@ import logging -from ..util.tty import get_terminal_size from . import telnetlib from .protocol import Protocol @@ -52,13 +51,12 @@ def _telnetlib_received(self, data): def _connect(self): assert self._tn is None - rows, cols = get_terminal_size() self._tn = telnetlib.Telnet( self.host, self.port, encoding=self.encoding, connect_timeout=self.connect_timeout, - termsize=(rows, cols), + termsize=(self.term_height, self.term_width), termtype=self.termtype, receive_callback=self._telnetlib_received, ) diff --git a/tests/Exscript/protocols/SSH2Test.py b/tests/Exscript/protocols/SSH2Test.py index d32742f2..dabe6832 100644 --- a/tests/Exscript/protocols/SSH2Test.py +++ b/tests/Exscript/protocols/SSH2Test.py @@ -7,7 +7,7 @@ import paramiko -from Exscript import PrivateKey +from Exscript import Account, PrivateKey from Exscript.protocols import SSH2, Netconf from Exscript.protocols.ssh2 import _prefer_key_type from Exscript.servers import SSHd, sshd @@ -224,6 +224,43 @@ def testTriesAllAddresses(self): self.assertTrue(conn.connected) +class _Shell(_PasswordOnly): + """Opens a shell with a terminal, and records the size of each terminal requested.""" + + sizes: list[tuple[int, int]] = [] + + def check_channel_request(self, kind, chanid): + return paramiko.OPEN_SUCCEEDED + + def check_channel_pty_request( + self, channel, term, width, height, pixelwidth, pixelheight, modes + ): + self.sizes.append((width, height)) + return True + + def check_channel_shell_request(self, channel): + return True + + +class TerminalSizeTest(unittest.TestCase): + """The terminal size does not depend on the terminal that runs the script.""" + + def requestedSize(self, **kwargs): + interface = type('Shell', (_Shell,), {'sizes': []}) + server = KeyServer(self, paramiko.RSAKey(filename=keyfile), interface=interface) + conn = SSH2(timeout=1, connect_rate_limit=0, verify_fingerprint=False, **kwargs) + self.addCleanup(conn.close, force=True) + conn.connect(HOST, server.port) + conn.protocol_authenticate(Account('admin', 'password', needs_lock=False)) + return interface.sizes + + def testDefault(self): + self.assertEqual(self.requestedSize(), [(511, 1000)]) + + def testOptions(self): + self.assertEqual(self.requestedSize(term_width=80, term_height=25), [(80, 25)]) + + def suite(): loader = unittest.TestLoader() return unittest.TestSuite( @@ -232,6 +269,7 @@ def suite(): loader.loadTestsFromTestCase(HostKeyTest), loader.loadTestsFromTestCase(ReconnectTest), loader.loadTestsFromTestCase(ConnectTest), + loader.loadTestsFromTestCase(TerminalSizeTest), ] ) diff --git a/tests/Exscript/protocols/TelnetTest.py b/tests/Exscript/protocols/TelnetTest.py index c720fdb9..e501488b 100644 --- a/tests/Exscript/protocols/TelnetTest.py +++ b/tests/Exscript/protocols/TelnetTest.py @@ -1,6 +1,9 @@ +import socket +import threading import unittest from Exscript.protocols import Telnet +from Exscript.protocols.telnetlib import DO, IAC, NAWS, SB, SE, WILL from Exscript.servers import Telnetd from .ProtocolTest import ProtocolTest @@ -20,8 +23,47 @@ def testConstructor(self): self.assertIsInstance(self.protocol, Telnet) +class TerminalSizeTest(unittest.TestCase): + """The size sent with NAWS does not depend on the terminal that runs the script.""" + + def sentSize(self, **kwargs): + listener = socket.create_server(('127.0.0.1', 0)) + self.addCleanup(listener.close) + received = bytearray() + + def serve(): + sock, _ = listener.accept() + with sock: + sock.sendall(bytes([IAC, DO, NAWS]) + b'ready') + while not received.endswith(bytes([IAC, SE])) and (data := sock.recv(64)): + received.extend(data) + + server = threading.Thread(target=serve) + server.start() + conn = Telnet(timeout=2, connect_rate_limit=0, **kwargs) + self.addCleanup(conn.close, force=True) + conn.connect('127.0.0.1', listener.getsockname()[1]) + conn.expect('ready') # processes DO NAWS + server.join(timeout=5) + return bytes(received) + + def testDefault(self): + # 511 columns = 0x01FF, so the byte 255 is doubled; 1000 rows = 0x03E8. + naws = bytes([IAC, SB, NAWS, 0x01, 0xFF, 0xFF, 0x03, 0xE8, IAC, SE]) + self.assertEqual(self.sentSize(), bytes([IAC, WILL, NAWS]) + naws) + + def testOptions(self): + naws = bytes([IAC, SB, NAWS, 0, 80, 0, 25, IAC, SE]) + self.assertEqual( + self.sentSize(term_width=80, term_height=25), bytes([IAC, WILL, NAWS]) + naws + ) + + def suite(): - return unittest.TestLoader().loadTestsFromTestCase(TelnetTest) + loader = unittest.TestLoader() + return unittest.TestSuite( + [loader.loadTestsFromTestCase(TelnetTest), loader.loadTestsFromTestCase(TerminalSizeTest)] + ) if __name__ == '__main__': From 281dfa8687b04522a53628a616faae21e9490b8c Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 3 Nov 2026 16:45:37 +0100 Subject: [PATCH 040/250] Record with Exscript's terminal size and replay with the recorded one The recorder asked for 80x24, so recordings would show wrapping and paging that Exscript does not get. Its Telnet NAWS now doubles the byte 255, which a width of 511 needs. --- tests/Exscript/protocols/ReplayTest.py | 7 +++++-- tests/recordings/README.md | 4 ++-- tests/recordings/commands/ios-pager.txt | 2 ++ tests/recordings/commands/junos-pager.txt | 2 ++ tools/record_session.py | 12 +++++++++--- 5 files changed, 20 insertions(+), 7 deletions(-) diff --git a/tests/Exscript/protocols/ReplayTest.py b/tests/Exscript/protocols/ReplayTest.py index b31ef004..de087dc4 100644 --- a/tests/Exscript/protocols/ReplayTest.py +++ b/tests/Exscript/protocols/ReplayTest.py @@ -101,8 +101,11 @@ def replay(path, chunks): return _results[path, chunks] recording = Recording(path) options = {'timeout': 2, 'connect_timeout': 2, 'connect_rate_limit': 0} - if recording.header.get('terminal'): - options['termtype'] = recording.header['terminal']['type'] + terminal = recording.header.get('terminal') + if terminal: # request what the device was asked for when recorded + options.update( + termtype=terminal['type'], term_width=terminal['width'], term_height=terminal['height'] + ) if recording.protocol != 'telnet': methods = (recording.header.get('ssh') or {}).get('auth_methods') or [] options['verify_fingerprint'] = False diff --git a/tests/recordings/README.md b/tests/recordings/README.md index 57653426..9b6821f1 100644 --- a/tests/recordings/README.md +++ b/tests/recordings/README.md @@ -26,8 +26,8 @@ runs must give the same result. A third test checks the device's ground truth: l - The protocol is `ssh`, `telnet` or `netconf`. - `--os` is the ground truth: the Exscript driver name that `guess_os()` should return (`unknown` if no driver fits). - - Keep the default terminal type `dumb`. It is Exscript's default, and the replay test uses the - recorded type. + - Keep the default terminal type `dumb` and size 511×1000. They are Exscript's defaults, and + the replay test uses the recorded values. - Use `--auth password` for SSH servers that drop the connection after a failed keyboard-interactive attempt. - See `--help` for the other options. diff --git a/tests/recordings/commands/ios-pager.txt b/tests/recordings/commands/ios-pager.txt index 2b022469..2be6dcb8 100644 --- a/tests/recordings/commands/ios-pager.txt +++ b/tests/recordings/commands/ios-pager.txt @@ -1,4 +1,6 @@ # Cisco IOS, IOS XE and IOS XR (R2): output with the pager on. +# The recorder asks for 1000 lines, so set the pager length explicitly. +terminal length 24 show running-config !send \x20 !send \x20 diff --git a/tests/recordings/commands/junos-pager.txt b/tests/recordings/commands/junos-pager.txt index ead18383..b500babd 100644 --- a/tests/recordings/commands/junos-pager.txt +++ b/tests/recordings/commands/junos-pager.txt @@ -1,4 +1,6 @@ # Juniper Junos (R2): output with the pager on. +# The recorder asks for 1000 lines, so set the pager length explicitly. +set cli screen-length 24 show configuration !send \x20 !send \x20 diff --git a/tools/record_session.py b/tools/record_session.py index 4af0ca49..2fa37deb 100755 --- a/tools/record_session.py +++ b/tools/record_session.py @@ -29,6 +29,7 @@ import os import re import socket +import struct import sys import time @@ -105,7 +106,8 @@ class TelnetLink(Link): def __init__(self, conn, recording, term, width, height): super().__init__(conn, recording) self.term = term.encode('ascii') - self.size = bytes([width >> 8, width & 255, height >> 8, height & 255]) + # The byte 255 (e.g. in a width of 511) is doubled, like all IAC bytes in data. + self.size = struct.pack('!HH', width, height).replace(b'\xff', b'\xff\xff') self.pending = b'' self.answered = set() @@ -321,8 +323,12 @@ def main(): parser.add_argument( '--term', default='dumb', help="terminal type (default: Exscript's default)" ) - parser.add_argument('--width', type=int, default=80) - parser.add_argument('--height', type=int, default=24) + parser.add_argument( + '--width', type=int, default=511, help='terminal width (default: 511, like Exscript)' + ) + parser.add_argument( + '--height', type=int, default=1000, help='terminal height (default: 1000, like Exscript)' + ) parser.add_argument('-y', '--yes', action='store_true', help='accept unknown SSH host keys') args = parser.parse_args() if args.protocol == 'netconf' and not args.port: From 050e9e75d26d6104153f31ca0d34a127d9cb01c9 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 3 Nov 2026 16:49:46 +0100 Subject: [PATCH 041/250] Fail the benchmark job when a limit is exceeded The step piped the check into tee, and without an explicit shell, GitHub runs bash without pipefail, so only tee's exit status counted. shell: bash adds pipefail. --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e8ef90f2..d2ebe871 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -82,6 +82,7 @@ jobs: fetch-depth: 0 - uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3 - name: Run the benchmarks (fails on 4x the targets) + shell: bash # with pipefail; the default shell lets tee hide a failure run: uv run --frozen --python 3.14 python tools/bench_bulk.py --json bench.json --check 2>&1 | tee bench.txt - name: Add the results to the summary if: always() From 42497a84ef124c9122061ae98bb0a68b92d82703 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 3 Nov 2026 16:49:59 +0100 Subject: [PATCH 042/250] Report the OS-guess accuracy in CI tools/osguess_report.py compares guess_os() with the known OS of each banner and recording. The CI job fails if fewer sessions are guessed right than MIN_CORRECT, so the accuracy only goes up. Today it is 83%. --- .github/workflows/ci.yml | 13 ++++++ tests/recordings/README.md | 4 +- tools/osguess_report.py | 90 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 106 insertions(+), 1 deletion(-) create mode 100644 tools/osguess_report.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d2ebe871..90baaaa4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -99,6 +99,19 @@ jobs: name: benchmarks path: bench.json + os-guessing: + name: OS-guess accuracy + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + fetch-depth: 0 + - uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3 + - name: Compare the guesses with the ground truth (fails if fewer are right) + shell: bash # with pipefail; the default shell lets tee hide a failure + run: uv run --frozen --python 3.14 python tools/osguess_report.py --check | tee -a "$GITHUB_STEP_SUMMARY" + api-diff: name: Public API changes (report only) if: github.event_name == 'pull_request' diff --git a/tests/recordings/README.md b/tests/recordings/README.md index 9b6821f1..775ca7e5 100644 --- a/tests/recordings/README.md +++ b/tests/recordings/README.md @@ -7,7 +7,9 @@ recorded device output, so the tests need no devices. Each recording runs twice: with the output as recorded, and split into small random pieces. Both runs must give the same result. A third test checks the device's ground truth: login succeeds and -`guess_os()` returns `device.os`. +`guess_os()` returns `device.os`. `tools/osguess_report.py` compares the guesses for all recordings +and for the banners in `tests/Exscript/protocols/banners` with their ground truth; CI shows its +report in the job summary and fails if fewer sessions are guessed right than before. ## Adding a recording diff --git a/tools/osguess_report.py b/tools/osguess_report.py new file mode 100644 index 00000000..84c37784 --- /dev/null +++ b/tools/osguess_report.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python3 +""" +OS-guess accuracy: compares guess_os() with the ground truth, and prints the +result per OS and the wrong guesses in Markdown. + +The truth of a banner in tests/Exscript/protocols/banners is its file name +(``ios.3``: ios). Each banner is replayed through the OS guesser like a Telnet +session and like an SSH session whose version string names no vendor, in small +and large reads. The truth of a recording in tests/recordings is ``device.os`` +in its header; recordings are replayed against the real adapters. + +With --check, it fails if fewer sessions are right than MIN_CORRECT, so the +accuracy only goes up; raise MIN_CORRECT when a change improves it. Run from +the repository root:: + + python tools/osguess_report.py [--check] +""" + +import argparse +import collections +import glob +import os +import sys + +sys.path.insert(0, os.getcwd()) # for the test helpers + +from tests.Exscript.protocols import OsGuesserCorpusTest as corpus # noqa: E402 +from tests.Exscript.protocols import ReplayTest as replays # noqa: E402 +from tests.Exscript.protocols.replay import Recording # noqa: E402 + +MIN_CORRECT = 204 # 2026-11-03; raise it when a change guesses more sessions right + + +def banner_sessions(): + """Yields (session, truth, guess) for each banner, protocol and read size.""" + results = {} + for name in sorted(os.listdir(corpus.BANNER_DIR)): + if name.startswith('.'): + continue + with open(os.path.join(corpus.BANNER_DIR, name), encoding='utf-8') as fp: + banner = fp.read().rstrip('\n') + last_line = banner.replace('\r', '\n').rstrip('\n').split('\n')[-1] + after_auth = '\r\n' + corpus.GENERIC_OUTPUT + '\r\n' + last_line + for info in '', 'SSH-2.0-OpenSSH_9.6': + corpus.replay(name, banner, info, after_auth, results) + for key, result in sorted(results.items()): + name, info, size = key.split('|') + protocol = 'Telnet' if info == '-' else 'SSH' + yield f'banner {name}, {protocol}, {size}-byte reads', name.split('.')[0], result['final'][0] + + +def recording_sessions(): + """Yields (session, truth, guess) for each recording with a ground truth.""" + for path in sorted(glob.glob(os.path.join(replays.RECORDINGS, '*', '*.jsonl'))): + truth = Recording(path).header['device'].get('os') + if truth: + name = os.path.relpath(path, replays.RECORDINGS).replace(os.sep, '/') + yield f'recording {name}', truth, replays.replay(path, 'recorded')[0]['os'] + + +def main(): + parser = argparse.ArgumentParser(description=__doc__.split('\n\n')[0]) + parser.add_argument('--check', action='store_true', help='fail below MIN_CORRECT') + args = parser.parse_args() + + sessions = [*banner_sessions(), *recording_sessions()] + right = sum(truth == guess for _, truth, guess in sessions) + per_os = collections.defaultdict(collections.Counter) + for _, truth, guess in sessions: + per_os[truth][guess] += 1 + + print('### OS-guess accuracy\n') + print(f'{right} of {len(sessions)} sessions guessed right ({right / len(sessions):.0%}).\n') + print('| Truth | Sessions | Right | Guessed instead |') + print('|---|---:|---:|---|') + for truth, guesses in sorted(per_os.items()): + instead = ', '.join(f'{os_} ({n})' for os_, n in guesses.most_common() if os_ != truth) + print(f'| {truth} | {guesses.total()} | {guesses[truth]} | {instead} |') + wrong = [(session, guess) for session, truth, guess in sessions if truth != guess] + if wrong: + print('\n
Wrong guesses\n') + for session, guess in wrong: + print(f'- {session}: {guess}') + print('\n
') + if args.check and right < MIN_CORRECT: + sys.exit(f'\nFewer sessions guessed right than before: {right} < {MIN_CORRECT}') + + +if __name__ == '__main__': + main() From bcbafcf36eb2883f6622b6223c01c6f103f3e5cf Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 3 Nov 2026 17:13:20 +0100 Subject: [PATCH 043/250] Stop weak signals from selecting the wrong driver isam claimed every "name#" and "name>" prompt, ericsson_ban and sp_vxoa every "Last login" line, and junos every "host login:", which other systems print, too. Now 213 instead of 204 of 245 sessions are guessed right, and isam uses its error patterns, which a typo had disabled. --- CHANGES.md | 20 + .../protocols/drivers/ericsson_ban.py | 7 - src/Exscript/protocols/drivers/isam.py | 6 +- src/Exscript/protocols/drivers/junos.py | 5 +- src/Exscript/protocols/drivers/shell.py | 15 +- src/Exscript/protocols/drivers/sp_vxoa.py | 6 - tests/Exscript/KnownIssuesTest.py | 46 -- tests/Exscript/protocols/OsGuesserTest.py | 64 ++- tests/Exscript/protocols/PromptRegexTest.py | 7 + tests/Exscript/protocols/ReplayTest.py | 13 +- .../Exscript/protocols/osguess-baseline.json | 408 +++++++++--------- .../frr/frr-10.4-vtysh-ssh.golden.json | 119 +++++ .../alpine-3.22-bash-ssh-vt100.golden.json | 4 +- .../linux/alpine-3.22-bash-ssh.golden.json | 4 +- .../linux/busybox-1.37-ash-telnet.golden.json | 57 ++- .../netopeer2/netopeer2-netconf.golden.json | 4 +- tools/osguess_report.py | 8 +- 17 files changed, 502 insertions(+), 291 deletions(-) delete mode 100644 tests/Exscript/KnownIssuesTest.py create mode 100644 tests/recordings/frr/frr-10.4-vtysh-ssh.golden.json diff --git a/CHANGES.md b/CHANGES.md index ef262cf3..352dc167 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -228,6 +228,26 @@ marked with **BREAKING**; a migration guide will accompany the release. checks every driver pattern for this. The isam driver's prompt pattern, which the OS guesser runs on all received data, matches the same strings 3.5 times faster. +- **BREAKING - More Accurate OS Guessing:** On the test corpus (60 device banners and + 5 recorded sessions), `guess_os()` is right for 213 of 245 sessions instead of 204, + and no guess that was right changed. Signals that other systems show, too, no + longer select a driver: + - `isam` claimed every `name#` and `name>` prompt, e.g. of FRR, NETCONF servers and + unknown devices, which then ran with ISAM's prompt pattern; now it claims only + ISAM's `name>#` prompts. Its error patterns are used now; a typo had disabled + them. + - `ericsson_ban` and `sp_vxoa` claimed every "Last login" line, so logins to Linux + hosts with a `$` prompt timed out. Their banner checks stay. + - `junos` claimed every `router1 login:` prompt, e.g. of Linux and BusyBox Telnet + logins; now only after the `(ttyp0)` line that Junos prints before `login:`. + - `shell` is also recognized by prompts like `router1:~$ ` or + `admin@router1:/etc# `. + + Devices whose banner has no vendor signature, e.g. some EOS, ADTRAN and RiOS + devices, are now guessed as `unknown` (generic driver) instead of a wrong driver. + The CI job "OS-guess accuracy" reports the accuracy per OS + (`tools/osguess_report.py`). Pass `driver=` if a device needs a specific driver. + - **BREAKING - Graceful Logout:** `Protocol.close(force=False)` now calls the driver's new `logout()` method before the connection is closed. The base `Driver` sends `exit\r`, the custom `VRPDriver` sends `quit\r` and `SROSDriver` sends `logout\r`. diff --git a/src/Exscript/protocols/drivers/ericsson_ban.py b/src/Exscript/protocols/drivers/ericsson_ban.py index 6b6bfae5..7cc2d016 100644 --- a/src/Exscript/protocols/drivers/ericsson_ban.py +++ b/src/Exscript/protocols/drivers/ericsson_ban.py @@ -37,8 +37,6 @@ _ban_re = re.compile(r"BLM\d+ - Broadband Loop Multiplexer", re.I) -_banner_re = re.compile(r"Last login", re.I) - _login_fail_re = [r'Login failed'] @@ -60,8 +58,3 @@ def check_head_for_os(self, string): if _ban_re.search(string): return 90 return 0 - - def check_response_for_os(self, string): - if _banner_re.search(string): - return 20 - return 0 diff --git a/src/Exscript/protocols/drivers/isam.py b/src/Exscript/protocols/drivers/isam.py index 4c86c8ca..dedf14ae 100644 --- a/src/Exscript/protocols/drivers/isam.py +++ b/src/Exscript/protocols/drivers/isam.py @@ -39,6 +39,8 @@ _isam_re = re.compile( r"last login : \d{1,2}/\d{1,2}/\d{1,2} \d{1,2}:\d{1,2}:\d{1,2}", ) +# ISAM prompts end with ">#", e.g. "typ:isadmin>#"; "name#" and "name>" are common. +_isam_prompt_re = re.compile(r'(?:\r\n|[\r\n])[\w:.\-]+>#\s*$') class IsamDriver(Driver): @@ -47,10 +49,10 @@ def __init__(self): self.user_re = _user_re self.password_re = _password_re self.prompt_re = _prompt_re - self._error_re = _error_re + self.error_re = _error_re def check_response_for_os(self, string): - if _prompt_re[0].search(string): + if _isam_prompt_re.search(string): return 20 return 0 diff --git a/src/Exscript/protocols/drivers/junos.py b/src/Exscript/protocols/drivers/junos.py index 91203aee..5ef1b8d9 100644 --- a/src/Exscript/protocols/drivers/junos.py +++ b/src/Exscript/protocols/drivers/junos.py @@ -61,6 +61,9 @@ _prompt_re = [re.compile(_prefix + r'?' + _prompt)] _error_re = [re.compile(r'^(unknown|invalid|error|syntax error)', re.I)] _junos_re = re.compile(r'\bjunos\b', re.I) +# Junos prints its BSD terminal name, e.g. "router1 (ttyp0)", before "login:". +# Linux and BusyBox print "router1 login:", so a login prompt alone is no sign. +_bsd_tty_re = re.compile(r'\(ttyp\d+\)') _netconf_caps_re = [re.compile(r"juniper\.net", re.I)] @@ -76,7 +79,7 @@ def __init__(self): def check_head_for_os(self, string): if _junos_re.search(string): return 80 - if _user_re[0].search(string): + if _bsd_tty_re.search(string): return 35 return 0 diff --git a/src/Exscript/protocols/drivers/shell.py b/src/Exscript/protocols/drivers/shell.py index 5a8f6b6c..85476f79 100644 --- a/src/Exscript/protocols/drivers/shell.py +++ b/src/Exscript/protocols/drivers/shell.py @@ -31,6 +31,14 @@ _user_re = [re.compile(r'(user|login): $', re.I)] _password_re = [re.compile(r'Password: ?$')] _linux_re = re.compile(r'\blinux\b', re.I) +# The guesser also sees the data that ends with "Last login: " for a moment +# when a read ends there. +_login_prompt_re = re.compile(r'(?#] ?$')] _error_re = [re.compile(r'^(unknown|invalid|error|syntax error)', re.I)] _banner_re = re.compile(r'Silver[\r\n\s]Peak[\r\n\s]Systems[\r\n\s]Inc\.') -_auth_banner_re = re.compile(r'Last login:\s') class SilverPeakVXOADriver(Driver): @@ -49,11 +48,6 @@ def check_head_for_os(self, string): return 90 return 0 - def check_response_for_os(self, string): - if _auth_banner_re.search(string): - return 20 - return 0 - def init_terminal(self, conn): conn.execute('terminal length 65536') diff --git a/tests/Exscript/KnownIssuesTest.py b/tests/Exscript/KnownIssuesTest.py deleted file mode 100644 index 6a7c5f77..00000000 --- a/tests/Exscript/KnownIssuesTest.py +++ /dev/null @@ -1,46 +0,0 @@ -""" -Regression net for known issues found during the 3.0 analysis. - -Every test describes the *correct* behavior and is marked as an expected -failure while the issue exists. Once a fix makes a test pass, unittest reports -an "unexpected success" and the suite fails, so the fix must remove the marker -and move the test next to the tests of the fixed module. -""" - -import sys -import unittest - -from Exscript import Account -from Exscript.emulators import VirtualDevice -from Exscript.protocols import Dummy -from Exscript.protocols.drivers import driver_map, isam - - -class DriverTest(unittest.TestCase): - @unittest.expectedFailure # assigned to self._error_re instead of self.error_re - def testIsamUsesItsErrorPatterns(self): - self.assertIs(driver_map['isam'].error_re, isam._error_re) - - @unittest.expectedFailure # any "Last login" line selects ericsson_ban, which never matches "$" - def testLastLoginLineDoesNotBreakLogin(self): - device = VirtualDevice( - 'host', - login_type=VirtualDevice.LOGIN_TYPE_NONE, - strict=False, - banner='Welcome\nLast login: Mon Oct 26 10:00:00 2026 from 192.0.2.1\n', - ) - device.set_prompt('user@host:~$ ') - device.add_command('uname', 'Linux') - conn = Dummy(device=device) - conn.connect('host') - conn.login(Account('user', 'password', needs_lock=False)) - conn.execute('uname') - self.assertEqual(conn.get_response().strip(), 'Linux') - - -def suite(): - return unittest.TestLoader().loadTestsFromModule(sys.modules[__name__]) - - -if __name__ == '__main__': - unittest.TextTestRunner(verbosity=2).run(suite()) diff --git a/tests/Exscript/protocols/OsGuesserTest.py b/tests/Exscript/protocols/OsGuesserTest.py index 433f77b8..bd52dbfe 100644 --- a/tests/Exscript/protocols/OsGuesserTest.py +++ b/tests/Exscript/protocols/OsGuesserTest.py @@ -2,7 +2,10 @@ import re import unittest -from Exscript.protocols import drivers +from Exscript import Account +from Exscript.emulators import VirtualDevice +from Exscript.protocols import Dummy, drivers +from Exscript.protocols.drivers import isam from Exscript.protocols.osguesser import OsGuesser @@ -100,8 +103,65 @@ def testCheckNetconfCapabilities(self): self.assertEqual(osg.get('os'), "junos") +def guess(head='', response='', protocol_info=''): + """Returns the guess after the given data, fed one character per read (the worst case).""" + guesser = OsGuesser() + guesser.check_protocol_info(protocol_info) + for char in head: + guesser.check_received_data(char, False) + for char in response: + guesser.check_received_data(char, True) + return guesser.get('os') + + +class SignalTest(unittest.TestCase): + """Weak signals must not claim what other systems print, too.""" + + def testLastLoginIsNoSign(self): + # Linux, EOS, IOS XR and many others print it, so no driver may claim it. + last_login = '\r\nLast login: Mon Oct 26 10:00:00 2026 from 192.0.2.1\r\n' + self.assertEqual(guess(head=last_login), 'unknown') + self.assertEqual(guess(response=last_login + 'switch>'), 'unknown') + + def testLoginPrompt(self): + self.assertEqual(guess(head='\r\nrouter3 login: '), 'shell') # Linux, BusyBox + self.assertEqual(guess(head='\r\nrouter1 (ttyp0)\r\n\r\nlogin: '), 'junos') + + def testShellPrompt(self): + for prompt in 'router1:~$ ', 'admin@router1:/etc# ', '\x1b[?2004hrouter1:~$ ': + self.assertEqual(guess(response='\r\nuname\r\nLinux\r\n' + prompt), 'shell', prompt) + for prompt in 'router#', 'RP/0/RSP0/CPU0:xr-pe1#', 'A:sros-pe1#', 'admin@junos-mx> ': + self.assertNotEqual(guess(response='\r\nshow\r\n' + prompt), 'shell', prompt) + + def testIsamPrompt(self): + self.assertEqual(guess(response='\r\ninfo\r\ntyp:isadmin>#'), 'isam') + for prompt in 'router#', 'switch>', 'hostname >', 'S>* 10.0.0.0/8 is directly connected': + self.assertEqual(guess(response='\r\nshow\r\n' + prompt), 'unknown', prompt) + + def testIsamUsesItsErrorPatterns(self): + self.assertIs(drivers.driver_map['isam'].error_re, isam._error_re) + + def testLastLoginLineDoesNotBreakLogin(self): + device = VirtualDevice( + 'host', + login_type=VirtualDevice.LOGIN_TYPE_NONE, + strict=False, + banner='Welcome\nLast login: Mon Oct 26 10:00:00 2026 from 192.0.2.1\n', + ) + device.set_prompt('user@host:~$ ') + device.add_command('uname', 'Linux') + conn = Dummy(device=device) + conn.connect('host') + conn.login(Account('user', 'password', needs_lock=False)) + conn.execute('uname') + self.assertEqual(conn.get_response().strip(), 'Linux') + + def suite(): - return unittest.TestLoader().loadTestsFromTestCase(OsGuesserTest) + loader = unittest.TestLoader() + return unittest.TestSuite( + [loader.loadTestsFromTestCase(OsGuesserTest), loader.loadTestsFromTestCase(SignalTest)] + ) if __name__ == '__main__': diff --git a/tests/Exscript/protocols/PromptRegexTest.py b/tests/Exscript/protocols/PromptRegexTest.py index 4f7f4542..fb2d64a0 100644 --- a/tests/Exscript/protocols/PromptRegexTest.py +++ b/tests/Exscript/protocols/PromptRegexTest.py @@ -89,3 +89,10 @@ def testAllRegexesNeedLinearTime(self): regex.search(text) elapsed = time.perf_counter() - start self.assertLess(elapsed, 0.2, f'{name}.{attr} on {text[:30]!r}') + # The OS guesser runs these on every read until it is sure. + for check in 'check_protocol_for_os', 'check_head_for_os', 'check_response_for_os': + for text in texts: + start = time.perf_counter() + getattr(driver, check)(text) + elapsed = time.perf_counter() - start + self.assertLess(elapsed, 0.2, f'{name}.{check} on {text[:30]!r}') diff --git a/tests/Exscript/protocols/ReplayTest.py b/tests/Exscript/protocols/ReplayTest.py index de087dc4..1ae32b44 100644 --- a/tests/Exscript/protocols/ReplayTest.py +++ b/tests/Exscript/protocols/ReplayTest.py @@ -21,16 +21,9 @@ UPDATE_GOLDEN = os.environ.get('EXSCRIPT_UPDATE_GOLDEN') == '1' ADAPTERS = {'ssh': SSH2, 'telnet': Telnet, 'netconf': Netconf} # Recordings that Exscript does not handle like the recorded device yet, and why. -KNOWN_ISSUES = { - 'linux/alpine-3.22-bash-ssh': 'bash prompts are not guessed as shell', - 'linux/alpine-3.22-bash-ssh-vt100': 'bash prompts are not guessed as shell', - 'linux/busybox-1.37-ash-telnet': '"router3 login:" is guessed as junos, whose prompt does not match', - 'netopeer2/netopeer2-netconf': 'the unanchored isam prompt regex matches NETCONF replies', -} -# Recordings whose results depend on the read sizes until the issue is fixed; their tests are skipped. -TIMING_ISSUES = { - 'frr/frr-10.4-vtysh-ssh': 'guessed as isam, whose unanchored prompt pattern matches route codes like "S>"', -} +KNOWN_ISSUES: dict[str, str] = {} +# Recordings whose results depend on read timing until the issue is fixed; their tests are skipped. +TIMING_ISSUES: dict[str, str] = {} _message_id = re.compile(r'message-id="[^"]*"') _results = {} diff --git a/tests/Exscript/protocols/osguess-baseline.json b/tests/Exscript/protocols/osguess-baseline.json index 59e475af..6f3b7092 100644 --- a/tests/Exscript/protocols/osguess-baseline.json +++ b/tests/Exscript/protocols/osguess-baseline.json @@ -121,22 +121,22 @@ }, "adtran.2|-|64": { "after_auth": [ - "ericsson_ban", - 20 + "unknown", + 0 ], "final": [ - "ericsson_ban", - 20 + "unknown", + 0 ] }, "adtran.2|-|65536": { "after_auth": [ - "ericsson_ban", - 20 + "unknown", + 0 ], "final": [ - "ericsson_ban", - 20 + "unknown", + 0 ] }, "adtran.2|SSH-2.0-Cisco-1.25|64": { @@ -161,22 +161,22 @@ }, "adtran.2|SSH-2.0-OpenSSH_9.6|64": { "after_auth": [ - "ericsson_ban", - 20 + "unknown", + 0 ], "final": [ - "ericsson_ban", - 20 + "unknown", + 0 ] }, "adtran.2|SSH-2.0-OpenSSH_9.6|65536": { "after_auth": [ - "ericsson_ban", - 20 + "unknown", + 0 ], "final": [ - "ericsson_ban", - 20 + "unknown", + 0 ] }, "aironet.1|-|64": { @@ -1261,22 +1261,22 @@ }, "eos.1|-|64": { "after_auth": [ - "junos", - 35 + "shell", + 20 ], "final": [ - "junos", - 35 + "shell", + 20 ] }, "eos.1|-|65536": { "after_auth": [ - "junos", - 35 + "shell", + 20 ], "final": [ - "junos", - 35 + "shell", + 20 ] }, "eos.1|SSH-2.0-Cisco-1.25|64": { @@ -1301,42 +1301,42 @@ }, "eos.1|SSH-2.0-OpenSSH_9.6|64": { "after_auth": [ - "junos", - 35 + "shell", + 20 ], "final": [ - "junos", - 35 + "shell", + 20 ] }, "eos.1|SSH-2.0-OpenSSH_9.6|65536": { "after_auth": [ - "junos", - 35 + "shell", + 20 ], "final": [ - "junos", - 35 + "shell", + 20 ] }, "eos.2|-|64": { "after_auth": [ - "junos", - 35 + "unknown", + 0 ], "final": [ - "junos", - 35 + "unknown", + 0 ] }, "eos.2|-|65536": { "after_auth": [ - "junos", - 35 + "unknown", + 0 ], "final": [ - "junos", - 35 + "unknown", + 0 ] }, "eos.2|SSH-2.0-Cisco-1.25|64": { @@ -1361,42 +1361,42 @@ }, "eos.2|SSH-2.0-OpenSSH_9.6|64": { "after_auth": [ - "junos", - 35 + "unknown", + 0 ], "final": [ - "junos", - 35 + "unknown", + 0 ] }, "eos.2|SSH-2.0-OpenSSH_9.6|65536": { "after_auth": [ - "junos", - 35 + "unknown", + 0 ], "final": [ - "junos", - 35 + "unknown", + 0 ] }, "eos.3|-|64": { "after_auth": [ - "junos", - 35 + "unknown", + 0 ], "final": [ - "junos", - 35 + "unknown", + 0 ] }, "eos.3|-|65536": { "after_auth": [ - "junos", - 35 + "unknown", + 0 ], "final": [ - "junos", - 35 + "unknown", + 0 ] }, "eos.3|SSH-2.0-Cisco-1.25|64": { @@ -1421,42 +1421,42 @@ }, "eos.3|SSH-2.0-OpenSSH_9.6|64": { "after_auth": [ - "junos", - 35 + "unknown", + 0 ], "final": [ - "junos", - 35 + "unknown", + 0 ] }, "eos.3|SSH-2.0-OpenSSH_9.6|65536": { "after_auth": [ - "junos", - 35 + "unknown", + 0 ], "final": [ - "junos", - 35 + "unknown", + 0 ] }, "eos.4|-|64": { "after_auth": [ - "isam", - 20 + "unknown", + 0 ], "final": [ - "isam", - 20 + "unknown", + 0 ] }, "eos.4|-|65536": { "after_auth": [ - "isam", - 20 + "unknown", + 0 ], "final": [ - "isam", - 20 + "unknown", + 0 ] }, "eos.4|SSH-2.0-Cisco-1.25|64": { @@ -1481,42 +1481,42 @@ }, "eos.4|SSH-2.0-OpenSSH_9.6|64": { "after_auth": [ - "isam", - 20 + "unknown", + 0 ], "final": [ - "isam", - 20 + "unknown", + 0 ] }, "eos.4|SSH-2.0-OpenSSH_9.6|65536": { "after_auth": [ - "isam", - 20 + "unknown", + 0 ], "final": [ - "isam", - 20 + "unknown", + 0 ] }, "eos.5|-|64": { "after_auth": [ - "junos", - 35 + "unknown", + 0 ], "final": [ - "junos", - 35 + "unknown", + 0 ] }, "eos.5|-|65536": { "after_auth": [ - "junos", - 35 + "unknown", + 0 ], "final": [ - "junos", - 35 + "unknown", + 0 ] }, "eos.5|SSH-2.0-Cisco-1.25|64": { @@ -1541,22 +1541,22 @@ }, "eos.5|SSH-2.0-OpenSSH_9.6|64": { "after_auth": [ - "junos", - 35 + "unknown", + 0 ], "final": [ - "junos", - 35 + "unknown", + 0 ] }, "eos.5|SSH-2.0-OpenSSH_9.6|65536": { "after_auth": [ - "junos", - 35 + "unknown", + 0 ], "final": [ - "junos", - 35 + "unknown", + 0 ] }, "ericsson_ban|-|64": { @@ -2941,68 +2941,68 @@ }, "prompt-only:|SSH-2.0-OpenSSH_9.6|64": { "after_auth": [ - "ericsson_ban", - 20 + "unknown", + 0 ], "final": [ - "ericsson_ban", - 20 + "unknown", + 0 ] }, "prompt-only:|SSH-2.0-OpenSSH_9.6|65536": { "after_auth": [ - "ericsson_ban", - 20 + "unknown", + 0 ], "final": [ - "ericsson_ban", - 20 + "unknown", + 0 ] }, "prompt-only:A:sros-pe1#|SSH-2.0-OpenSSH_9.6|64": { "after_auth": [ - "ericsson_ban", - 20 + "unknown", + 0 ], "final": [ - "ericsson_ban", - 20 + "unknown", + 0 ] }, "prompt-only:A:sros-pe1#|SSH-2.0-OpenSSH_9.6|65536": { "after_auth": [ - "ericsson_ban", - 20 + "unknown", + 0 ], "final": [ - "ericsson_ban", - 20 + "unknown", + 0 ] }, "prompt-only:FGT60E #|SSH-2.0-OpenSSH_9.6|64": { "after_auth": [ - "ericsson_ban", - 20 + "unknown", + 0 ], "final": [ - "ericsson_ban", - 20 + "unknown", + 0 ] }, "prompt-only:FGT60E #|SSH-2.0-OpenSSH_9.6|65536": { "after_auth": [ - "ericsson_ban", - 20 + "unknown", + 0 ], "final": [ - "ericsson_ban", - 20 + "unknown", + 0 ] }, "prompt-only:RP/0/RSP0/CPU0:xr-pe1#|SSH-2.0-OpenSSH_9.6|64": { "after_auth": [ - "ericsson_ban", - 20 + "unknown", + 0 ], "final": [ "ios_xr", @@ -3011,8 +3011,8 @@ }, "prompt-only:RP/0/RSP0/CPU0:xr-pe1#|SSH-2.0-OpenSSH_9.6|65536": { "after_auth": [ - "ericsson_ban", - 20 + "unknown", + 0 ], "final": [ "ios_xr", @@ -3021,162 +3021,162 @@ }, "prompt-only:[admin@MikroTik] >|SSH-2.0-OpenSSH_9.6|64": { "after_auth": [ - "ericsson_ban", - 20 + "unknown", + 0 ], "final": [ - "ericsson_ban", - 20 + "unknown", + 0 ] }, "prompt-only:[admin@MikroTik] >|SSH-2.0-OpenSSH_9.6|65536": { "after_auth": [ - "ericsson_ban", - 20 + "unknown", + 0 ], "final": [ - "ericsson_ban", - 20 + "unknown", + 0 ] }, "prompt-only:admin@junos-mx> |SSH-2.0-OpenSSH_9.6|64": { "after_auth": [ - "ericsson_ban", - 20 + "unknown", + 0 ], "final": [ - "ericsson_ban", - 20 + "unknown", + 0 ] }, "prompt-only:admin@junos-mx> |SSH-2.0-OpenSSH_9.6|65536": { "after_auth": [ - "ericsson_ban", - 20 + "unknown", + 0 ], "final": [ - "ericsson_ban", - 20 + "unknown", + 0 ] }, "prompt-only:core-sw01(config)#|SSH-2.0-OpenSSH_9.6|64": { "after_auth": [ - "ericsson_ban", - 20 + "unknown", + 0 ], "final": [ - "ericsson_ban", - 20 + "unknown", + 0 ] }, "prompt-only:core-sw01(config)#|SSH-2.0-OpenSSH_9.6|65536": { "after_auth": [ - "ericsson_ban", - 20 + "unknown", + 0 ], "final": [ - "ericsson_ban", - 20 + "unknown", + 0 ] }, "prompt-only:root@host:~#|SSH-2.0-OpenSSH_9.6|64": { "after_auth": [ - "ericsson_ban", - 20 + "unknown", + 0 ], "final": [ - "ericsson_ban", - 20 + "shell", + 30 ] }, "prompt-only:root@host:~#|SSH-2.0-OpenSSH_9.6|65536": { "after_auth": [ - "ericsson_ban", - 20 + "unknown", + 0 ], "final": [ - "ericsson_ban", - 20 + "shell", + 30 ] }, "prompt-only:router#|SSH-2.0-OpenSSH_9.6|64": { "after_auth": [ - "ericsson_ban", - 20 + "unknown", + 0 ], "final": [ - "ericsson_ban", - 20 + "unknown", + 0 ] }, "prompt-only:router#|SSH-2.0-OpenSSH_9.6|65536": { "after_auth": [ - "ericsson_ban", - 20 + "unknown", + 0 ], "final": [ - "ericsson_ban", - 20 + "unknown", + 0 ] }, "prompt-only:switch>|SSH-2.0-OpenSSH_9.6|64": { "after_auth": [ - "ericsson_ban", - 20 + "unknown", + 0 ], "final": [ - "ericsson_ban", - 20 + "unknown", + 0 ] }, "prompt-only:switch>|SSH-2.0-OpenSSH_9.6|65536": { "after_auth": [ - "ericsson_ban", - 20 + "unknown", + 0 ], "final": [ - "ericsson_ban", - 20 + "unknown", + 0 ] }, "prompt-only:user@host:~$|SSH-2.0-OpenSSH_9.6|64": { "after_auth": [ - "ericsson_ban", - 20 + "unknown", + 0 ], "final": [ - "ericsson_ban", - 20 + "shell", + 30 ] }, "prompt-only:user@host:~$|SSH-2.0-OpenSSH_9.6|65536": { "after_auth": [ - "ericsson_ban", - 20 + "unknown", + 0 ], "final": [ - "ericsson_ban", - 20 + "shell", + 30 ] }, "rios.1|-|64": { "after_auth": [ - "isam", - 20 + "unknown", + 0 ], "final": [ - "isam", - 20 + "unknown", + 0 ] }, "rios.1|-|65536": { "after_auth": [ - "isam", - 20 + "unknown", + 0 ], "final": [ - "isam", - 20 + "unknown", + 0 ] }, "rios.1|SSH-2.0-Cisco-1.25|64": { @@ -3201,22 +3201,22 @@ }, "rios.1|SSH-2.0-OpenSSH_9.6|64": { "after_auth": [ - "isam", - 20 + "unknown", + 0 ], "final": [ - "isam", - 20 + "unknown", + 0 ] }, "rios.1|SSH-2.0-OpenSSH_9.6|65536": { "after_auth": [ - "isam", - 20 + "unknown", + 0 ], "final": [ - "isam", - 20 + "unknown", + 0 ] }, "smart_edge_os.1|-|64": { @@ -3525,8 +3525,8 @@ 0 ], "final": [ - "isam", - 20 + "unknown", + 0 ] }, "unknown|-|65536": { @@ -3535,8 +3535,8 @@ 0 ], "final": [ - "isam", - 20 + "unknown", + 0 ] }, "unknown|SSH-2.0-Cisco-1.25|64": { @@ -3565,8 +3565,8 @@ 0 ], "final": [ - "isam", - 20 + "unknown", + 0 ] }, "unknown|SSH-2.0-OpenSSH_9.6|65536": { @@ -3575,8 +3575,8 @@ 0 ], "final": [ - "isam", - 20 + "unknown", + 0 ] }, "vrp.1|-|64": { diff --git a/tests/recordings/frr/frr-10.4-vtysh-ssh.golden.json b/tests/recordings/frr/frr-10.4-vtysh-ssh.golden.json new file mode 100644 index 00000000..0c7f1af4 --- /dev/null +++ b/tests/recordings/frr/frr-10.4-vtysh-ssh.golden.json @@ -0,0 +1,119 @@ +{ + "login": "ok", + "driver": "generic", + "os": "unknown", + "commands": [ + { + "command": "terminal length 0", + "error": null, + "lines": 1, + "sha256": "fad757df55a72a893a0b38650bdfb70b3eba94e0115682d4bca0b3ddac92b95c", + "first": "terminal length 0", + "last": "terminal length 0" + }, + { + "command": "show version", + "error": null, + "lines": 5, + "sha256": "64c6d8f5dcc9153c7e73b80388631b432511dd467cc6e9eca1a083bba54c6716", + "first": "show version", + "last": " '--prefix=/usr' '--sysconfdir=/etc' '--localstatedir=/var' '--sbindir=/usr/lib/frr' '--libdir=/usr/lib' '--enable-rp" + }, + { + "command": "show running-config", + "error": null, + "lines": 15, + "sha256": "25ae6c900bc991bd2d6224e274bbe1111804dca1dde2de98cc055f948b37719b", + "first": "show running-config", + "last": "end" + }, + { + "command": "show interface brief", + "error": null, + "lines": 5, + "sha256": "e05bf0b4ac2a411ce311564466044a8eae23b579dc107946e342709b4aa15a7d", + "first": "show interface brief", + "last": "lo up default " + }, + { + "command": "show ip route", + "error": null, + "lines": 14, + "sha256": "9f5abd51bda361deb0a277acb8d643e55d1f98b0e4c23acb727ea4048e0baa7f", + "first": "show ip route", + "last": "S>* 198.51.100.0/24 [1/0] unreachable (blackhole), weight 1, 00:00:26" + }, + { + "command": "show nonexistent", + "error": "InvalidCommandException", + "lines": 2, + "sha256": "472b430eb716a63610ac6127a8296403fadd4b0928b6baa0b7e237cb3564fc14", + "first": "show nonexistent", + "last": "% Unknown command: show nonexistent" + }, + { + "command": "show ip", + "error": null, + "lines": 2, + "sha256": "27eab082a4deef01d1a85917af453b9dba92570b6363d0159ec47d676bccc0da", + "first": "show ip", + "last": "% Command incomplete: show ip" + }, + { + "command": "sh i", + "error": null, + "lines": 30, + "sha256": "4ee6e30dc23e9d8c82cab57885322da9784dc3c30c354bc3e88a8c10da497de9", + "first": "sh i", + "last": " protodown: off " + }, + { + "command": "configure terminal", + "error": null, + "lines": 1, + "sha256": "4c4a842f65abce035cfe0297cc753a244e6e0bc74dca1f7e95b50b3e97dcd1ff", + "first": "configure terminal", + "last": "configure terminal" + }, + { + "command": "interface lo", + "error": null, + "lines": 1, + "sha256": "f99ce7a196f42339ee65521fe4c0ff6cdfc751be2d25c42abc5cba3d2f0244a6", + "first": "interface lo", + "last": "interface lo" + }, + { + "command": "description loopback for the Exscript tests", + "error": null, + "lines": 1, + "sha256": "fde7b1c5ed8d64ed77dbe72d67333822c4e99697f21ea90f9b1ee63a33ad6875", + "first": "description loopback for the Exscript tests", + "last": "description loopback for the Exscript tests" + }, + { + "command": "end", + "error": null, + "lines": 1, + "sha256": "361e48d0308f20e32dba5fb56328baf18d72ef0ccb43b84f5c262d2a6a1fc6c8", + "first": "end", + "last": "end" + }, + { + "command": "show running-config", + "error": null, + "lines": 19, + "sha256": "167d2a05c5f592ccc705f22319ba66cef3236fa493740ce22bae9676ee27527e", + "first": "show running-config", + "last": "end" + }, + { + "command": "exit", + "error": "ConnectionClosedException", + "lines": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "first": "", + "last": "" + } + ] +} diff --git a/tests/recordings/linux/alpine-3.22-bash-ssh-vt100.golden.json b/tests/recordings/linux/alpine-3.22-bash-ssh-vt100.golden.json index ad6cbcfa..3b7804f4 100644 --- a/tests/recordings/linux/alpine-3.22-bash-ssh-vt100.golden.json +++ b/tests/recordings/linux/alpine-3.22-bash-ssh-vt100.golden.json @@ -1,7 +1,7 @@ { "login": "ok", - "driver": "generic", - "os": "unknown", + "driver": "shell", + "os": "shell", "commands": [ { "command": "uname -sr", diff --git a/tests/recordings/linux/alpine-3.22-bash-ssh.golden.json b/tests/recordings/linux/alpine-3.22-bash-ssh.golden.json index f51d2fb4..c35605d3 100644 --- a/tests/recordings/linux/alpine-3.22-bash-ssh.golden.json +++ b/tests/recordings/linux/alpine-3.22-bash-ssh.golden.json @@ -1,7 +1,7 @@ { "login": "ok", - "driver": "generic", - "os": "unknown", + "driver": "shell", + "os": "shell", "commands": [ { "command": "uname -sr", diff --git a/tests/recordings/linux/busybox-1.37-ash-telnet.golden.json b/tests/recordings/linux/busybox-1.37-ash-telnet.golden.json index 7307e489..19be44b5 100644 --- a/tests/recordings/linux/busybox-1.37-ash-telnet.golden.json +++ b/tests/recordings/linux/busybox-1.37-ash-telnet.golden.json @@ -1,6 +1,55 @@ { - "login": "TimeoutException", - "driver": "junos", - "os": "junos", - "commands": [] + "login": "ok", + "driver": "shell", + "os": "shell", + "commands": [ + { + "command": "uname -sr", + "error": null, + "lines": 2, + "sha256": "3cbc3fd8aee393a4d91615acfa6f59c14e10b97dd2158f66721ff4ecb75cefb3", + "first": "uname -sr", + "last": "Linux 6.18.54.1-microsoft-standard-WSL2" + }, + { + "command": "cat /etc/passwd", + "error": null, + "lines": 11, + "sha256": "285bcc71b24bb46caf324c4490c3c2098d3461c084d981dc19deb5f40b9eaa88", + "first": "cat /etc/passwd", + "last": "admin:x:1000:1000:Linux User,,,:/home/admin:/bin/sh" + }, + { + "command": "ls -l /", + "error": null, + "lines": 14, + "sha256": "eec563e5fb030400147498d45bdacfc0a4c321c317187874d9c879c81f2d4057", + "first": "ls -l /", + "last": "drwxr-xr-x 4 root root 4096 Sep 26 2024 \u001b[1;34mvar\u001b[m" + }, + { + "command": "seq 1 5000", + "error": null, + "lines": 5001, + "sha256": "3f4e8d91515497e6d24014249a254ed0088b2673095b3e57b1446385b7ae821c", + "first": "seq 1 5000", + "last": "5000" + }, + { + "command": "ls /nonexistent", + "error": null, + "lines": 2, + "sha256": "c44056b106b058656342e300441def2785e91ac86b8e7be94ba989ebe963466b", + "first": "ls /nonexistent", + "last": "ls: /nonexistent: No such file or directory" + }, + { + "command": "exit", + "error": "ConnectionClosedException", + "lines": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "first": "", + "last": "" + } + ] } diff --git a/tests/recordings/netopeer2/netopeer2-netconf.golden.json b/tests/recordings/netopeer2/netopeer2-netconf.golden.json index fa2ac6f6..7f8ab70f 100644 --- a/tests/recordings/netopeer2/netopeer2-netconf.golden.json +++ b/tests/recordings/netopeer2/netopeer2-netconf.golden.json @@ -1,7 +1,7 @@ { "login": "ok", - "driver": "isam", - "os": "isam", + "driver": "generic", + "os": "unknown", "commands": [ { "command": "", diff --git a/tools/osguess_report.py b/tools/osguess_report.py index 84c37784..12aacde3 100644 --- a/tools/osguess_report.py +++ b/tools/osguess_report.py @@ -28,7 +28,7 @@ from tests.Exscript.protocols import ReplayTest as replays # noqa: E402 from tests.Exscript.protocols.replay import Recording # noqa: E402 -MIN_CORRECT = 204 # 2026-11-03; raise it when a change guesses more sessions right +MIN_CORRECT = 213 # 2026-11-03; raise it when a change guesses more sessions right def banner_sessions(): @@ -46,7 +46,11 @@ def banner_sessions(): for key, result in sorted(results.items()): name, info, size = key.split('|') protocol = 'Telnet' if info == '-' else 'SSH' - yield f'banner {name}, {protocol}, {size}-byte reads', name.split('.')[0], result['final'][0] + yield ( + f'banner {name}, {protocol}, {size}-byte reads', + name.split('.')[0], + result['final'][0], + ) def recording_sessions(): From e9863530cee857b8f556aa18e4e26e3eef34ca4a Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 3 Nov 2026 18:09:37 +0100 Subject: [PATCH 044/250] Run the tests with warnings as errors The warnings showed two leaks in the package: Queue opened os.devnull for every instance and never closed it, and get_terminal_size() in util.tty left the pipe of "stty size" open; it uses shutil's now. --- CHANGES.md | 4 ++ pyproject.toml | 7 ++ src/Exscript/queue.py | 13 +++- src/Exscript/servers/sshd.py | 6 +- src/Exscript/util/tty.py | 116 +++---------------------------- tests/Exscript/FileLoggerTest.py | 16 ++--- tests/Exscript/LoggerTest.py | 28 ++++++-- tests/Exscript/util/ttyTest.py | 87 ++++++----------------- 8 files changed, 88 insertions(+), 189 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 352dc167..9216a11e 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -323,6 +323,10 @@ The names below refer to functions callable from templates, not protocol methods which has no `syslog` module. - **Pseudo Protocol on Windows:** `pseudo://` URLs accept absolute Windows paths such as `pseudo://C:\devices\router.py`; the drive letter was taken for a port number. +- **`Exscript.util.tty.get_terminal_size()`:** Uses `shutil.get_terminal_size()`: the + `LINES` and `COLUMNS` environment variables, then the terminal of the standard + output, then the defaults. It no longer asks standard input, standard error and the + controlling terminal, and no longer runs `stty size`, which leaked a pipe. ## Multi-Threading diff --git a/pyproject.toml b/pyproject.toml index 237666c2..bac81864 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -132,3 +132,10 @@ warn_redundant_casts = true testpaths = ["tests"] python_files = ["*Test.py"] addopts = ["-ra", "--strict-config", "--strict-markers"] +filterwarnings = [ + "error", + # The process mode of the queue forks a process that has threads (Python 3.12 and 3.13). + "ignore:This process .* is multi-threaded, use of fork\\(\\) may lead to deadlocks:DeprecationWarning", + # lxml enables the GIL of free-threaded Python; CI runs these tests with PYTHON_GIL=0. + "ignore:The global interpreter lock \\(GIL\\) has been enabled:RuntimeWarning", +] diff --git a/src/Exscript/queue.py b/src/Exscript/queue.py index e273d7df..cccdf678 100644 --- a/src/Exscript/queue.py +++ b/src/Exscript/queue.py @@ -26,7 +26,6 @@ import gc import logging -import os import sys import threading import weakref @@ -46,6 +45,16 @@ _logger.addHandler(logging.NullHandler()) +class _Discard: + """Discards what is written to it, like os.devnull, but has no file to close.""" + + def write(self, data): + return len(data) + + def flush(self): + pass + + def _account_factory(accm, host, account): if account is None: account = host.get_account() @@ -265,7 +274,7 @@ def __init__( self.host_driver = host_driver self.exc_cb = exc_cb self.lock = threading.RLock() - self.devnull = open(os.devnull, 'w', encoding='utf-8') + self.devnull = _Discard() self.channel_map = {'fatal_errors': self.stderr} self.completed = 0 self.total = 0 diff --git a/src/Exscript/servers/sshd.py b/src/Exscript/servers/sshd.py index 03b90e1f..cc5f43ca 100644 --- a/src/Exscript/servers/sshd.py +++ b/src/Exscript/servers/sshd.py @@ -137,11 +137,11 @@ def _handle_connection(self, conn): server = _ParamikoServer() try: t.start_server(server=server) - except EOFError: - self._dbg(1, 'Client disappeared before establishing connection') + except (EOFError, OSError, paramiko.SSHException) as err: + self._dbg(1, 'Client disappeared before establishing connection: %s' % err) t.close() conn.close() - raise + return # Validate that the connection succeeded. if not t.is_active(): diff --git a/src/Exscript/util/tty.py b/src/Exscript/util/tty.py index f4324ba0..88067564 100644 --- a/src/Exscript/util/tty.py +++ b/src/Exscript/util/tty.py @@ -24,115 +24,21 @@ TTY utilities. """ -import os -import struct -import sys -from subprocess import PIPE, Popen - - -def _get_terminal_size(fd): - try: - import fcntl - import termios - except ImportError: - return None - s = struct.pack('HHHH', 0, 0, 0, 0) - try: - x = fcntl.ioctl(fd, termios.TIOCGWINSZ, s) - except OSError: # Window size ioctl not supported. - return None - try: - rows, cols, x_pixels, y_pixels = struct.unpack('HHHH', x) - except struct.error: - return None - return rows, cols +import shutil def get_terminal_size(default_rows=25, default_cols=80): """ - Returns the number of lines and columns of the current terminal. - It attempts several strategies to determine the size and if all fail, - it returns (80, 25). - + Returns the number of lines and columns of the terminal: from the LINES + and COLUMNS environment variables if they are set, else from the terminal + of the standard output, else the given defaults. + + :type default_rows: int + :param default_rows: The lines if there is no terminal. + :type default_cols: int + :param default_cols: The columns if there is no terminal. :rtype: int, int :return: The rows and columns of the terminal. """ - # Collect a list of viable input channels that may tell us something - # about the terminal dimensions. - fileno_list = [] - try: - fileno_list.append(sys.stdout.fileno()) - except AttributeError: - # Channel was redirected to an object that has no fileno() - pass - except ValueError: - # Channel was closed while attempting to read it - pass - try: - fileno_list.append(sys.stdin.fileno()) - except AttributeError: - pass - except ValueError: - # Channel was closed while attempting to read it - pass - try: - fileno_list.append(sys.stderr.fileno()) - except AttributeError: - pass - except ValueError: - # Channel was closed while attempting to read it - pass - - # Ask each channel for the terminal window size. - for fd in fileno_list: - try: - rows, cols = _get_terminal_size(fd) - except TypeError: - # _get_terminal_size() returned None. - pass - else: - return rows, cols - - # Try os.ctermid() - try: - fd = os.open(os.ctermid(), os.O_RDONLY) - except AttributeError: - # os.ctermid does not exist on Windows. - pass - except OSError: - # The device pointed to by os.ctermid() does not exist. - pass - else: - try: - rows, cols = _get_terminal_size(fd) - except TypeError: - # _get_terminal_size() returned None. - pass - else: - return rows, cols - finally: - os.close(fd) - - # Try `stty size` - with open(os.devnull, 'w', encoding='utf-8') as devnull: - try: - process = Popen(['stty', 'size'], stderr=devnull, stdout=PIPE, close_fds=True) - except (OSError, ValueError): - pass - else: - errcode = process.wait() - output = process.stdout.read() - try: - rows, cols = output.split() - return int(rows), int(cols) - except (ValueError, TypeError): - pass - - # Try environment variables. - try: - return tuple(int(os.getenv(var)) for var in ('LINES', 'COLUMNS')) - except (ValueError, TypeError): - pass - - # Give up. - return default_rows, default_cols + size = shutil.get_terminal_size((default_cols, default_rows)) + return size.lines, size.columns diff --git a/tests/Exscript/FileLoggerTest.py b/tests/Exscript/FileLoggerTest.py index cf7a9d56..64b06518 100644 --- a/tests/Exscript/FileLoggerTest.py +++ b/tests/Exscript/FileLoggerTest.py @@ -32,26 +32,26 @@ def testConstructor(self): self.assertFalse(os.path.exists(self.logfile)) self.assertFalse(os.path.exists(self.errfile)) - def testAddLog(self): - log = LoggerTest.testAddLog(self) + def _addLog(self): + log = LoggerTest._addLog(self) self.assertTrue(os.path.isfile(self.logfile), 'No such file: ' + self.logfile) self.assertFalse(os.path.exists(self.errfile)) return log - def testLog(self): - log = LoggerTest.testLog(self) + def _log(self): + log = LoggerTest._log(self) self.assertTrue(os.path.isfile(self.logfile)) self.assertFalse(os.path.exists(self.errfile)) return log - def testLogAborted(self): - log = LoggerTest.testLogAborted(self) + def _logAborted(self): + log = LoggerTest._logAborted(self) self.assertTrue(os.path.isfile(self.logfile)) self.assertTrue(os.path.isfile(self.errfile)) return log - def testLogSucceeded(self): - log = LoggerTest.testLogSucceeded(self) + def _logSucceeded(self): + log = LoggerTest._logSucceeded(self) self.assertTrue(os.path.isfile(self.logfile)) self.assertFalse(os.path.isfile(self.errfile)) return log diff --git a/tests/Exscript/LoggerTest.py b/tests/Exscript/LoggerTest.py index 8ed50d5c..b621cc00 100644 --- a/tests/Exscript/LoggerTest.py +++ b/tests/Exscript/LoggerTest.py @@ -128,21 +128,23 @@ def testGetAbortedLogs(self): self.assertEqual(count(self.logger.get_aborted_logs()), 1) self.assertIsInstance(nth(self.logger.get_aborted_logs(), 0), Log) - def testAddLog(self): + # The helpers return the log, so that the steps build on each other, also + # with the checks that FileLoggerTest adds; test methods must return None. + def _addLog(self): self.assertEqual(count(self.logger.get_logs()), 0) log = self.logger.add_log(id(self.job), self.job.name, 1) self.assertEqual(count(self.logger.get_logs()), 1) self.assertEqual(str(log), '') return log - def testLog(self): - log = self.testAddLog() + def _log(self): + log = self._addLog() self.logger.log(id(self.job), 'hello world') self.assertEqual(str(log), 'hello world') return log - def testLogAborted(self): - log = self.testLog() + def _logAborted(self): + log = self._log() try: raise FakeError() except Exception: @@ -150,12 +152,24 @@ def testLogAborted(self): self.assertIn('FakeError', str(log)) return log - def testLogSucceeded(self): - log = self.testLog() + def _logSucceeded(self): + log = self._log() self.logger.log_succeeded(id(self.job)) self.assertEqual(str(log), 'hello world') return log + def testAddLog(self): + self._addLog() + + def testLog(self): + self._log() + + def testLogAborted(self): + self._logAborted() + + def testLogSucceeded(self): + self._logSucceeded() + def suite(): return unittest.TestLoader().loadTestsFromTestCase(LoggerTest) diff --git a/tests/Exscript/util/ttyTest.py b/tests/Exscript/util/ttyTest.py index 6028ae0e..c2351f4c 100644 --- a/tests/Exscript/util/ttyTest.py +++ b/tests/Exscript/util/ttyTest.py @@ -1,76 +1,35 @@ import os -import sys -import tempfile import unittest +from unittest import mock import Exscript.util.tty +from Exscript.util.tty import get_terminal_size + +NO_SIZE_VARIABLES = {'LINES': '', 'COLUMNS': ''} class ttyTest(unittest.TestCase): CORRELATE = Exscript.util.tty - def setUp(self): - self.tempfile = tempfile.TemporaryFile() - self.stdout = sys.stdout - self.stderr = sys.stderr - self.stdin = sys.stdin - sys.stdout = sys.stderr = sys.stdin = self.tempfile - - def _unredirect(self): - sys.stdout = self.stdout - sys.stderr = self.stderr - sys.stdin = self.stdin - - def tearDown(self): - self._unredirect() - self.tempfile.close() - - def testGetTerminalSize(self): - from Exscript.util.tty import get_terminal_size - - # This hack really makes the test incomplete because - # get_terminal_size() won't be able to check the cterm size, - # but it is the only way to test at least partially. - os.ctermid = lambda: '/nosuchfileexists' - - # By deleting PATH we prevent get_terminal_size() from asking - # the stty unix program. - oldpath = os.environ['PATH'] - os.environ['PATH'] = '' - - # If the LINES and COLUMNS variables are not set, all methods should - # now fail, and the default values are returned. - os.environ['LINES'] = '' - os.environ['COLUMNS'] = '' - self.assertEqual(get_terminal_size(), (25, 80)) - self.assertEqual(get_terminal_size(10, 10), (10, 10)) - - # If the LINES and COLUMNS variables are set, they should be used. - os.environ['LINES'] = '1111' - os.environ['COLUMNS'] = '1111' - self.assertEqual(get_terminal_size(), (1111, 1111)) - self.assertEqual(get_terminal_size(10, 10), (1111, 1111)) - - # If the stty program exists, it should be used. - os.environ['PATH'] = oldpath - try: - # Unfortunately, the 'stty' program on travis actually returns - # (1111, 1111) if the LINES and COLUMNS env variables are set, - # so there is no way to tests whether stty was used or the env - # vars. - # self.assertNotEqual(get_terminal_size(), (1111, 1111)) - self.assertNotEqual(get_terminal_size(10, 10), (25, 80)) - except OSError: - pass # "stty" not found. - - # Lastly, if stdin/stderr/stdout exist, they should tell us something. - os.environ['PATH'] = '' - self._unredirect() - # Same travis problem as above... - # self.assertNotEqual(get_terminal_size(), (1111, 1111)) - # self.assertNotEqual(get_terminal_size(10, 10), (1111, 1111)) - self.assertNotEqual(get_terminal_size(10, 10), (25, 80)) - os.environ['PATH'] = oldpath + def testEnvironmentVariables(self): + with mock.patch.dict(os.environ, {'LINES': '50', 'COLUMNS': '132'}): + self.assertEqual(get_terminal_size(), (50, 132)) + + def testTerminal(self): + size = os.terminal_size((120, 40)) + with ( + mock.patch.dict(os.environ, NO_SIZE_VARIABLES), + mock.patch('os.get_terminal_size', return_value=size), + ): + self.assertEqual(get_terminal_size(), (40, 120)) + + def testDefaults(self): + with ( + mock.patch.dict(os.environ, NO_SIZE_VARIABLES), + mock.patch('os.get_terminal_size', side_effect=OSError), + ): + self.assertEqual(get_terminal_size(), (25, 80)) + self.assertEqual(get_terminal_size(10, 20), (10, 20)) def suite(): From b806c4c176cb67ff328ba6b20367553aa3bd1860 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 3 Nov 2026 18:12:41 +0100 Subject: [PATCH 045/250] Remove util.daemonize, util.pidutil and util.sigintcatcher daemonize() failed on Python 3 in every released version, and service managers such as systemd do this job today. --- CHANGES.md | 7 +- pyproject.toml | 1 - src/Exscript/util/daemonize.py | 69 -------------- src/Exscript/util/pidutil.py | 144 ----------------------------- src/Exscript/util/sigintcatcher.py | 42 --------- 5 files changed, 5 insertions(+), 258 deletions(-) delete mode 100644 src/Exscript/util/daemonize.py delete mode 100644 src/Exscript/util/pidutil.py delete mode 100644 src/Exscript/util/sigintcatcher.py diff --git a/CHANGES.md b/CHANGES.md index 9216a11e..ebf5e8fa 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -14,8 +14,7 @@ marked with **BREAKING**; a migration guide will accompany the release. - **UTF-8 Files:** Exscript reads and writes text files, e.g. templates, host lists and log files, as UTF-8 on every platform, as Python 3.15 does by default. Before, it used the encoding of the locale, e.g. cp1252 on Windows, where writing device - output to a log file could fail. `util.daemonize` no longer fails with - `ValueError: can't have unbuffered text I/O`. + output to a log file could fail. - **Python 2.7 Support:** - **Python 3.11+ Support:** - **Dependencies:** @@ -327,6 +326,10 @@ The names below refer to functions callable from templates, not protocol methods `LINES` and `COLUMNS` environment variables, then the terminal of the standard output, then the defaults. It no longer asks standard input, standard error and the controlling terminal, and no longer runs `stty size`, which leaked a pipe. +- **BREAKING - Removed Modules:** `Exscript.util.daemonize`, `Exscript.util.pidutil` + and `Exscript.util.sigintcatcher` are removed, and + `daemonize()` failed on Python 3 with `ValueError: can't have unbuffered text I/O`. + Run long-lived scripts under a service manager such as systemd instead. ## Multi-Threading diff --git a/pyproject.toml b/pyproject.toml index bac81864..36599e1b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -114,7 +114,6 @@ ignore = [ "src/Exscript/stdlib/mysys.py" = ["S603"] "src/Exscript/util/crypt.py" = ["S303"] "src/Exscript/util/file.py" = ["S107"] -"src/Exscript/util/pidutil.py" = ["S110"] "src/Exscript/util/tty.py" = ["S607"] "src/Exscript/workqueue/pipeline.py" = ["S101"] "tests/Exscript/QueueTest.py" = ["S110"] diff --git a/src/Exscript/util/daemonize.py b/src/Exscript/util/daemonize.py deleted file mode 100644 index 3886a2fa..00000000 --- a/src/Exscript/util/daemonize.py +++ /dev/null @@ -1,69 +0,0 @@ -# -# Copyright (C) 2010-2017 Samuel Abels -# The MIT License (MIT) -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files -# (the "Software"), to deal in the Software without restriction, -# including without limitation the rights to use, copy, modify, merge, -# publish, distribute, sublicense, and/or sell copies of the Software, -# and to permit persons to whom the Software is furnished to do so, -# subject to the following conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY -# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, -# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE -# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -""" -Daemonizing a process. -""" - -import os -import sys - - -def _redirect_output(filename): - out_log = open(filename, 'ab', 0) - err_log = open(filename, 'ab', 0) - dev_null = open(os.devnull, 'rb') - os.close(sys.stdin.fileno()) - os.close(sys.stdout.fileno()) - os.close(sys.stderr.fileno()) - os.dup2(out_log.fileno(), sys.stdout.fileno()) - os.dup2(err_log.fileno(), sys.stderr.fileno()) - os.dup2(dev_null.fileno(), sys.stdin.fileno()) - - -def daemonize(): - """ - Forks and daemonizes the current process. Does not automatically track - the process id; to do this, use :class:`Exscript.util.pidutil`. - """ - sys.stdout.flush() - sys.stderr.flush() - - # UNIX double-fork magic. We need to fork before any threads are - # created. - pid = os.fork() - if pid > 0: - # Exit first parent. - sys.exit(0) - - # Decouple from parent environment. - os.chdir('/') - os.setsid() - os.umask(0) - - # Now fork again. - pid = os.fork() - if pid > 0: - # Exit second parent. - sys.exit(0) - - _redirect_output(os.devnull) diff --git a/src/Exscript/util/pidutil.py b/src/Exscript/util/pidutil.py deleted file mode 100644 index c5af403e..00000000 --- a/src/Exscript/util/pidutil.py +++ /dev/null @@ -1,144 +0,0 @@ -# -# Copyright (C) 2010-2017 Samuel Abels -# The MIT License (MIT) -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files -# (the "Software"), to deal in the Software without restriction, -# including without limitation the rights to use, copy, modify, merge, -# publish, distribute, sublicense, and/or sell copies of the Software, -# and to permit persons to whom the Software is furnished to do so, -# subject to the following conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY -# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, -# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE -# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -""" -Handling PID (process id) files. -""" - -import errno -import fcntl -import logging -import os - -_logger = logging.getLogger(__name__) -_logger.addHandler(logging.NullHandler()) - - -def read(path): - """ - Returns the process id from the given file if it exists, or None - otherwise. Raises an exception for all other types of OSError - while trying to access the file. - - :type path: str - :param path: The name of the pidfile. - :rtype: int or None - :return: The PID, or none if the file was not found. - """ - # Try to read the pid from the pidfile. - _logger.info("Checking pidfile '%s'", path) - try: - return int(open(path, encoding="utf-8").read()) - except OSError as xxx_todo_changeme: - (code, text) = xxx_todo_changeme.args - if code == errno.ENOENT: # no such file or directory - return None - raise - - -def isalive(path): - """ - Returns True if the file with the given name contains a process - id that is still alive. - Returns False otherwise. - - :type path: str - :param path: The name of the pidfile. - :rtype: bool - :return: Whether the process is alive. - """ - # try to read the pid from the pidfile - pid = read(path) - if pid is None: - return False - - # Check if a process with the given pid exists. - try: - os.kill(pid, 0) # Signal 0 does not kill, but check. - except OSError as xxx_todo_changeme1: - (code, text) = xxx_todo_changeme1.args - if code == errno.ESRCH: # No such process. - return False - return True - - -def kill(path): - """ - Kills the process, if it still exists. - - :type path: str - :param path: The name of the pidfile. - """ - # try to read the pid from the pidfile - pid = read(path) - if pid is None: - return - - # Try to kill the process. - _logger.info("Killing PID %s", pid) - try: - os.kill(pid, 9) - except OSError as xxx_todo_changeme2: - # re-raise if the error wasn't "No such process" - (code, text) = xxx_todo_changeme2.args - # re-raise if the error wasn't "No such process" - if code != errno.ESRCH: - raise - - -def write(path): - """ - Writes the current process id to the given pidfile. - - :type path: str - :param path: The name of the pidfile. - """ - pid = os.getpid() - _logger.info("Writing PID %s to '%s'", pid, path) - try: - pidfile = open(path, 'wb') - # get a non-blocking exclusive lock - fcntl.flock(pidfile.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) - # clear out the file - pidfile.seek(0) - pidfile.truncate(0) - # write the pid - pidfile.write(str(pid)) - finally: - try: - pidfile.close() - except: - pass - - -def remove(path): - """ - Deletes the pidfile if it exists. - - :type path: str - :param path: The name of the pidfile. - """ - _logger.info("Removing pidfile '%s'", path) - try: - os.unlink(path) - except OSError: - pass diff --git a/src/Exscript/util/sigintcatcher.py b/src/Exscript/util/sigintcatcher.py deleted file mode 100644 index 3c680dd8..00000000 --- a/src/Exscript/util/sigintcatcher.py +++ /dev/null @@ -1,42 +0,0 @@ -# -# Copyright (C) 2010-2017 Samuel Abels -# The MIT License (MIT) -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files -# (the "Software"), to deal in the Software without restriction, -# including without limitation the rights to use, copy, modify, merge, -# publish, distribute, sublicense, and/or sell copies of the Software, -# and to permit persons to whom the Software is furnished to do so, -# subject to the following conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY -# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, -# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE -# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -""" -When imported, this module catches KeyboardInterrupt (SIGINT). -It is a convenience wrapper around sigint.SigIntWatcher() -such that all that is required for the change to take effect -is the following statement:: - - import Exscript.util.sigintcatcher - -Be warned that this way of importing breaks on some systems, because a -fork during an import may cause the following error:: - - RuntimeError: not holding the import lock - -So in general it is recommended to use the :class:`sigint.SigIntWatcher()` -class directly. -""" - -from Exscript.util.sigint import SigIntWatcher - -_watcher = SigIntWatcher() From deb87044757f6f19cdc1d934f6dae2090bfb2c08 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 3 Nov 2026 18:15:15 +0100 Subject: [PATCH 046/250] Report API changes in the job summary instead of failing griffe exits with 1 when it finds incompatible changes, which every 3.0 pull request has, so the job showed as failed although it only reports. --- .github/workflows/ci.yml | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 90baaaa4..5e0fb4c9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -116,7 +116,6 @@ jobs: name: Public API changes (report only) if: github.event_name == 'pull_request' runs-on: ubuntu-latest - continue-on-error: true steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -124,7 +123,17 @@ jobs: fetch-depth: 0 - uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3 - name: Compare with the target branch + shell: bash env: BASE_REF: ${{ github.base_ref }} + # griffe exits with 1 when it finds breaking changes; this job only reports them. # Both search paths: the target branch may still have the package in the repository root. - run: uvx griffe check Exscript --search src --search . --against "origin/$BASE_REF" --verbose + run: | + status=0 + uvx griffe check Exscript --search src --search . --against "origin/$BASE_REF" --verbose \ + > api-changes.txt 2>&1 || status=$? + cat api-changes.txt + { + echo "### Breaking API changes against $BASE_REF" + if [ "$status" -eq 0 ]; then echo 'None.'; else echo '```'; cat api-changes.txt; echo '```'; fi + } >> "$GITHUB_STEP_SUMMARY" From 4ff58f6861d5cb1ff3bcf601feb40fd38da5e46d Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 3 Nov 2026 18:23:48 +0100 Subject: [PATCH 047/250] Build the documentation with Sphinx and furo, in CI and on Read the Docs The old configuration imported sphinx.apidoc, which Sphinx removed. The API reference is now generated from the docstrings at build time, and CI builds the documentation with warnings as errors. The sources move from doc/ to docs/. --- .github/workflows/ci.yml | 12 + .gitignore | 2 + .readthedocs.yaml | 18 + doc/Makefile | 220 ---------- doc/conf.py | 311 -------------- doc/install.rst | 35 -- {doc => docs}/_static/logo.png | Bin docs/api.rst | 10 + docs/changelog.md | 2 + {doc => docs}/cli_tutorial.rst | 0 docs/conf.py | 42 ++ {doc => docs}/exscript.rst | 0 {doc => docs}/index.rst | 18 +- docs/install.rst | 42 ++ {doc => docs}/python_tutorial.rst | 2 +- {doc => docs}/templates.rst | 8 +- {doc => docs}/trouble.rst | 0 pyproject.toml | 3 +- src/Exscript/protocols/protocol.py | 2 +- src/Exscript/util/template.py | 4 +- uv.lock | 644 ++++++++++++++++++++++++++++- 21 files changed, 786 insertions(+), 589 deletions(-) create mode 100644 .readthedocs.yaml delete mode 100644 doc/Makefile delete mode 100755 doc/conf.py delete mode 100644 doc/install.rst rename {doc => docs}/_static/logo.png (100%) create mode 100644 docs/api.rst create mode 100644 docs/changelog.md rename {doc => docs}/cli_tutorial.rst (100%) create mode 100755 docs/conf.py rename {doc => docs}/exscript.rst (100%) rename {doc => docs}/index.rst (81%) create mode 100644 docs/install.rst rename {doc => docs}/python_tutorial.rst (99%) rename {doc => docs}/templates.rst (99%) rename {doc => docs}/trouble.rst (100%) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5e0fb4c9..46ac62e7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -99,6 +99,18 @@ jobs: name: benchmarks path: bench.json + docs: + name: Documentation + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + fetch-depth: 0 + - uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3 + - name: Build the documentation (warnings are errors, like on Read the Docs) + run: uv run --frozen --python 3.14 --group docs sphinx-build -W --keep-going docs docs/_build/html + os-guessing: name: OS-guess accuracy runs-on: ubuntu-latest diff --git a/.gitignore b/.gitignore index 7f99e872..76cf3fde 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,5 @@ unit_test.cfg .ruff_cache htmlcov coverage.xml +docs/_build +docs/_autosummary diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 00000000..3fe79bf8 --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,18 @@ +# Read the Docs: https://docs.readthedocs.com/platform/stable/config-file/v2.html +version: 2 +build: + os: ubuntu-24.04 + tools: + python: "3.14" + jobs: + post_checkout: + # hatch-vcs takes the version from the git tags, which a shallow clone lacks. + - git fetch --unshallow --tags || true +python: + install: + - method: uv + command: sync + groups: [docs] +sphinx: + configuration: docs/conf.py + fail_on_warning: true diff --git a/doc/Makefile b/doc/Makefile deleted file mode 100644 index 7d33da65..00000000 --- a/doc/Makefile +++ /dev/null @@ -1,220 +0,0 @@ -# Makefile for Sphinx documentation -# - -# You can set these variables from the command line. -SPHINXOPTS = -SPHINXBUILD = sphinx-build -PAPER = -BUILDDIR = _build - -# User-friendly check for sphinx-build -ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) -$(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) -endif - -# Internal variables. -PAPEROPT_a4 = -D latex_paper_size=a4 -PAPEROPT_letter = -D latex_paper_size=letter -ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . -# the i18n builder cannot share the environment and doctrees with the others -I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . - -.PHONY: help -help: - @echo "Please use \`make ' where is one of" - @echo " html to make standalone HTML files" - @echo " dirhtml to make HTML files named index.html in directories" - @echo " singlehtml to make a single large HTML file" - @echo " pickle to make pickle files" - @echo " json to make JSON files" - @echo " htmlhelp to make HTML files and a HTML help project" - @echo " qthelp to make HTML files and a qthelp project" - @echo " applehelp to make an Apple Help Book" - @echo " devhelp to make HTML files and a Devhelp project" - @echo " epub to make an epub" - @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" - @echo " latexpdf to make LaTeX files and run them through pdflatex" - @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" - @echo " text to make text files" - @echo " man to make manual pages" - @echo " texinfo to make Texinfo files" - @echo " info to make Texinfo files and run them through makeinfo" - @echo " gettext to make PO message catalogs" - @echo " changes to make an overview of all changed/added/deprecated items" - @echo " xml to make Docutils-native XML files" - @echo " pseudoxml to make pseudoxml-XML files for display purposes" - @echo " linkcheck to check all external links for integrity" - @echo " doctest to run all doctests embedded in the documentation (if enabled)" - @echo " coverage to run coverage check of the documentation (if enabled)" - -.PHONY: clean -clean: - rm -rf $(BUILDDIR)/* - -.PHONY: apidoc -apidoc: - sphinx-apidoc -d5 -Mefo . ../Exscript `realpath ../Exscript/interpreter` `realpath ../Exscript/external` `realpath ../Exscript/parselib` `realpath ../Exscript/workqueue` - -.PHONY: html -html: - $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html - @echo - @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." - -.PHONY: dirhtml -dirhtml: - $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml - @echo - @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." - -.PHONY: singlehtml -singlehtml: - $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml - @echo - @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." - -.PHONY: pickle -pickle: - $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle - @echo - @echo "Build finished; now you can process the pickle files." - -.PHONY: json -json: - $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json - @echo - @echo "Build finished; now you can process the JSON files." - -.PHONY: htmlhelp -htmlhelp: - $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp - @echo - @echo "Build finished; now you can run HTML Help Workshop with the" \ - ".hhp project file in $(BUILDDIR)/htmlhelp." - -.PHONY: qthelp -qthelp: - $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp - @echo - @echo "Build finished; now you can run "qcollectiongenerator" with the" \ - ".qhcp project file in $(BUILDDIR)/qthelp, like this:" - @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/Exscript.qhcp" - @echo "To view the help file:" - @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/Exscript.qhc" - -.PHONY: applehelp -applehelp: - $(SPHINXBUILD) -b applehelp $(ALLSPHINXOPTS) $(BUILDDIR)/applehelp - @echo - @echo "Build finished. The help book is in $(BUILDDIR)/applehelp." - @echo "N.B. You won't be able to view it unless you put it in" \ - "~/Library/Documentation/Help or install it in your application" \ - "bundle." - -.PHONY: devhelp -devhelp: - $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp - @echo - @echo "Build finished." - @echo "To view the help file:" - @echo "# mkdir -p $$HOME/.local/share/devhelp/Exscript" - @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/Exscript" - @echo "# devhelp" - -.PHONY: epub -epub: - $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub - @echo - @echo "Build finished. The epub file is in $(BUILDDIR)/epub." - -.PHONY: latex -latex: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo - @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." - @echo "Run \`make' in that directory to run these through (pdf)latex" \ - "(use \`make latexpdf' here to do that automatically)." - -.PHONY: latexpdf -latexpdf: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo "Running LaTeX files through pdflatex..." - $(MAKE) -C $(BUILDDIR)/latex all-pdf - @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." - -.PHONY: latexpdfja -latexpdfja: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo "Running LaTeX files through platex and dvipdfmx..." - $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja - @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." - -.PHONY: text -text: - $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text - @echo - @echo "Build finished. The text files are in $(BUILDDIR)/text." - -.PHONY: man -man: - $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man - @echo - @echo "Build finished. The manual pages are in $(BUILDDIR)/man." - -.PHONY: texinfo -texinfo: - $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo - @echo - @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." - @echo "Run \`make' in that directory to run these through makeinfo" \ - "(use \`make info' here to do that automatically)." - -.PHONY: info -info: - $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo - @echo "Running Texinfo files through makeinfo..." - make -C $(BUILDDIR)/texinfo info - @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." - -.PHONY: gettext -gettext: - $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale - @echo - @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." - -.PHONY: changes -changes: - $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes - @echo - @echo "The overview file is in $(BUILDDIR)/changes." - -.PHONY: linkcheck -linkcheck: - $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck - @echo - @echo "Link check complete; look for any errors in the above output " \ - "or in $(BUILDDIR)/linkcheck/output.txt." - -.PHONY: doctest -doctest: - $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest - @echo "Testing of doctests in the sources finished, look at the " \ - "results in $(BUILDDIR)/doctest/output.txt." - -.PHONY: coverage -coverage: - $(SPHINXBUILD) -b coverage $(ALLSPHINXOPTS) $(BUILDDIR)/coverage - @echo "Testing of coverage in the sources finished, look at the " \ - "results in $(BUILDDIR)/coverage/python.txt." - -.PHONY: xml -xml: - $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml - @echo - @echo "Build finished. The XML files are in $(BUILDDIR)/xml." - -.PHONY: pseudoxml -pseudoxml: - $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml - @echo - @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." diff --git a/doc/conf.py b/doc/conf.py deleted file mode 100755 index 4bb96342..00000000 --- a/doc/conf.py +++ /dev/null @@ -1,311 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Exscript documentation build configuration file, created by -# sphinx-quickstart on Tue Sep 4 14:21:46 2012. -# -# This file is execfile()d with the current directory set to its containing dir. -# -# Note that not all possible configuration values are present in this -# autogenerated file. -# -# All configuration values have a default; values that are commented out -# serve to show the default. - -import sys, os - -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. -sys.path.insert(0, os.path.abspath('..')) - -# Auto-generate API documentation. -from sphinx.apidoc import main -apidoc_exclude = [os.path.join('..', 'Exscript', 'interpreter'), - os.path.join('..', 'Exscript', 'external'), - os.path.join('..', 'Exscript', 'parselib'), - os.path.join('..', 'Exscript', 'workqueue')] -main(['-d5', '-Mef', '-o', '.', '../Exscript'] - + [os.path.abspath(p) for p in apidoc_exclude]) - -# -- General configuration ----------------------------------------------------- - -# If your documentation needs a minimal Sphinx version, state it here. -#needs_sphinx = '1.0' - -# Add any Sphinx extension module names here, as strings. They can be extensions -# coming with Sphinx (named 'sphinx.ext.*') or your custom ones. -#extensions = ['sphinx.ext.autodoc', 'sphinx.ext.doctest', 'sphinx.ext.todo', 'sphinx.ext.coverage', 'sphinx.ext.pngmath', 'sphinx.ext.mathjax', 'sphinx.ext.ifconfig', 'sphinx.ext.viewcode'] -extensions = ['sphinx.ext.autodoc', # 'sphinx.ext.coverage', - 'sphinx.ext.viewcode', - 'sphinx.ext.autosummary', - #'sphinx.ext.intersphinx', - ] - -# Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] - -# The suffix of source filenames. -source_suffix = '.rst' - -# The encoding of source files. -#source_encoding = 'utf-8-sig' - -# The master toctree document. -master_doc = 'index' - -# General information about the project. -project = u'Exscript' -copyright = u'2017, Samuel Abels' - -# The version info for the project you're documenting, acts as replacement for -# |version| and |release|, also used in various other places throughout the -# built documents. -# -# The short X.Y version. -version = '0.3' -# The full version, including alpha/beta/rc tags. -release = '0.3.4' - -# The language for content autogenerated by Sphinx. Refer to documentation -# for a list of supported languages. -#language = None - -# There are two options for replacing |today|: either, you set today to some -# non-false value, then it is used: -#today = '' -# Else, today_fmt is used as the format for a strftime call. -#today_fmt = '%B %d, %Y' - -# List of patterns, relative to source directory, that match files and -# directories to ignore when looking for source files. -exclude_patterns = ['_build'] - -# The reST default role (used for this markup: `text`) to use for all documents. -#default_role = None - -# If true, '()' will be appended to :func: etc. cross-reference text. -#add_function_parentheses = True - -# If true, the current module name will be prepended to all description -# unit titles (such as .. function::). -#add_module_names = True - -# If true, sectionauthor and moduleauthor directives will be shown in the -# output. They are ignored by default. -#show_authors = False - -# The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' - -# A list of ignored prefixes for module index sorting. -#modindex_common_prefix = [] - - -# -- Options for HTML output --------------------------------------------------- - -# The theme to use for HTML and HTML Help pages. See the documentation for -# a list of builtin themes. -html_theme = 'default' - -# Theme options are theme-specific and customize the look and feel of a theme -# further. For a list of options available for each theme, see the -# documentation. -#html_theme_options = {} - -# Add any paths that contain custom themes here, relative to this directory. -#html_theme_path = [] - -# The name for this set of Sphinx documents. If None, it defaults to -# " v documentation". -#html_title = None - -# A shorter title for the navigation bar. Default is the same as html_title. -#html_short_title = None - -# The name of an image file (relative to this directory) to place at the top -# of the sidebar. -#html_logo = None - -# The name of an image file (within the static path) to use as favicon of the -# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 -# pixels large. -#html_favicon = None - -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] - -# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, -# using the given strftime format. -#html_last_updated_fmt = '%b %d, %Y' - -# If true, SmartyPants will be used to convert quotes and dashes to -# typographically correct entities. -#html_use_smartypants = True - -# Custom sidebar templates, maps document names to template names. -#html_sidebars = {} - -# Additional templates that should be rendered to pages, maps page names to -# template names. -#html_additional_pages = {} - -# If false, no module index is generated. -#html_domain_indices = True - -# If false, no index is generated. -#html_use_index = True - -# If true, the index is split into individual pages for each letter. -#html_split_index = False - -# If true, links to the reST sources are added to the pages. -#html_show_sourcelink = True - -# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. -#html_show_sphinx = True - -# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. -#html_show_copyright = True - -# If true, an OpenSearch description file will be output, and all pages will -# contain a tag referring to it. The value of this option must be the -# base URL from which the finished HTML is served. -#html_use_opensearch = '' - -# This is the file name suffix for HTML files (e.g. ".xhtml"). -#html_file_suffix = None - -# Output file base name for HTML help builder. -htmlhelp_basename = 'Exscript' - - -# -- Options for LaTeX output -------------------------------------------------- - -latex_elements = { -# The paper size ('letterpaper' or 'a4paper'). -#'papersize': 'letterpaper', - -# The font size ('10pt', '11pt' or '12pt'). -#'pointsize': '10pt', - -# Additional stuff for the LaTeX preamble. -#'preamble': '', -} - -# Grouping the document tree into LaTeX files. List of tuples -# (source start file, target name, title, author, documentclass [howto/manual]). -latex_documents = [ - ('index', 'Exscript.tex', u'Exscript Documentation', - u'Samuel Abels', 'manual'), -] - -# The name of an image file (relative to this directory) to place at the top of -# the title page. -#latex_logo = None - -# For "manual" documents, if this is true, then toplevel headings are parts, -# not chapters. -#latex_use_parts = False - -# If true, show page references after internal links. -#latex_show_pagerefs = False - -# If true, show URL addresses after external links. -#latex_show_urls = False - -# Documents to append as an appendix to all manuals. -#latex_appendices = [] - -# If false, no module index is generated. -#latex_domain_indices = True - - -# -- Options for manual page output -------------------------------------------- - -# One entry per manual page. List of tuples -# (source start file, name, description, authors, manual section). -man_pages = [ - ('index', 'exscript', u'Exscript Documentation', - [u'Samuel Abels'], 1) -] - -# If true, show URL addresses after external links. -#man_show_urls = False - - -# -- Options for Texinfo output ------------------------------------------------ - -# Grouping the document tree into Texinfo files. List of tuples -# (source start file, target name, title, author, -# dir menu entry, description, category) -texinfo_documents = [ - ('index', 'Exscript', u'Exscript Documentation', - u'Samuel Abels', 'Exscript', 'One line description of project.', - 'Miscellaneous'), -] - -# Documents to append as an appendix to all manuals. -#texinfo_appendices = [] - -# If false, no module index is generated. -#texinfo_domain_indices = True - -# How to display URL addresses: 'footnote', 'no', or 'inline'. -#texinfo_show_urls = 'footnote' - - -# -- Options for Epub output --------------------------------------------------- - -# Bibliographic Dublin Core info. -epub_title = u'Exscript' -epub_author = u'Samuel Abels' -epub_publisher = u'Samuel Abels' -epub_copyright = u'2017, Samuel Abels' - -# The language of the text. It defaults to the language option -# or en if the language is not set. -#epub_language = '' - -# The scheme of the identifier. Typical schemes are ISBN or URL. -#epub_scheme = '' - -# The unique identifier of the text. This can be a ISBN number -# or the project homepage. -#epub_identifier = '' - -# A unique identification for the text. -#epub_uid = '' - -# A tuple containing the cover image and cover page html template filenames. -#epub_cover = () - -# HTML files that should be inserted before the pages created by sphinx. -# The format is a list of tuples containing the path and title. -#epub_pre_files = [] - -# HTML files shat should be inserted after the pages created by sphinx. -# The format is a list of tuples containing the path and title. -#epub_post_files = [] - -# A list of files that should not be packed into the epub file. -#epub_exclude_files = [] - -# The depth of the table of contents in toc.ncx. -#epub_tocdepth = 3 - -# Allow duplicate toc entries. -#epub_tocdup = True - - -# Example configuration for intersphinx: refer to the Python standard library. -intersphinx_mapping = {'http://docs.python.org/': None} - -def skip(app, what, name, obj, skip, options): - if name == "__init__": - return False - return skip - -def setup(app): - app.connect("autodoc-skip-member", skip) diff --git a/doc/install.rst b/doc/install.rst deleted file mode 100644 index 91ac1015..00000000 --- a/doc/install.rst +++ /dev/null @@ -1,35 +0,0 @@ -Installation -============ - -Prerequisites -------------- - -Exscript requires Python 2.7, and the following modules: - -.. literalinclude:: ../requirements.txt - -Installing with PIP -------------------- - -:: - - sudo pip install exscript - -Installing from GitHub ----------------------- - -:: - - git clone git://github.com/knipknap/exscript - cd exscript - sudo make install - -Running the automated test suite --------------------------------- - -If you installed from GitHub, you can run the integrated testsuite:: - - make tests - -There shouldn't be any errors, so if something comes up, -`please file a bug `_. diff --git a/doc/_static/logo.png b/docs/_static/logo.png similarity index 100% rename from doc/_static/logo.png rename to docs/_static/logo.png diff --git a/docs/api.rst b/docs/api.rst new file mode 100644 index 00000000..204e8c08 --- /dev/null +++ b/docs/api.rst @@ -0,0 +1,10 @@ +API Reference +============= + +The API reference is generated from the docstrings of the source code. + +.. autosummary:: + :toctree: _autosummary + :recursive: + + Exscript diff --git a/docs/changelog.md b/docs/changelog.md new file mode 100644 index 00000000..c6d99d60 --- /dev/null +++ b/docs/changelog.md @@ -0,0 +1,2 @@ +```{include} ../CHANGES.md +``` diff --git a/doc/cli_tutorial.rst b/docs/cli_tutorial.rst similarity index 100% rename from doc/cli_tutorial.rst rename to docs/cli_tutorial.rst diff --git a/docs/conf.py b/docs/conf.py new file mode 100755 index 00000000..ad98a624 --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,42 @@ +""" +Sphinx configuration for the Exscript documentation. + +Build it from the repository root:: + + uv run --group docs sphinx-build -W --keep-going docs docs/_build/html +""" + +import importlib.metadata + +project = "Exscript" +author = "Samuel Abels and contributors" +copyright = "2007-2026, Samuel Abels and contributors" +release = importlib.metadata.version("Exscript") +version = ".".join(release.split(".")[:2]) + +extensions = [ + "sphinx.ext.autodoc", + "sphinx.ext.autosummary", + "sphinx.ext.intersphinx", + "sphinx.ext.viewcode", + "myst_parser", # for the changelog, which is Markdown +] +source_suffix = {".rst": "restructuredtext", ".md": "markdown"} +exclude_patterns = ["_build"] + +# The API reference is generated from the docstrings at build time. +autosummary_generate = True +autodoc_default_options = { + "members": True, + "show-inheritance": True, + # Document each class once, in the module that defines it, not again + # where a package re-exports it (e.g. Exscript.Account). + "ignore-module-all": True, +} +autodoc_member_order = "bysource" +intersphinx_mapping = {"python": ("https://docs.python.org/3", None)} + +html_theme = "furo" +html_title = project # the version is long between releases, e.g. 3.0.1.dev4+g1a2b3c4 +html_logo = "_static/logo.png" +html_static_path = ["_static"] diff --git a/doc/exscript.rst b/docs/exscript.rst similarity index 100% rename from doc/exscript.rst rename to docs/exscript.rst diff --git a/doc/index.rst b/docs/index.rst similarity index 81% rename from doc/index.rst rename to docs/index.rst index ed00fa26..ce05410d 100755 --- a/doc/index.rst +++ b/docs/index.rst @@ -1,15 +1,11 @@ .. image:: _static/logo.png - :target: http://exscript.readthedocs.io + :target: https://exscript.readthedocs.io -.. image:: https://travis-ci.org/knipknap/exscript.svg?branch=master - :target: https://travis-ci.org/knipknap/exscript +.. image:: https://github.com/knipknap/exscript/actions/workflows/ci.yml/badge.svg?branch=master + :target: https://github.com/knipknap/exscript/actions/workflows/ci.yml -.. image:: https://coveralls.io/repos/github/knipknap/exscript/badge.svg?branch=master - :target: https://coveralls.io/github/knipknap/exscript?branch=master - -.. image:: https://lima.codeclimate.com/github/knipknap/exscript/badges/gpa.svg - :target: https://lima.codeclimate.com/github/knipknap/exscript - :alt: Code Climate +.. image:: https://img.shields.io/pypi/v/Exscript.svg + :target: https://pypi.org/project/Exscript/ .. image:: https://img.shields.io/github/stars/knipknap/exscript.svg :target: https://github.com/knipknap/exscript/stargazers @@ -17,7 +13,6 @@ .. image:: https://img.shields.io/github/license/knipknap/exscript.svg :target: https://github.com/knipknap/exscript/blob/master/COPYING -| What is Exscript? ================= @@ -80,4 +75,5 @@ Contents exscript templates trouble - API Documentation + api + changelog diff --git a/docs/install.rst b/docs/install.rst new file mode 100644 index 00000000..a8f64cbe --- /dev/null +++ b/docs/install.rst @@ -0,0 +1,42 @@ +Installation +============ + +Prerequisites +------------- + +Exscript requires Python 3.12 or later and runs on Linux, macOS and Windows. +pip and uv install the other required packages, among them paramiko for SSH +and ncclient for NETCONF. + +Installing with pip or uv +------------------------- + +:: + + pip install Exscript + +or, in a project managed with uv:: + + uv add Exscript + +To trace sessions with OpenTelemetry, install the ``otel`` extra, e.g. +``pip install "Exscript[otel]"``. + +Installing from GitHub +---------------------- + +:: + + git clone https://github.com/knipknap/exscript + cd exscript + uv sync + +Running the automated test suite +-------------------------------- + +In a checkout from GitHub, run the test suite with:: + + uv run pytest + +There shouldn\x27t be any errors, so if something comes up, +`please file a bug `_. diff --git a/doc/python_tutorial.rst b/docs/python_tutorial.rst similarity index 99% rename from doc/python_tutorial.rst rename to docs/python_tutorial.rst index ca8cd067..f34e5d83 100644 --- a/doc/python_tutorial.rst +++ b/docs/python_tutorial.rst @@ -15,7 +15,7 @@ complete. We will also assume that you have at least a little bit of programming experience, though most of the examples should be pretty simple. -Exscript also has extensive :doc:`API documentation `, which may +Exscript also has extensive :doc:`API documentation `, which may be used as a reference throughout this tutorial. Getting started diff --git a/doc/templates.rst b/docs/templates.rst similarity index 99% rename from doc/templates.rst rename to docs/templates.rst index f86ed8be..92b2ec0a 100644 --- a/doc/templates.rst +++ b/docs/templates.rst @@ -394,12 +394,8 @@ For example, the following function instructs Exscript to wait for {sys.wait(10)} -For a list of supported functions please check here: - -.. toctree:: - :maxdepth: 3 - - Exscript.stdlib +The supported functions are documented in the API reference of +:doc:`Exscript.stdlib <_autosummary/Exscript.stdlib>`. Exiting A Script ~~~~~~~~~~~~~~~~ diff --git a/doc/trouble.rst b/docs/trouble.rst similarity index 100% rename from doc/trouble.rst rename to docs/trouble.rst diff --git a/pyproject.toml b/pyproject.toml index 36599e1b..f0c12fde 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -64,6 +64,7 @@ test = [ "hypothesis>=6.170.14", ] types = ["mypy>=2.4"] +docs = ["sphinx>=9", "furo>=2025.12", "myst-parser>=5"] dev = [{ include-group = "test" }, { include-group = "types" }] [tool.hatch.version] @@ -80,7 +81,7 @@ packages = ["src/Exscript"] line-length = 100 src = ["src", "tests"] extend-include = ["scripts/exscript", "scripts/otp"] -extend-exclude = ["doc"] +extend-exclude = ["docs/_build"] [tool.ruff.format] quote-style = "preserve" diff --git a/src/Exscript/protocols/protocol.py b/src/Exscript/protocols/protocol.py index ec619d92..e3de3991 100644 --- a/src/Exscript/protocols/protocol.py +++ b/src/Exscript/protocols/protocol.py @@ -1576,7 +1576,7 @@ def waitfor(self, prompt): called in a callback. :raise ProtocolException: On other internal errors. :raise Exception: May raise other exceptions that are caused - within the underlying protocol implementations. + within the underlying protocol implementations. """ while True: try: diff --git a/src/Exscript/util/template.py b/src/Exscript/util/template.py index 526eceee..21319352 100644 --- a/src/Exscript/util/template.py +++ b/src/Exscript/util/template.py @@ -131,7 +131,7 @@ def eval(conn, string, strip_command=True, no_escaping=False, *, secure=True, ** touch $filename {end} - If strip_command is False, the response, (and hence, the `filenames' + If strip_command is False, the response, (and hence, the ``filenames`` variable) contains the following:: ls -1 @@ -194,7 +194,7 @@ def paste(conn, string, no_escaping=False, *, secure=True, **kwargs): connection. This function differs from eval() such that it does not wait for a prompt after sending each command to the connected host. That means that the script can no longer read the response of the - host, making commands such as `extract' or `set_prompt' useless. + host, making commands such as ``extract`` or ``set_prompt`` useless. The function raises an exception if the compilation fails, or if the template contains a command that requires a response from the diff --git a/uv.lock b/uv.lock index 609c4979..0c7586bc 100644 --- a/uv.lock +++ b/uv.lock @@ -3,7 +3,29 @@ revision = 5 requires-python = ">=3.12" resolution-markers = [ "python_full_version >= '3.15'", - "python_full_version < '3.15'", + "python_full_version >= '3.13' and python_full_version < '3.15'", + "python_full_version < '3.13'", +] + +[[package]] +name = "accessible-pygments" +version = "0.0.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/bc/c1/bbac6a50d02774f91572938964c582fff4270eee73ab822a4aeea4d8b11b/accessible_pygments-0.0.5.tar.gz", hash = "sha256:40918d3e6a2b619ad424cb91e556bd3bd8865443d9f22f1dcdf79e33c8046872", size = 1377899, upload-time = "2024-05-10T11:23:10.216Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8d/3f/95338030883d8c8b91223b4e21744b04d11b161a3ef117295d8241f50ab4/accessible_pygments-0.0.5-py3-none-any.whl", hash = "sha256:88ae3211e68a1d0b011504b2ffc1691feafce124b845bd072ab6f9f66f34d4b7", size = 1395903, upload-time = "2024-05-10T11:23:08.421Z" }, +] + +[[package]] +name = "alabaster" +version = "1.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a6/f8/d9c74d0daf3f742840fd818d69cfae176fa332022fd44e3469487d5a9420/alabaster-1.0.0.tar.gz", hash = "sha256:c00dca57bca26fa62a6d7d0a9fcce65f3e026e9bfe33e9c538fd3fbb2144fd9e", size = 24210, upload-time = "2024-07-26T18:15:03.762Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7e/b3/6b4067be973ae96ba0d615946e314c5ae35f9f993eca561b356540bb0c2b/alabaster-1.0.0-py3-none-any.whl", hash = "sha256:fc6786402dc3fcb2de3cabd5fe455a2db534b371124f1f21de8731783dec828b", size = 13929, upload-time = "2024-07-26T18:15:02.05Z" }, ] [[package]] @@ -70,6 +92,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/11/25/4235f4b375a67a37ac595bf478a5407a744eaf7b64113c1c3a5432b8d954/ast_serialize-0.13.1-cp39-abi3-win_arm64.whl", hash = "sha256:48f11d96c9ec71119612f8e79a8c55c12f2b136716d68347433c561d27a25f43", size = 1139035, upload-time = "2026-10-17T00:04:15.767Z" }, ] +[[package]] +name = "babel" +version = "2.18.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7d/b2/51899539b6ceeeb420d40ed3cd4b7a40519404f9baf3d4ac99dc413a834b/babel-2.18.0.tar.gz", hash = "sha256:b80b99a14bd085fcacfa15c9165f651fbb3406e66cc603abf11c5750937c992d", size = 9959554, upload-time = "2026-02-01T12:30:56.078Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/77/f5/21d2de20e8b8b0408f0681956ca2c69f1320a3848ac50e6e7f39c6159675/babel-2.18.0-py3-none-any.whl", hash = "sha256:e2b422b277c2b9a9630c1d7903c2a00d0830c409c59ac8cae9081c92f1aeba35", size = 10196845, upload-time = "2026-02-01T12:30:53.445Z" }, +] + [[package]] name = "bcrypt" version = "5.0.0" @@ -136,6 +167,28 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/27/44/d2ef5e87509158ad2187f4dd0852df80695bb1ee0cfe0a684727b01a69e0/bcrypt-5.0.0-cp39-abi3-win_arm64.whl", hash = "sha256:f2347d3534e76bf50bca5500989d6c1d05ed64b440408057a37673282c654927", size = 144953, upload-time = "2025-09-25T19:50:37.32Z" }, ] +[[package]] +name = "beautifulsoup4" +version = "4.15.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "soupsieve" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/43/65/318323f98dbee45d42dff61d8f047181bc6f2268a9068cfad035a46be5af/beautifulsoup4-4.15.0.tar.gz", hash = "sha256:288e3ca7d54b06f2ac191970bc275c1939cb46d450b255bf6718b04aa37ab4f7", size = 632571, upload-time = "2026-06-07T16:44:20.453Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/88/c6/92fcd42f1ba33e1184263f25bfabf3d27c383410470f169e4b8163bf9c17/beautifulsoup4-4.15.0-py3-none-any.whl", hash = "sha256:d6f88de62e1d4e38ecb1077eb9724cd0eff29d2a08ca16a401e9b9e93f117cf9", size = 109924, upload-time = "2026-06-07T16:44:21.566Z" }, +] + +[[package]] +name = "certifi" +version = "2026.10.23" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fd/11/b524cbd9bbdaca58608e4cf4053fd463417549e1959fa8d3aa18319f9555/certifi-2026.10.23.tar.gz", hash = "sha256:a9895e6e2758b371e0545e321a3a8fd1fcfd9829ada8824a936dfb16aaaf36cd", size = 136225, upload-time = "2026-10-23T11:36:16.478Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/32/d1/a3652fe2c19134d0d15922177821d863cbd7e842d53a21bbe7e8e3a1b5ac/certifi-2026.10.23-py3-none-any.whl", hash = "sha256:e9b54de76e530bb6cf0ccd036787103acb58fd42ba930ea8bf948ddbaa2f61d1", size = 135082, upload-time = "2026-10-23T11:36:14.763Z" }, +] + [[package]] name = "cffi" version = "2.1.1" @@ -221,6 +274,137 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/aa/29/35e016098c814cd93de9cd320c66b5bfba14dc6ecedd3cb518fa7c408c69/cffi-2.1.1-cp315-cp315t-win_arm64.whl", hash = "sha256:d18e5ac0f2f03f4f518d3e23db0f0cad7faa1da8620e9c09461d443bbf6e6692", size = 186360, upload-time = "2026-08-03T21:21:13.636Z" }, ] +[[package]] +name = "charset-normalizer" +version = "3.5.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/33/1c/f41d4e74c28ab327ff3acd36053f7ea506c55872d7a90b0fa71aa3ab0c89/charset_normalizer-3.5.2.tar.gz", hash = "sha256:39de2a259fc954455c57274dc94c79d5842774e1247a016aff30bc0efed0f4ef", size = 172659, upload-time = "2026-09-30T04:39:23.398Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e7/c8/693809898870237d82785a03f3b2b58fe4c9f14669f84a7d4e623c92a59e/charset_normalizer-3.5.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:ed2a239c0ea213acc1908150a3037257083c7c083128f1a4cec2ec4b97dca491", size = 367780, upload-time = "2026-09-30T04:35:30.888Z" }, + { url = "https://files.pythonhosted.org/packages/c9/87/2fea8c13dc24b3ca9c6f803a5b2dfdeae73eb4f9e12c7885ed908ff0433c/charset_normalizer-3.5.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b91363207bd9dc966a691e959bb47f64b30f7ac4b072be9968b366982f7db77c", size = 246730, upload-time = "2026-09-30T04:35:32.286Z" }, + { url = "https://files.pythonhosted.org/packages/a8/9e/09efac30b937722f46d3110ba30b875b24b2e3a266ed746cc4e376a94d80/charset_normalizer-3.5.2-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:38a873987f3be698494da8b2e3085e29da02da7b633dce73e79c699a113d7bf0", size = 237707, upload-time = "2026-09-30T04:35:33.709Z" }, + { url = "https://files.pythonhosted.org/packages/9e/18/70d76670b13686237863a379928d60bd10e021f17d243ab3d7014c4a5f4e/charset_normalizer-3.5.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:355ad8011081dec5412240c087a9a0c9d4d5039f3ed11a3f13e18c2b29b56c51", size = 273050, upload-time = "2026-09-30T04:35:35.138Z" }, + { url = "https://files.pythonhosted.org/packages/54/e2/77a8b09d5adc013ed07b95b01b8b8fa5441c4e810e83ee7e4aae2fa4d91a/charset_normalizer-3.5.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ee21e28f0430bd6dc9086c6e525d5e818a44a5ad19720c8a0ef766792f3eb5e5", size = 270345, upload-time = "2026-09-30T04:35:36.502Z" }, + { url = "https://files.pythonhosted.org/packages/7f/c5/38806a25ab5e65fc178f39affeda20858efafede2fce1ffc2556cfc9fe73/charset_normalizer-3.5.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3d31298449090ab8d47b7b1b2a555ff73cac7ed438a08b7ac160980c7ebed649", size = 257601, upload-time = "2026-09-30T04:35:37.919Z" }, + { url = "https://files.pythonhosted.org/packages/ae/8d/213565184708fdb263ae55e2c04ee1ff748129dd65d48ed0e3502da9c85a/charset_normalizer-3.5.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5cde776b7cc66e4f6c99612cea4aa7269aa65863f7a15841b2c264f103822f4e", size = 252222, upload-time = "2026-09-30T04:35:39.544Z" }, + { url = "https://files.pythonhosted.org/packages/7e/24/76d2cefc25472531e4c5c7dfff68865eb1c39b78482f0fdc15b46f047830/charset_normalizer-3.5.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ae4f5fea5b8b8ccff88238cc8569303e5ee95efae67fa62922a311397a71f346", size = 248482, upload-time = "2026-09-30T04:35:41.088Z" }, + { url = "https://files.pythonhosted.org/packages/7d/dc/65a801b66ab4c197e22c433ab25e7ac24324ac6f45a2269aca42cce309bf/charset_normalizer-3.5.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:f7d486c83842422badd511868fd8a9a20e9407ace71564b6af47ce7e60a336c1", size = 241206, upload-time = "2026-09-30T04:35:42.59Z" }, + { url = "https://files.pythonhosted.org/packages/a7/95/ca9b5eabde673002c6f1e7ada1b223916fe18f6d661da7aabd4d643718f1/charset_normalizer-3.5.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:11a4d68a6ecda3292cb1e50239e111543ba5d709bb62a6b4ea1afcfa729d8875", size = 273190, upload-time = "2026-09-30T04:35:44.347Z" }, + { url = "https://files.pythonhosted.org/packages/2d/8b/803b4d2a3f6e1740f63f1e87b04d14b42f3d4fdfe6ed7d4db2d34102b14f/charset_normalizer-3.5.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:d6734d2ef8a50fbf8445c139477da401f50d62a0606bf00e20ec6d87773fefb1", size = 253527, upload-time = "2026-09-30T04:35:45.915Z" }, + { url = "https://files.pythonhosted.org/packages/a9/55/93c0e5dbd085ae0471346026abbe7e0db9ea2d6fea74e51f0b5a46f233a7/charset_normalizer-3.5.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:a815775b6c38d4e0ff7bcffbeba67feded90202bb6a226b8dd35f1c855217413", size = 271285, upload-time = "2026-09-30T04:35:47.49Z" }, + { url = "https://files.pythonhosted.org/packages/95/69/0dbd0e0b9b16cfa816cdfcb3e2e3854a1f680dc07fb1245ea125e7448060/charset_normalizer-3.5.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:23851fb4e1b85ed3f6c2a27b777cdfe2e19fb5b38429a8faf38c7542b7665869", size = 260010, upload-time = "2026-09-30T04:35:48.996Z" }, + { url = "https://files.pythonhosted.org/packages/58/9d/e7b88e7b1bf403590c3b573277b5e1e488c68c7a6fbacca310a2c324e90c/charset_normalizer-3.5.2-cp312-cp312-win32.whl", hash = "sha256:db19d07e2e0129e974a0e65d0064fc222a446cd5122c2fd4184d2af9fc734a9e", size = 184125, upload-time = "2026-09-30T04:35:50.777Z" }, + { url = "https://files.pythonhosted.org/packages/eb/e6/e6e083884cbcfd49c64865af05027fe7011be7b2d9179524f099a1b611f3/charset_normalizer-3.5.2-cp312-cp312-win_amd64.whl", hash = "sha256:780fbe7cab297b81dad9fb8dc5eb003c0468ffb0d9e5f65068c53a34661a96bc", size = 207486, upload-time = "2026-09-30T04:35:52.194Z" }, + { url = "https://files.pythonhosted.org/packages/c4/e3/017aea0911ada7405a825c7d937eb3a13009664e2f5b38e8c4bbf2abf894/charset_normalizer-3.5.2-cp312-cp312-win_arm64.whl", hash = "sha256:e2af3aad578aa6bd1384bcf4750fc285e5a9de53f40b7d41e5a0bf748edeb2b3", size = 196734, upload-time = "2026-09-30T04:35:53.636Z" }, + { url = "https://files.pythonhosted.org/packages/c5/34/68292d68512768591aaff07c59bb53ee31341c87759433a859c4641a50c2/charset_normalizer-3.5.2-cp313-cp313-android_24_arm64_v8a.whl", hash = "sha256:ed905975ab14056a2e5eb1c376cb2e1ebc5396baf84163939c518556fccde9f5", size = 225946, upload-time = "2026-09-30T04:35:55.313Z" }, + { url = "https://files.pythonhosted.org/packages/e3/80/bee0b01b90ccd5322ae1d0abb33fab1bd95b7c2eadaf02aeccf22e04ee83/charset_normalizer-3.5.2-cp313-cp313-android_24_x86_64.whl", hash = "sha256:a66c3bc5ab1f0ff2164fc9965ddd611ff0802173f4b9d24554c563f6ab7e1d6e", size = 238619, upload-time = "2026-09-30T04:35:56.863Z" }, + { url = "https://files.pythonhosted.org/packages/78/6e/60ce52a85a7fd631ae8482ae6d74521014ca2f255892679484dc04d7ef56/charset_normalizer-3.5.2-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:d2374b62878abb00cd8309b32af6c0b715cd02dec0ca74ef12e5069bdc64144a", size = 206701, upload-time = "2026-09-30T04:35:58.639Z" }, + { url = "https://files.pythonhosted.org/packages/36/8c/71aafad23f971afc84c2b295bc0c560739ce1dac558aad9fec22e39f3639/charset_normalizer-3.5.2-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:d376bbd28b3a8999db1a103b3b388aee6f1ddeb3e51bc2172993efdcd86e064d", size = 210506, upload-time = "2026-09-30T04:36:00.147Z" }, + { url = "https://files.pythonhosted.org/packages/91/da/3c5a7798c046df7d2d68ad653cf5b6c5a8bfee225055a843c6f2f42aac1a/charset_normalizer-3.5.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:6045373d5a89a5ec71afde535db987ca28e76dfa276c2d4c818265b375d4b055", size = 366626, upload-time = "2026-09-30T04:36:01.77Z" }, + { url = "https://files.pythonhosted.org/packages/e1/16/710ac3de2ee354e2bd1a9c94efe45a2d27b5c6ad39b2d6a905be2c094b6c/charset_normalizer-3.5.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:849df64e889b2e17230d58410a03dba311a65b163508fd33679b2b737d4b7858", size = 244624, upload-time = "2026-09-30T04:36:03.389Z" }, + { url = "https://files.pythonhosted.org/packages/d6/39/45c7439f5b63d24f7d5b2a1d760f34af7628782d7144b4cc8ded45c2d4bc/charset_normalizer-3.5.2-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:15c44f7edfd477b06f517a5cc317fc1707edb9de2c865f43d4b6513907473234", size = 235731, upload-time = "2026-09-30T04:36:04.987Z" }, + { url = "https://files.pythonhosted.org/packages/4d/34/38f3154785ce92e9f56eb226f4d35bdfae6b008480dd055f58837a89c810/charset_normalizer-3.5.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a89012d6d5476ee112d20d998570ed58df2260a852afb1758809cd6900411d21", size = 270352, upload-time = "2026-09-30T04:36:06.412Z" }, + { url = "https://files.pythonhosted.org/packages/04/f3/859f74e7babc977705026b30593b3be04049632a522fb7000f83c033d747/charset_normalizer-3.5.2-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:0c951d5e6dd9c2ff60609476752bee49da4206adde960ebc247766937f72e718", size = 267393, upload-time = "2026-09-30T04:36:07.865Z" }, + { url = "https://files.pythonhosted.org/packages/4b/85/41d27f234b82e47c167a5f6c0f62501dc0c640585ff4aba79e08a390336a/charset_normalizer-3.5.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7218e8f32b0956cfcd048fd42d9d5779809745ca1d86113ca56f66e7ae1549c4", size = 254660, upload-time = "2026-09-30T04:36:09.248Z" }, + { url = "https://files.pythonhosted.org/packages/58/ca/5d1a997587febe5b26d8daffe363b5c1a091cece19828eec6502fd09c5ef/charset_normalizer-3.5.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a19a731138fc27d5682277d3b9df22855cea1239bce7fcec5f78f42ef2d1f3c3", size = 249996, upload-time = "2026-09-30T04:36:10.73Z" }, + { url = "https://files.pythonhosted.org/packages/b3/1f/d1e78246f7ed60c8c8d606b4ac27f66ce49cc3e95f24893ccbeba9f77302/charset_normalizer-3.5.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:62603db9a7caa0802eaa28c1c46fecd7b3a263a774069c24c3c28c302448721c", size = 247028, upload-time = "2026-09-30T04:36:12.294Z" }, + { url = "https://files.pythonhosted.org/packages/8e/37/eba316edd4f0c4d3a5d945924c4eeeae59abac4056aa815d8a4268f863a2/charset_normalizer-3.5.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:b6856554c4f44d79fc2307d5768854310a8f0096e501c75637542c82292b0429", size = 240562, upload-time = "2026-09-30T04:36:13.887Z" }, + { url = "https://files.pythonhosted.org/packages/c8/8e/aaa037d40ca9ef045977f1a661048b1aa33f223adfce3452fe9be9f79d14/charset_normalizer-3.5.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:1bc0baf5ef96b6ede57d47f4b8fe4d9d84019c3bfcbeb20a41edc6a6ee341f1f", size = 271699, upload-time = "2026-09-30T04:36:15.41Z" }, + { url = "https://files.pythonhosted.org/packages/26/19/1c1c9f75974adf523b87f34b8a2adc5a435cd65916812bcbd0dfa45f9a29/charset_normalizer-3.5.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:56bc200a365efb37383b7852e4cc5898d3b2da5987289b543956cf8cad71018a", size = 252030, upload-time = "2026-09-30T04:36:16.839Z" }, + { url = "https://files.pythonhosted.org/packages/bc/90/0660ef18e18df0a4d2a1a0edff7dfbba42d4e50ef2425557a5bb7051f77b/charset_normalizer-3.5.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:2c9ad19a6cfcd5ea5c0d41161d22f9df1dcc277e9bef2751391334546a314c00", size = 268565, upload-time = "2026-09-30T04:36:18.468Z" }, + { url = "https://files.pythonhosted.org/packages/79/ba/57adc269824e8658f1a0f97a9e514c247445a9632b3419b97e0ba37f16dc/charset_normalizer-3.5.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:e243bd13217235fc7290c621941c3f5cc8b66e4872495be821d7436ba2fb838d", size = 257627, upload-time = "2026-09-30T04:36:19.938Z" }, + { url = "https://files.pythonhosted.org/packages/9a/85/33abd4315c052d3d4f54c92b1ee49bfbc0dc7115a981e462a793b6d2ab87/charset_normalizer-3.5.2-cp313-cp313-pyemscripten_2025_0_wasm32.whl", hash = "sha256:a090bb2c68df85450502e3e20d665e3a5af9c65a84d6508ed477badd49166fd3", size = 143407, upload-time = "2026-09-30T04:36:21.376Z" }, + { url = "https://files.pythonhosted.org/packages/4f/de/6435e18d1aaa5d910b896d551411c96af1f42a0c56c29afc2016c61ccc2e/charset_normalizer-3.5.2-cp313-cp313-win32.whl", hash = "sha256:2b7b3bbfb4fe8ef40600792d762fbaa9057559f9d3fad209525b7a22b99e91fd", size = 183207, upload-time = "2026-09-30T04:36:22.776Z" }, + { url = "https://files.pythonhosted.org/packages/9c/76/b8ec57f4e9ee3253541abf95e4a462c0175fe8032dcd070f1f2421240942/charset_normalizer-3.5.2-cp313-cp313-win_amd64.whl", hash = "sha256:78456a747de8dc58360ffa581f30a002baf5aa28cb262536545e91f113ed7639", size = 206489, upload-time = "2026-09-30T04:36:24.306Z" }, + { url = "https://files.pythonhosted.org/packages/3e/60/c647c6ae47480221e875ea5d743ff94946f7416e3c69415ab772928e8d32/charset_normalizer-3.5.2-cp313-cp313-win_arm64.whl", hash = "sha256:11912e4bb14baae7c5d8791aa55ba0a3a03ec6729073307b0f57270abaa713d3", size = 195501, upload-time = "2026-09-30T04:36:25.846Z" }, + { url = "https://files.pythonhosted.org/packages/58/ca/7aa91362a2f77ac8e9e28a9b902a74f7d0e11a851ef0d27a74308da8cd90/charset_normalizer-3.5.2-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:1afb975bd5d68d5ce9f6b6d44fdf2f7e34b895a35e95708a7a91b20a3b51d187", size = 226010, upload-time = "2026-09-30T04:36:27.669Z" }, + { url = "https://files.pythonhosted.org/packages/a8/cf/ac8878d0322cf88a1aad4c7b147db32ca0bd806eb0060957b2e31486dbe6/charset_normalizer-3.5.2-cp314-cp314-android_24_x86_64.whl", hash = "sha256:bbbfc8e28816f19d7c0f1816664980c0a9875d01b27cdf8eedddb639d9e108ad", size = 239266, upload-time = "2026-09-30T04:36:29.434Z" }, + { url = "https://files.pythonhosted.org/packages/c9/6d/9a08d7e0b29b7208e2c6c01dc56c8e0520e7c7beadbbfb024b58fd69c8a5/charset_normalizer-3.5.2-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:7967d08cf06dee78443b874f98c98036f624f3a4e73e11f9f64f5be4d25393cf", size = 206783, upload-time = "2026-09-30T04:36:30.872Z" }, + { url = "https://files.pythonhosted.org/packages/82/44/b0aa350280e6ff5a5492d17cf10460dd39d5ee848f872f7ba2df10607f60/charset_normalizer-3.5.2-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:4c2b5031f63e331e3839b40aed2dd6f191e9c07edbde303e7876846ea1946995", size = 210449, upload-time = "2026-09-30T04:36:32.625Z" }, + { url = "https://files.pythonhosted.org/packages/7c/8a/40db9aa9f5907bb0e6f8b6d64064bf8852fb33d4b813ff9414911df7647c/charset_normalizer-3.5.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:fcff63213e8e6e47770541a4607175404f47cbb3ebea7b6058cc82d524a0e424", size = 367815, upload-time = "2026-09-30T04:36:34.197Z" }, + { url = "https://files.pythonhosted.org/packages/7f/72/9c5e7707b57c8ddfa9ddf7b0b1d009d7fbab9e9e887d5b721060f37e307d/charset_normalizer-3.5.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8d86d6fc60743dc916eb79e2eb1ec4818e21e427731543af40a3021851174a13", size = 247095, upload-time = "2026-09-30T04:36:35.803Z" }, + { url = "https://files.pythonhosted.org/packages/83/09/71e453691e927de4ddf792770cfaab3f49d494e222f66ea5e404bbd5e39c/charset_normalizer-3.5.2-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:7a881931aa470808df94a8c380eed2bbbc76cd9dc622310f99665658c821eb6d", size = 233942, upload-time = "2026-09-30T04:36:37.407Z" }, + { url = "https://files.pythonhosted.org/packages/9f/86/85c84e4da8b27dd409577d9437926ff581c5f9d3c66038dc68c1a526de51/charset_normalizer-3.5.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8024d00c3faf3fc0c16e07a69f4405e8eac7cc0ab15f65fe6cf43827c4cf72b4", size = 273762, upload-time = "2026-09-30T04:36:38.904Z" }, + { url = "https://files.pythonhosted.org/packages/92/08/564955a4b5f2ccb410ab480bbe8c6a18063ff27f2d35458731c4a5335df9/charset_normalizer-3.5.2-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4d48f2d08b9de5864e2c8744d4461b862fb149a18274abc8b698c45975573438", size = 269514, upload-time = "2026-09-30T04:36:40.469Z" }, + { url = "https://files.pythonhosted.org/packages/18/24/bad3ac4271589df29cf5ce2f5ae490518a5739358052bd0d61209e6fea54/charset_normalizer-3.5.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:34276fd796040bf0993ab33a369aa572e6979c7aab225a88893667ad8eac8f7a", size = 255056, upload-time = "2026-09-30T04:36:42.02Z" }, + { url = "https://files.pythonhosted.org/packages/d6/3e/350d89ad49916b86554d6f5f2d03ec1152148f87e5ff735106c6a03b1a36/charset_normalizer-3.5.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0521c5665880b33d603717defa76c094048900010897909952397feb3039da56", size = 251311, upload-time = "2026-09-30T04:36:43.577Z" }, + { url = "https://files.pythonhosted.org/packages/56/5b/4970a2d154df502e133402906dd04e3ae7cada7b3011283c88d0479a2585/charset_normalizer-3.5.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:eff0ac9dbe711a4aee69bf04a83896aa9b85f19641264053a9f6d48573abb7dd", size = 250190, upload-time = "2026-09-30T04:36:45.185Z" }, + { url = "https://files.pythonhosted.org/packages/88/8c/f1a91bddc8fb47c2889e29ea7ea49a194eb0d9868675d786806519c00d76/charset_normalizer-3.5.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:1503bccbeb36d5527790c3930327704c39af22de3112f1b1666a9f3ce15ee204", size = 237098, upload-time = "2026-09-30T04:36:46.689Z" }, + { url = "https://files.pythonhosted.org/packages/24/0e/bb5dace3cc7e79068425386a6589c19b5a2ab5fefc2a46abea6919683332/charset_normalizer-3.5.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:52aa6992700996af31f375de0c6bacd402b0097fe40b53c426b9f51a90ebabc7", size = 274228, upload-time = "2026-09-30T04:36:48.31Z" }, + { url = "https://files.pythonhosted.org/packages/9d/79/b849ad523017ea9f5a45581bbebed91439e0cf42fd2860a6f64e358eb5a6/charset_normalizer-3.5.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:e09a3942ecbdee5cce73ea9d42da82b81b72ac1bf031ce069b93b5adf4eac8cd", size = 252401, upload-time = "2026-09-30T04:36:50.091Z" }, + { url = "https://files.pythonhosted.org/packages/89/8c/75469d690cf47200bce8f6cad7655724fc23148e147abfc5ce78b5f65863/charset_normalizer-3.5.2-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:c7c9ab723cde841fefb34efbad91e87f00a674b1fe1cd0784fde742bf2c154dc", size = 271846, upload-time = "2026-09-30T04:36:51.719Z" }, + { url = "https://files.pythonhosted.org/packages/26/cd/6d52d3c7437cdcf2e310ce9f28f282e733d4ef60ed19105d1819c356255f/charset_normalizer-3.5.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ddc7dacc8ece3a182e7f15cb862d1fd616b46d076cb1ae9dd232b2c38b655874", size = 258151, upload-time = "2026-09-30T04:36:53.234Z" }, + { url = "https://files.pythonhosted.org/packages/f7/4c/070b38bdb5f49a70199fce923ec0726a49536a63ab262abbfcaaf351110b/charset_normalizer-3.5.2-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:ee43c17b173d46a3212baa6ead3ae258eeabdae48c263a01ccf0218c366dd655", size = 143570, upload-time = "2026-09-30T04:36:54.816Z" }, + { url = "https://files.pythonhosted.org/packages/81/84/9ebfc8ed6c8c4fcd8e726ff6bf220cc8deb3966e31dce9be8dd8aa017e64/charset_normalizer-3.5.2-cp314-cp314-win32.whl", hash = "sha256:4f87960d57feabfb618e4e0af6e7371645fa26a277860739d6e5d6e0012c92f0", size = 186412, upload-time = "2026-09-30T04:36:56.643Z" }, + { url = "https://files.pythonhosted.org/packages/d1/78/5ed86f743d4bc350db307e7636419a0a5ee1d91806d30c7f667bd5c80dae/charset_normalizer-3.5.2-cp314-cp314-win_amd64.whl", hash = "sha256:e4e81e09c1578b8df602e3db08b0b3ea0a6947ad612f52bf8dc5ea8d47691f0c", size = 210965, upload-time = "2026-09-30T04:36:58.205Z" }, + { url = "https://files.pythonhosted.org/packages/53/94/a3a7698e9b1a395e1eb99ccd9a324be9347973bff4e72db2a06496d7cd27/charset_normalizer-3.5.2-cp314-cp314-win_arm64.whl", hash = "sha256:80d02b6f04e92601a081dd97b23d3128033098bff5d35d392ddcc0476ea11253", size = 200406, upload-time = "2026-09-30T04:36:59.764Z" }, + { url = "https://files.pythonhosted.org/packages/c1/48/c5dd00d5ef7791f02666de250a5bb6071e29b7e133cf4b835800b6d3bc27/charset_normalizer-3.5.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:dca9ab98072a5a54ebacebdc45f53e645336b320c667410b061be1ca588ae709", size = 396871, upload-time = "2026-09-30T04:37:01.543Z" }, + { url = "https://files.pythonhosted.org/packages/12/c8/8379554b42e8368161d898476686947a0fdbd3e8865170d7909dcabfdee8/charset_normalizer-3.5.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f0aa869112ef88429ae17820d99c3dd9504c9e9c671d3c246f3d7442cb051084", size = 249711, upload-time = "2026-09-30T04:37:03.111Z" }, + { url = "https://files.pythonhosted.org/packages/4a/eb/2ddb1035d17320caa9f41682935123a9a250277b261c3efc86b2d2a21343/charset_normalizer-3.5.2-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c0afc6800ba57ccc350374c5bd6150419915d95ce93cdbab2d783d75eaf30ecb", size = 231146, upload-time = "2026-09-30T04:37:04.721Z" }, + { url = "https://files.pythonhosted.org/packages/4a/24/2ecb4bde104322cd7859d6594fcfa74649f8d90b3221c9feecbef149875b/charset_normalizer-3.5.2-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7dcd882da75ef9adf94903b1e3b9419e8aa8fb4c7396822b834b9ef7fb96954f", size = 271552, upload-time = "2026-09-30T04:37:06.295Z" }, + { url = "https://files.pythonhosted.org/packages/3f/98/9d5f6ebc3aee9fef5d30b4aff11fb2ab7a1222b4064f8ef2c7c87cde217a/charset_normalizer-3.5.2-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2e06a3a98f916dd41d27f3105e02e7a40181c98c94b9158733d03a6f80506c09", size = 269810, upload-time = "2026-09-30T04:37:07.905Z" }, + { url = "https://files.pythonhosted.org/packages/09/e1/a3b06a10461b1b7628853c934c644e03bc28e42767116afb52f19a56519b/charset_normalizer-3.5.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6bd128f206a7752ae1f2ab6c61bf8a24ba28913a10df8b14c2637b973ff97a80", size = 259845, upload-time = "2026-09-30T04:37:09.554Z" }, + { url = "https://files.pythonhosted.org/packages/fd/d3/6f561f74a296cf27d61775a1dc665ad13f3bff6a798810ca05907f37a7c4/charset_normalizer-3.5.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c8f3d67aeaf55f017982b73683f0e7342ba2f6635a78f69ce89ebb26aa411e5c", size = 252825, upload-time = "2026-09-30T04:37:11.274Z" }, + { url = "https://files.pythonhosted.org/packages/26/9f/69e13ca3b18f43e0eafcd34c04a45b732ae22a43b54a5fc9e119103356eb/charset_normalizer-3.5.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:fe9753dfee015c570d73df76f899f18444d41388bffcde097deba51c4fadbb9f", size = 252659, upload-time = "2026-09-30T04:37:12.941Z" }, + { url = "https://files.pythonhosted.org/packages/73/a9/ace29806a0dae18939919c76ba526472d83214afa101105fabff2cf30625/charset_normalizer-3.5.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:92888bb3187c5ba50500b00b3b310c9f2c651709d28036077680cb5255450a03", size = 238819, upload-time = "2026-09-30T04:37:14.659Z" }, + { url = "https://files.pythonhosted.org/packages/f8/c1/6116d52a2e3311ec80f21f5fb5e17b27405f10b9608af8f6e69516841a1b/charset_normalizer-3.5.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:d008d90a7f2471519aef0c90dfbe73b3e6e4d5e66ac48e19154c17e89e98b604", size = 272973, upload-time = "2026-09-30T04:37:16.346Z" }, + { url = "https://files.pythonhosted.org/packages/19/aa/9955c7e93bba10a9c7e8f7a5031b7ced66f3a1883a55c00712b8d5850ff3/charset_normalizer-3.5.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:31f3930700408d211f13378ccbe1c40845d8da54bd0681fac3a9b5aae81c7aa8", size = 253692, upload-time = "2026-09-30T04:37:18.212Z" }, + { url = "https://files.pythonhosted.org/packages/bb/33/2a6ae7fdc1b10cb581cef91addd8cdfc5f40d50abb5702309369d5834579/charset_normalizer-3.5.2-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:2a925889534b3748302dae5dead07cc13480de1dac3aea80a941b729b471ef93", size = 268757, upload-time = "2026-09-30T04:37:19.877Z" }, + { url = "https://files.pythonhosted.org/packages/a2/22/80992720a0282cd39bba1db35868e6b9c22f41281160143a836544bc1d8a/charset_normalizer-3.5.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:f5ec61164adcec446f8969a3358ec3f9b26bbda3b9213e5586d219afa8df2915", size = 262220, upload-time = "2026-09-30T04:37:21.583Z" }, + { url = "https://files.pythonhosted.org/packages/92/9f/181fd07e1bffea1d95cd80c84ac537354f50699c22cfc4d3c02b6fc16208/charset_normalizer-3.5.2-cp314-cp314t-win32.whl", hash = "sha256:598a11a2c7ebaa5334bf698bf29568c9c390abac6a154d8170fedecd1cea38c5", size = 197797, upload-time = "2026-09-30T04:37:23.235Z" }, + { url = "https://files.pythonhosted.org/packages/49/1c/25d8415ec1c4f2f41f1680435e4c87cfb378ff2f677d950946f2a45d0632/charset_normalizer-3.5.2-cp314-cp314t-win_amd64.whl", hash = "sha256:7fdde2c9fd9e3eca40631e024664cf2584272cc8f96308cbe5fdfc930f51d8bc", size = 222765, upload-time = "2026-09-30T04:37:24.891Z" }, + { url = "https://files.pythonhosted.org/packages/3e/b4/46b48f013dadfc0d0d33b375438e31bdf5a989dc68389c6bf627054d4df9/charset_normalizer-3.5.2-cp314-cp314t-win_arm64.whl", hash = "sha256:d1befeed746d247c81127bb14de9dc3d30edb6e5976d34f83f86ed262b1d9105", size = 210985, upload-time = "2026-09-30T04:37:26.634Z" }, + { url = "https://files.pythonhosted.org/packages/ca/e9/34e597dee616d0b8ee4b34d29399e85c2204ade174157a48505d42baa4ff/charset_normalizer-3.5.2-cp315-cp315-macosx_10_15_universal2.whl", hash = "sha256:87475fabc8d9996fd9c27debb395e642e8c838d78a00b6e932227a0e06b81e26", size = 367676, upload-time = "2026-09-30T04:37:28.329Z" }, + { url = "https://files.pythonhosted.org/packages/60/9f/a5d1c91c0263745e2cd344c5a4415d787c575501ab1d449f1148ac6b495d/charset_normalizer-3.5.2-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9409a8bf35cf78353942504b24a57de3d75b708997a1e4bd8db71ac8633ce364", size = 248414, upload-time = "2026-09-30T04:37:30.167Z" }, + { url = "https://files.pythonhosted.org/packages/26/79/e697f77464748a3ee3cf490c83d592459400d4898380d66c38366b03080c/charset_normalizer-3.5.2-cp315-cp315-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:498dc3188ca05a68231ac3fdbfc7f57eb67e1343c30e0fea17f8218c1599b253", size = 244775, upload-time = "2026-09-30T04:37:31.964Z" }, + { url = "https://files.pythonhosted.org/packages/ca/87/3d42a42e18ea066e2513936fd678a00696e77878b5ae04528976abdbcb83/charset_normalizer-3.5.2-cp315-cp315-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e242bb1c5e76e97dfa9e7f209a71e93a01d7f19ffdd5cfbb2e2d55b4f08f8ab0", size = 273877, upload-time = "2026-09-30T04:37:33.661Z" }, + { url = "https://files.pythonhosted.org/packages/c3/76/8a28136f3938ba9836f84280ce0c4d61ed1cf15a036b2034900c62634162/charset_normalizer-3.5.2-cp315-cp315-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:def79fa35ef0cef8d2accec024f4fdc7ead3012ff02f5215c783f39f03ef8cfc", size = 269455, upload-time = "2026-09-30T04:37:35.573Z" }, + { url = "https://files.pythonhosted.org/packages/a0/a1/4fbf5d0f0f1b2a080474c1cf9a2f12c4c6531bb0e8ba591055e846d2b4e9/charset_normalizer-3.5.2-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3df041de8887954562c9b261cba85ca0e9ded74048daf125f45edcfaa4832229", size = 257336, upload-time = "2026-09-30T04:37:37.397Z" }, + { url = "https://files.pythonhosted.org/packages/ba/a2/8b50aa320adb880ad579518e6f718f24944804b42a88b83d267d5d444125/charset_normalizer-3.5.2-cp315-cp315-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:04851f73ae72b8413dddadb16a49dfee95263553741fd42d546f7d66907e6be5", size = 257584, upload-time = "2026-09-30T04:37:39.522Z" }, + { url = "https://files.pythonhosted.org/packages/a5/57/50e3fed84e175f40349bd0da7a4fce94c87f0378f52d74f511d89e0bdc20/charset_normalizer-3.5.2-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:183b88127acdb4fabe59d951ab424faf1af7b63cdbb5f776186c1ea2ffcaed98", size = 251854, upload-time = "2026-09-30T04:37:41.23Z" }, + { url = "https://files.pythonhosted.org/packages/d6/54/f7fbb3493c9f49091213b9c2d6dd65800696f1ce1a3f196a4205f50417b1/charset_normalizer-3.5.2-cp315-cp315-musllinux_1_2_armv7l.whl", hash = "sha256:16fa0eccf81304b79c5cd87f9271c3b85dd9dd99245e4422ae9c0dd45e0f99d3", size = 248767, upload-time = "2026-09-30T04:37:42.883Z" }, + { url = "https://files.pythonhosted.org/packages/d9/37/b3a6385acc5a1e45b39ae9c90bfb9cf838a09b9dd37ef2740ab4c6b4a2eb/charset_normalizer-3.5.2-cp315-cp315-musllinux_1_2_ppc64le.whl", hash = "sha256:7441d755b7ab94f8d4eb3e43ec05482d760842fd263d003a99102d742cd835e2", size = 274479, upload-time = "2026-09-30T04:37:44.658Z" }, + { url = "https://files.pythonhosted.org/packages/89/44/809913e2cfd279e635a9294fdbbfb1b1dc62a8189d473d561f649fce98d8/charset_normalizer-3.5.2-cp315-cp315-musllinux_1_2_riscv64.whl", hash = "sha256:ca403d7e4798f525fdfc78e258820419cbbd0f0ecbab9de7840e3c017cf6b8cf", size = 258310, upload-time = "2026-09-30T04:37:46.529Z" }, + { url = "https://files.pythonhosted.org/packages/af/a2/f28400ab13359d91bd39179df8e149376b9bf36588e739a3a4f9de2b84b2/charset_normalizer-3.5.2-cp315-cp315-musllinux_1_2_s390x.whl", hash = "sha256:df29a0a7107f7011e77f4eebdddec4c7331e24d787a0b21a46d63bdf7445da95", size = 271100, upload-time = "2026-09-30T04:37:48.399Z" }, + { url = "https://files.pythonhosted.org/packages/e9/89/9bab37955edf0adb3b66f8a3a6617d9f2f487e0d56f295a6a286cb640aa6/charset_normalizer-3.5.2-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:f3c96f633825733f735c5a9cf21d21a257d8e1edf0b1cee0a064b9c424ca0f7d", size = 259936, upload-time = "2026-09-30T04:37:50.023Z" }, + { url = "https://files.pythonhosted.org/packages/23/b5/4459e08d45a679f903d50fea08bc52cfa728cca4d7bd02c757b5e5abda2e/charset_normalizer-3.5.2-cp315-cp315-win32.whl", hash = "sha256:281cb91036248400f4cc957495cccd44c275c2e0c5854f7e45ac5cf7dc193847", size = 186301, upload-time = "2026-09-30T04:37:51.722Z" }, + { url = "https://files.pythonhosted.org/packages/98/e8/55d5fd3935b4bce6da4fe0df61898e8c82653e317e677bd58aceb9c60f13/charset_normalizer-3.5.2-cp315-cp315-win_amd64.whl", hash = "sha256:89b53f3cda69831909888e0494f4fa0bcd3537e3e138dabeb620bd6ad946bae8", size = 210858, upload-time = "2026-09-30T04:37:53.427Z" }, + { url = "https://files.pythonhosted.org/packages/a9/5b/974423c2fd8e524c7a7f64318c1e02240ef954912fa2b4d70344107b9c68/charset_normalizer-3.5.2-cp315-cp315-win_arm64.whl", hash = "sha256:6be488a102b8cf28d0391d8c4ba7748938ae28b78ad901f8585520fca33ead1a", size = 200309, upload-time = "2026-09-30T04:37:55.015Z" }, + { url = "https://files.pythonhosted.org/packages/ee/f9/00ee0195db1013d8f7c416fd770fbeb560bb46eb2e36b054d05cb56f6cfa/charset_normalizer-3.5.2-cp315-cp315t-macosx_10_15_universal2.whl", hash = "sha256:915563965d418f986e7e145accc592eae9e1a1be3566ff98a05d7a9ec42a76e1", size = 395343, upload-time = "2026-09-30T04:37:56.743Z" }, + { url = "https://files.pythonhosted.org/packages/04/3a/c00b50e94c964cf934c7899cd47c97952fc11dad71cc5884b3c61795b09b/charset_normalizer-3.5.2-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:65cd72beeeca9d3aaea1201e5923859f308f952f9c71de93f06063c79f0f7a3b", size = 250114, upload-time = "2026-09-30T04:37:58.607Z" }, + { url = "https://files.pythonhosted.org/packages/50/27/d102dc880bbcffd0479ab64dfc1fb96777a854355a55e2bda72a71efadcb/charset_normalizer-3.5.2-cp315-cp315t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b7fd005a73d9e657273b7a10dc71a9e03c8fb9ee6999798d6918ce095b81ac7f", size = 239997, upload-time = "2026-09-30T04:38:00.511Z" }, + { url = "https://files.pythonhosted.org/packages/a5/4a/bf7ef45794dd293fab5f98a9309817977fbb845b9998f171b8cc5d8437a3/charset_normalizer-3.5.2-cp315-cp315t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e54da4baf05720032d527874d40b65fa4d7e5c6c6a43d0c3adbeffcaf275a2b3", size = 272313, upload-time = "2026-09-30T04:38:02.509Z" }, + { url = "https://files.pythonhosted.org/packages/e8/ee/008a2837737991474c5754bb3191010007663860979701990982a502cbaf/charset_normalizer-3.5.2-cp315-cp315t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:124fbf1a8ff966d87ae05bb8bd45a71f966055ed8bba320d0c7cf450bc5f4d0e", size = 269986, upload-time = "2026-09-30T04:38:04.435Z" }, + { url = "https://files.pythonhosted.org/packages/93/ad/bd74a283940dc910c5b14f8e4f80a248082bc9c0fcbe1f54530cb6d9cc5e/charset_normalizer-3.5.2-cp315-cp315t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:28b4f0d66fb834ff90f28209ac7bce77868c45d8c93e26f906709d9b7c2e1af9", size = 260202, upload-time = "2026-09-30T04:38:06.549Z" }, + { url = "https://files.pythonhosted.org/packages/8a/7b/ed341c66f69f688723501fac752be3d63c7159ca0d0d4174fc611e5710bb/charset_normalizer-3.5.2-cp315-cp315t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:58ca3755ee7ff7f59b57789ec9833c9de9ea275405cdd240eda1f193112e398a", size = 259083, upload-time = "2026-09-30T04:38:08.311Z" }, + { url = "https://files.pythonhosted.org/packages/cc/9d/e41588b777965e5031a43128a1e96173ebb35ac75fc53ec3b517e7c21cd4/charset_normalizer-3.5.2-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:443eae2bf318abeaf6f15d785138f71fd6de770e99a92158b8b814265e079115", size = 253081, upload-time = "2026-09-30T04:38:10.402Z" }, + { url = "https://files.pythonhosted.org/packages/81/35/b761eb6d8c1eb218b9b42b9b4d5ac902afdc399fb6dac6f9a9aac7bda589/charset_normalizer-3.5.2-cp315-cp315t-musllinux_1_2_armv7l.whl", hash = "sha256:58f361dcbab699cf8f42db3f47c8e7fd1036f138c23a5d08de9fde5f425a730c", size = 248712, upload-time = "2026-09-30T04:38:12.317Z" }, + { url = "https://files.pythonhosted.org/packages/4d/2c/147169a041b747759f37405c0a97157e8e92de967968373101ff14915cba/charset_normalizer-3.5.2-cp315-cp315t-musllinux_1_2_ppc64le.whl", hash = "sha256:1b4cbc7c3491ccb4aa17fcd8165649d01cf39f76de1696da8631b5f71b85401d", size = 274176, upload-time = "2026-09-30T04:38:14.138Z" }, + { url = "https://files.pythonhosted.org/packages/f0/2d/0ff8db0d373ba8538db686db11cd7e8912031490b9e4f383b41912e8d594/charset_normalizer-3.5.2-cp315-cp315t-musllinux_1_2_riscv64.whl", hash = "sha256:ba0b1d2620edf869789c3879223f52bf2afc5d31b3cb47cc57b3a12c05e2aa9d", size = 259413, upload-time = "2026-09-30T04:38:15.841Z" }, + { url = "https://files.pythonhosted.org/packages/8a/8e/b4a085fb47c9d3a7e43576a4784fdd8fe23f907514a972de8086edaf7a48/charset_normalizer-3.5.2-cp315-cp315t-musllinux_1_2_s390x.whl", hash = "sha256:5e2b6b57e9733d39f0c9fd3185efa6b8e29652c4cd8fe94180272cf6ed9a78c4", size = 269434, upload-time = "2026-09-30T04:38:17.626Z" }, + { url = "https://files.pythonhosted.org/packages/83/1c/d8d8d7322a7c3eecdf3237a4a419cf41d2eaad8e006ce7dfdd9d4c8fa2eb/charset_normalizer-3.5.2-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:51cf45226a9b588d0d2b4880c62d686934b63ab0bd79ca23ab0e9762eb27441b", size = 262716, upload-time = "2026-09-30T04:38:19.214Z" }, + { url = "https://files.pythonhosted.org/packages/a0/16/0e4c6ba9b44e97a2da150e52d331e8f9c968b21b358fbffa6c856cebcd89/charset_normalizer-3.5.2-cp315-cp315t-win32.whl", hash = "sha256:5fb29fb8cd1a46c27a1bf9613ad5ec2599310d46b4025d9556404a6b6a292800", size = 197159, upload-time = "2026-09-30T04:38:21.037Z" }, + { url = "https://files.pythonhosted.org/packages/be/33/e90bc2b1374f7f36ef106f56620de5a783907e19ca857efe2277e31cac3e/charset_normalizer-3.5.2-cp315-cp315t-win_amd64.whl", hash = "sha256:a192e2c40070d92c3ccf777e3a5c4ff515573cd2bb7ed0c537fdadbbec5bbf21", size = 221322, upload-time = "2026-09-30T04:38:22.886Z" }, + { url = "https://files.pythonhosted.org/packages/66/89/dfa6dcb08c200b7830ab56439e8c1890f2971d51aafbb3937894a2e7fcfc/charset_normalizer-3.5.2-cp315-cp315t-win_arm64.whl", hash = "sha256:749e97e1b32313717a565abbe321bc2190bc8b35f1a67e4cdbc7c56c8d8ffe58", size = 209192, upload-time = "2026-09-30T04:38:24.648Z" }, + { url = "https://files.pythonhosted.org/packages/8c/ab/176fbfd5b64939c55d652366aa5b9ef1d767af207a3aa6ebeb0d226c484d/charset_normalizer-3.5.2-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:4275811936e2f06feff5e598fb42a1b7ae852da8e39605211892b56b81a34efd", size = 331815, upload-time = "2026-09-30T04:38:26.216Z" }, + { url = "https://files.pythonhosted.org/packages/7e/84/371eac6b30bdbcbf2d632a1a01809103459216fcaae61b8b8d922c1bfb8a/charset_normalizer-3.5.2-cp37-abi3-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:1c50fe28bbc2ced33386f298650d91218076c05420e6cbd790b913adc41659e7", size = 253276, upload-time = "2026-09-30T04:38:28.032Z" }, + { url = "https://files.pythonhosted.org/packages/43/6f/c4fbae58febff71709c51bc7e18fdfa55341dc382704740f9f0cbf03817b/charset_normalizer-3.5.2-cp37-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d19fbd981a488e22cd04883659ca6b08f50b5974f9fd7c95655ef6a043e5893f", size = 241239, upload-time = "2026-09-30T04:38:29.732Z" }, + { url = "https://files.pythonhosted.org/packages/61/71/458c3f42164a07d0c5210798e9e704b39e540a6793b05aba67f3a35243a9/charset_normalizer-3.5.2-cp37-abi3-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:0fed1d06615f022ee3b13caf5e8b180cfea32bb2c5aded8a9d44277afc040f93", size = 231121, upload-time = "2026-09-30T04:38:31.462Z" }, + { url = "https://files.pythonhosted.org/packages/09/54/ab9e89367076f6331bb6c65c4bf14a5361fa5191cb6561bf534f18504e1b/charset_normalizer-3.5.2-cp37-abi3-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:838dcc90063569a0448120554591a1d6c4a4ffe11babf048908793154ab86ade", size = 260350, upload-time = "2026-09-30T04:38:33.239Z" }, + { url = "https://files.pythonhosted.org/packages/7c/c1/061431ecc688d9d76602502cb57cc01e691e682c18f1beb45f9673b5bbd2/charset_normalizer-3.5.2-cp37-abi3-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2ce45c6627b22c47e390bc91a41c3d13032192e699fa0bea96e9671b373d69b0", size = 255430, upload-time = "2026-09-30T04:38:34.865Z" }, + { url = "https://files.pythonhosted.org/packages/8d/1f/20c8949f0676f7ab811abdeb7f4d7f1cbc6e61ff20bef08b44edeb092bc8/charset_normalizer-3.5.2-cp37-abi3-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0774bf9bf620249fee3e0b8b9fd3065de213be30f3aa94ce2494b3b638949e26", size = 250612, upload-time = "2026-09-30T04:38:36.649Z" }, + { url = "https://files.pythonhosted.org/packages/2b/9e/46f2fa4c431fc98c4ae76a8cb5bdca54e0341e3cfc3fcfd8e82740250818/charset_normalizer-3.5.2-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:1db38f4c5496827c1a501846d64d14c3b80c7e6714e406cd7dc36a9899fa1011", size = 242083, upload-time = "2026-09-30T04:38:38.26Z" }, + { url = "https://files.pythonhosted.org/packages/bd/39/559be29a0c0f086e0bba6922babd38916cc5e0b58ced4de13ee01ea05508/charset_normalizer-3.5.2-cp37-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:304d8e4d493af723536393eee0c689eb7813f4a474c8b479dee63f1fdd98f621", size = 232738, upload-time = "2026-09-30T04:38:39.81Z" }, + { url = "https://files.pythonhosted.org/packages/ff/6c/387b0e4f756a282831c1d9fc6aeb6c51ca4507ca202767c8de15ce9b12e2/charset_normalizer-3.5.2-cp37-abi3-musllinux_1_2_ppc64le.whl", hash = "sha256:9b7f416ff0978e2f2249330527f0ad6fa02f4932e6199692d3b52da2048c19e4", size = 260703, upload-time = "2026-09-30T04:38:41.346Z" }, + { url = "https://files.pythonhosted.org/packages/96/92/1fdf015f09ef449f50d3ac4b67c90887c9c318b727daa95cc4f866e6521d/charset_normalizer-3.5.2-cp37-abi3-musllinux_1_2_riscv64.whl", hash = "sha256:01077390b03f7988f11d700a2194e69b119741a86b1a638b1db88891e3eced8e", size = 247622, upload-time = "2026-09-30T04:38:42.937Z" }, + { url = "https://files.pythonhosted.org/packages/dc/3c/8e7b8a5671ad5d433669fb2a76f1a0164df2d9b1718b0206bc2a16d840cc/charset_normalizer-3.5.2-cp37-abi3-musllinux_1_2_s390x.whl", hash = "sha256:7e841fb9010836c992c9f12fcbd43a831de93a5f726fc1ccd8ca1d0268c5014c", size = 257500, upload-time = "2026-09-30T04:38:44.604Z" }, + { url = "https://files.pythonhosted.org/packages/b4/f0/45b579df5cabc1d5d53ea1cc35e8437d3ca768c0acccc7041517cb6fbb32/charset_normalizer-3.5.2-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:9cae88599c7219005d879f98e5ed53341e9a122af585e1091200358a3003d2a0", size = 255100, upload-time = "2026-09-30T04:38:46.289Z" }, + { url = "https://files.pythonhosted.org/packages/31/68/fdec18a343f5fb3f310588dd478b09ac4799e0b187dbade3a8cd776f03ef/charset_normalizer-3.5.2-cp37-abi3-win32.whl", hash = "sha256:01b0c0d2262a9e28e8484a278c7e1b5d650e3ac8cf2683d2967e25899f208bdf", size = 174499, upload-time = "2026-09-30T04:38:47.999Z" }, + { url = "https://files.pythonhosted.org/packages/9d/8a/b618149cc5207943a0242068d7a27897f56a62947b5a039085f2a22029f8/charset_normalizer-3.5.2-cp37-abi3-win_amd64.whl", hash = "sha256:9f56f72050826f63dcee7a7f55b0a77168cb3bfc553fd405e7f8f9ece75a4036", size = 200092, upload-time = "2026-09-30T04:38:49.707Z" }, + { url = "https://files.pythonhosted.org/packages/03/cf/4c66866fa9e2b1c78e3c911516d1de497a677b7ac60f1eceda74ce777ca3/charset_normalizer-3.5.2-cp37-abi3-win_arm64.whl", hash = "sha256:40ab6bffa02ae10a0581e6c198be7d2d8ca5c2a0c64e4ed3465d766df457573e", size = 294363, upload-time = "2026-09-30T04:38:51.312Z" }, + { url = "https://files.pythonhosted.org/packages/fc/ad/d07d7862a62ffa6d79d68074d14823243dd235a77c45262acbf6adeb28bf/charset_normalizer-3.5.2-py3-none-any.whl", hash = "sha256:b6b751274acb69d77b3323d6b7dbaa3c7fdfc1eb829b7eb61d262f32e1af9685", size = 68872, upload-time = "2026-09-30T04:39:21.828Z" }, +] + [[package]] name = "colorama" version = "0.4.6" @@ -404,6 +588,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f6/b6/a1faf3a27ae9405fb34b1713cc73b2d8a26b04d5c561578fa2e6ef3e5bb9/cryptography-50.0.2-cp39-abi3-win_amd64.whl", hash = "sha256:4e81d95e5bafc2d6e34e4bed780e53e4d5b9a2f928573428aa4d35fbec1eb0de", size = 3854095, upload-time = "2026-09-30T15:29:46.782Z" }, ] +[[package]] +name = "docutils" +version = "0.22.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ae/b6/03bb70946330e88ffec97aefd3ea75ba575cb2e762061e0e62a213befee8/docutils-0.22.4.tar.gz", hash = "sha256:4db53b1fde9abecbb74d91230d32ab626d94f6badfc575d6db9194a49df29968", size = 2291750, upload-time = "2025-12-18T19:00:26.443Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/02/10/5da547df7a391dcde17f59520a231527b8571e6f46fc8efb02ccb370ab12/docutils-0.22.4-py3-none-any.whl", hash = "sha256:d0013f540772d1420576855455d050a2180186c91c15779301ac2ccb3eeb68de", size = 633196, upload-time = "2025-12-18T19:00:18.077Z" }, +] + [[package]] name = "exscript" source = { editable = "." } @@ -429,6 +622,12 @@ dev = [ { name = "pytest" }, { name = "pytest-cov" }, ] +docs = [ + { name = "furo" }, + { name = "myst-parser" }, + { name = "sphinx", version = "9.1.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13'" }, + { name = "sphinx", version = "9.2.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.13'" }, +] test = [ { name = "hypothesis" }, { name = "opentelemetry-instrumentation" }, @@ -460,6 +659,11 @@ dev = [ { name = "pytest", specifier = ">=9.2" }, { name = "pytest-cov", specifier = ">=7.1" }, ] +docs = [ + { name = "furo", specifier = ">=2025.12" }, + { name = "myst-parser", specifier = ">=5" }, + { name = "sphinx", specifier = ">=9" }, +] test = [ { name = "hypothesis", specifier = ">=6.170.14" }, { name = "opentelemetry-instrumentation", specifier = ">=0.48b0" }, @@ -469,6 +673,23 @@ test = [ ] types = [{ name = "mypy", specifier = ">=2.4" }] +[[package]] +name = "furo" +version = "2025.12.19" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "accessible-pygments" }, + { name = "beautifulsoup4" }, + { name = "pygments" }, + { name = "sphinx", version = "9.1.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13'" }, + { name = "sphinx", version = "9.2.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.13'" }, + { name = "sphinx-basic-ng" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ec/20/5f5ad4da6a5a27c80f2ed2ee9aee3f9e36c66e56e21c00fde467b2f8f88f/furo-2025.12.19.tar.gz", hash = "sha256:188d1f942037d8b37cd3985b955839fea62baa1730087dc29d157677c857e2a7", size = 1661473, upload-time = "2025-12-19T17:34:40.889Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/b2/50e9b292b5cac13e9e81272c7171301abc753a60460d21505b606e15cf21/furo-2025.12.19-py3-none-any.whl", hash = "sha256:bb0ead5309f9500130665a26bee87693c41ce4dbdff864dbfb6b0dae4673d24f", size = 339262, upload-time = "2025-12-19T17:34:38.905Z" }, +] + [[package]] name = "hypothesis" version = "6.170.14" @@ -550,6 +771,24 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0f/17/b28b4348a95694562357dc0c8ecb521dd607051dddd7af158c754b895710/hypothesis-6.170.14-pp312-pypy312_pp80-win_amd64.whl", hash = "sha256:90d4e85f22c3154a59df851bb2c59c89c129fba3155125d98ebad80bf69aad26", size = 692119, upload-time = "2026-10-30T06:51:19.8Z" }, ] +[[package]] +name = "idna" +version = "3.20" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f5/08/8eea9d4b8302028f3abb2c0813953f7aec26d33b7a8960ed760e65ff29fa/idna-3.20.tar.gz", hash = "sha256:a7db850025b95ded1eae8a46181a1a6c56c92c96f0e2b005d9ff8dc0210cab44", size = 216463, upload-time = "2026-09-17T14:11:04.752Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/58/a2/bb081bab032533a855d44de1d56f8e8426114ff1ba5d1f07a438a0a654f8/idna-3.20-py3-none-any.whl", hash = "sha256:ab7ae7122974553370f0bdb919e1a960b2cd1bc1ef0276416d896db81c14582c", size = 69583, upload-time = "2026-09-17T14:11:03.168Z" }, +] + +[[package]] +name = "imagesize" +version = "2.0.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fb/5e/513ff06670c84e7b9887c1fdf61b2d42b4f574a831f2f1d2222023049d8a/imagesize-2.0.1.tar.gz", hash = "sha256:b2ba6a4dea487a7ebcd53248d3476aca449d30db12a2dde5e0c5ca9624fd77e5", size = 1883774, upload-time = "2026-08-24T12:35:19.13Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/01/f9/575c8d760eae1fc99651b7cc5efd96ad5379ca4d6b53750b0fb4fe983f34/imagesize-2.0.1-py3-none-any.whl", hash = "sha256:ea0c9a0384df69ed86a943a15cde37d0360b82491b3910dc2215e202e62b5b02", size = 14794, upload-time = "2026-08-24T12:35:12.548Z" }, +] + [[package]] name = "iniconfig" version = "2.3.1" @@ -568,6 +807,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/5a/de/bbc12563bbf979618d17625a4e753ff7a078523e28d870d3626daa97261a/invoke-3.0.3-py3-none-any.whl", hash = "sha256:f11327165e5cbb89b2ad1d88d3292b5113332c43b8553b494da435d6ec6f5053", size = 160958, upload-time = "2026-04-07T15:17:46.875Z" }, ] +[[package]] +name = "jinja2" +version = "3.1.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markupsafe" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/df/bf/f7da0350254c0ed7c72f3e33cef02e048281fec7ecec5f032d4aac52226b/jinja2-3.1.6.tar.gz", hash = "sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d", size = 245115, upload-time = "2025-03-05T20:05:02.478Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67", size = 134899, upload-time = "2025-03-05T20:05:00.369Z" }, +] + [[package]] name = "librt" version = "0.16.0" @@ -779,6 +1030,152 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f8/b7/44edd7de434181c582892e68d1ffe6775ca403ce14aea07cb5a218a936cf/lxml-6.1.3-cp315-cp315t-win_arm64.whl", hash = "sha256:5a721a98c649855963811b59b55755b30566e7f7fc40bdc9803d66dee9f811cf", size = 3822836, upload-time = "2026-09-02T14:51:42.471Z" }, ] +[[package]] +name = "markdown-it-py" +version = "4.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mdurl" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/06/ff/7841249c247aa650a76b9ee4bbaeae59370dc8bfd2f6c01f3630c35eb134/markdown_it_py-4.2.0.tar.gz", hash = "sha256:04a21681d6fbb623de53f6f364d352309d4094dd4194040a10fd51833e418d49", size = 82454, upload-time = "2026-05-07T12:08:28.36Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/81/4da04ced5a082363ecfa159c010d200ecbd959ae410c10c0264a38cac0f5/markdown_it_py-4.2.0-py3-none-any.whl", hash = "sha256:9f7ebbcd14fe59494226453aed97c1070d83f8d24b6fc3a3bcf9a38092641c4a", size = 91687, upload-time = "2026-05-07T12:08:27.182Z" }, +] + +[[package]] +name = "markupsafe" +version = "3.0.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/38/9b/e422a865e1d5d57d0e509b4e0bf1c1a70a7f6382c29a5aa428df994c8bc8/markupsafe-3.0.4.tar.gz", hash = "sha256:2e9ad7dd851bf45fab9f75cbff4cb493fee9979e8d8c7c9c3ee119022518edd6", size = 153777, upload-time = "2026-10-02T23:07:22.29Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/81/09/4c59d56b8461ae8eb0d8ba34bb25b7e618547044679d58a82ef9b2479fc1/markupsafe-3.0.4-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:61631e08084be9e21a8967ec3139c7616ed7c5e9368e05c86d1b39562c8a57b6", size = 11658, upload-time = "2026-10-02T23:04:51.876Z" }, + { url = "https://files.pythonhosted.org/packages/a2/f0/d6613774d86fbf6d145751d43c59875e47a6f9f17daee0aef173bd36d90e/markupsafe-3.0.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0930db9bdc62d22944e10b066448bb65dc9abe9112880c7cab8da54db4284d5f", size = 12050, upload-time = "2026-10-02T23:04:52.931Z" }, + { url = "https://files.pythonhosted.org/packages/0d/f2/8f18e0b806eb13c1f8d07d917a720831ead54253a6dec011fbc78098a6f8/markupsafe-3.0.4-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6a45c3d514f2436064db00d7fc8778d888f0236ebfed649b53d13a59e69ad51b", size = 24363, upload-time = "2026-10-02T23:04:53.895Z" }, + { url = "https://files.pythonhosted.org/packages/60/ce/fa07dbe8a5675558fa36dea033e19995bc783de2dec5f540ccb9030b06aa/markupsafe-3.0.4-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:1e1451fab512d1bcc3dc26988ec1edb0b82c2db909132872cd9356070a6b63df", size = 28525, upload-time = "2026-10-02T23:04:54.905Z" }, + { url = "https://files.pythonhosted.org/packages/85/40/be87c01f3868ec217f8a2015089d71c22c8c5a75324822e5ed1cdd87210d/markupsafe-3.0.4-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:bd3ce56ae2cbae3ba82b683bc425cd7e48d2ed8b10f3e818186b6f5646d9271c", size = 24733, upload-time = "2026-10-02T23:04:56.229Z" }, + { url = "https://files.pythonhosted.org/packages/4f/a7/aeedb5140afa41fc74c225e9184ab96723a6e873b6ee1c9fede7283456d8/markupsafe-3.0.4-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8e124f974786f831d6043728e38296969d3579db8896fe004682f5758e613581", size = 22985, upload-time = "2026-10-02T23:04:57.521Z" }, + { url = "https://files.pythonhosted.org/packages/c3/fc/e91352bb08c6a59da3ef0909d457bf95a5f5908fbf151b30a06d9dbcfbb4/markupsafe-3.0.4-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c02e8f18bdedba082cef725942ac823b9b60656db07f7e265cb31618dfd00d77", size = 22001, upload-time = "2026-10-02T23:04:58.597Z" }, + { url = "https://files.pythonhosted.org/packages/5d/f8/bffee5e7d2a3deb59748a797650a48af7e672025cf641a79344a771ad106/markupsafe-3.0.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:9f098115c247e11d138ab83a28fa0323c77015007ea2df73ba5fd714dfefd67c", size = 23793, upload-time = "2026-10-02T23:04:59.686Z" }, + { url = "https://files.pythonhosted.org/packages/ed/59/b853d6628ecb4d658e1d637224846d5e9bb4adf4f8df97f3be9f29dce2ec/markupsafe-3.0.4-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:d5f93ebbeb8032d47e349328ec8662d973d9b05a70b3c35df1f91fe419b84749", size = 22640, upload-time = "2026-10-02T23:05:00.768Z" }, + { url = "https://files.pythonhosted.org/packages/09/b2/1506df394f0f075797c418d0301498f49e43be194e3ffcb49e6fe6ccf022/markupsafe-3.0.4-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:64511c54db4e4987aef4c41923235927428729e8174c5dba488429be70a998ed", size = 24074, upload-time = "2026-10-02T23:05:01.813Z" }, + { url = "https://files.pythonhosted.org/packages/c7/81/5ed69cda630ac69ef60d06c09ba5a7f84ff66a2e28cf986fd5614ab3c6e6/markupsafe-3.0.4-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:e1a622f13970d81f95d0c72f9dc090dce9085fccfa4c9f2174377ee32bd15786", size = 21563, upload-time = "2026-10-02T23:05:03.239Z" }, + { url = "https://files.pythonhosted.org/packages/0c/fe/fb1e79be0fea60aa32602ebefc9c35a82bb42b4df157285ab7dfec12341a/markupsafe-3.0.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c9a7f43c0b202b334cc9184af09bb8f21d3a209e038efaf106936fb69e6b026e", size = 23048, upload-time = "2026-10-02T23:05:04.479Z" }, + { url = "https://files.pythonhosted.org/packages/c8/52/7632a53360671a9b750cdbabaf9cdd89f18b42248b8e4cb42c0b0296e459/markupsafe-3.0.4-cp312-cp312-win32.whl", hash = "sha256:f0ec3b750b59375eab5b0fb2b9254810c00a3375be6d789899f1055a1d556237", size = 14108, upload-time = "2026-10-02T23:05:05.513Z" }, + { url = "https://files.pythonhosted.org/packages/3f/bf/62495e180b7000aaf30000fff849e933f74264638057176cf46852500adc/markupsafe-3.0.4-cp312-cp312-win_amd64.whl", hash = "sha256:11935df9bf455ed0c04eb87bcd720f02b1fe5e02128a9430f23aed6f93336fc7", size = 14303, upload-time = "2026-10-02T23:05:06.538Z" }, + { url = "https://files.pythonhosted.org/packages/c5/8e/4c24208776a65878d656996945aacfbfe010d3720d1a98fc0eb8491fc03b/markupsafe-3.0.4-cp312-cp312-win_arm64.whl", hash = "sha256:a4bbd2d87dd233b9fc5812160c3d0ffbe42edc22a26ce0469f58479ede633fe9", size = 14174, upload-time = "2026-10-02T23:05:07.617Z" }, + { url = "https://files.pythonhosted.org/packages/6d/18/4bc5ba32499e87bb2b0ef5b3a9bb9c00a131fa961ddf0be548cb550f548b/markupsafe-3.0.4-cp313-cp313-android_24_arm64_v8a.whl", hash = "sha256:de8b364c423ef0a4bad9069657d617f9a5d2b2062457a89b1fa16ee199c399c1", size = 12786, upload-time = "2026-10-02T23:05:08.709Z" }, + { url = "https://files.pythonhosted.org/packages/4e/6f/17f0c099bf25f3e31e63cc19244d9f6af861a9a4ab778c203997903cfdd0/markupsafe-3.0.4-cp313-cp313-android_24_x86_64.whl", hash = "sha256:34bdde374c5932765d7dc685c4a1d191a3207852d67e8e0a9eb6ea85156181f1", size = 12510, upload-time = "2026-10-02T23:05:09.93Z" }, + { url = "https://files.pythonhosted.org/packages/11/af/1a141081b905036ee904ec4bd945e1f70b4e1b32d33c4e59e8cf1d58b247/markupsafe-3.0.4-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:6bd9e1788e15bfcf6a9082de42e30387e7b85d211ab21e57a939bb8cfaaf8d96", size = 11583, upload-time = "2026-10-02T23:05:10.884Z" }, + { url = "https://files.pythonhosted.org/packages/e7/0a/a89385ae590232622a03e091805cff12f24fabe6c11e0e8bae096cece81c/markupsafe-3.0.4-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:5066b244f576f91afc8ee3ba029a89f99d39c79b1853fe9d39bea9f0afbec148", size = 11801, upload-time = "2026-10-02T23:05:11.913Z" }, + { url = "https://files.pythonhosted.org/packages/ed/85/ea548dc013962eb73653124bc595635fbf9e0fa41d1f181a967ccb784dfb/markupsafe-3.0.4-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:7a83aa6e4805df46fed18e989d3d16f86ef60cb50bbc8d9ce3a6be89165fbf6e", size = 11443, upload-time = "2026-10-02T23:05:12.887Z" }, + { url = "https://files.pythonhosted.org/packages/cc/72/15f2e5ec9cf2eb00d5cdfe968d94e4156a7bd7303832c3f3b2c403a36839/markupsafe-3.0.4-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:2d1b7d9308288661f56672b1b157d75fc536714d3638487bbea17b6318a78248", size = 11665, upload-time = "2026-10-02T23:05:13.829Z" }, + { url = "https://files.pythonhosted.org/packages/ca/e0/4030bea613677e333c8a2c901fd405055f657f9d06acba5b7357984b6ef7/markupsafe-3.0.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:73e77980c7207854f00fc4e71fb1626868d5740ab4012623d55c7a99ad122a72", size = 12061, upload-time = "2026-10-02T23:05:14.807Z" }, + { url = "https://files.pythonhosted.org/packages/f3/a5/28b76a7449eb702966b88bef599e2360b411fbb3afeee8fe560939be06ec/markupsafe-3.0.4-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7018d4af1cd272e847aa5917983ab5e83e4f6579f9dbfecd4a79c0ca80b144c2", size = 24395, upload-time = "2026-10-02T23:05:15.909Z" }, + { url = "https://files.pythonhosted.org/packages/07/6c/21232811afc3a063b5e934b1ae2efda52f46154ec382f585149c020e61fe/markupsafe-3.0.4-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c90d5b3d4e944e065a301d741b3c1d784f6bd1f503aa68b4967e32b2ba313d85", size = 27183, upload-time = "2026-10-02T23:05:16.976Z" }, + { url = "https://files.pythonhosted.org/packages/14/38/6ccdfa5b59049cb36fb80cbc80aee9cf1fc9bb77d1335ad435f2070b08cf/markupsafe-3.0.4-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:18a801868a884f216e784d7d14db2a4077143ce7610440aee2ce8f734e7cfcde", size = 24772, upload-time = "2026-10-02T23:05:18.209Z" }, + { url = "https://files.pythonhosted.org/packages/63/e0/cec6865dfe88cb48fedd4b20aed6af5158e41092adcbf3e028bcc6ec2108/markupsafe-3.0.4-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:434139499bb20b502ed3baa1f169e618f924a97e7a777fea1a49446d80106cf6", size = 23017, upload-time = "2026-10-02T23:05:19.286Z" }, + { url = "https://files.pythonhosted.org/packages/ee/76/6ed4940bb7648a9aac457c14f870cfdd5105f139a0fb1f29cd61fafa47d1/markupsafe-3.0.4-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9e227f3dbe6bde7491cf0a9965d00b88c6b1a4a95d11480ddf88bb96d397c19f", size = 22036, upload-time = "2026-10-02T23:05:20.352Z" }, + { url = "https://files.pythonhosted.org/packages/a1/4f/ed476226d4fe46a09090a36025bf319296810028df55eb12f1253b540f3a/markupsafe-3.0.4-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:b8cd1f918b26fd7b1832ece557cc18f2d8747309ff8b3f0ef9d4250c5ad67a39", size = 23786, upload-time = "2026-10-02T23:05:21.576Z" }, + { url = "https://files.pythonhosted.org/packages/9a/35/66ff30450e35ef5fba9ebc930c9411747e537fd9447b65e44f5007e2b84d/markupsafe-3.0.4-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:a5fcffb37e602b0b3c1638a97746b9b96125caa9bcf6fa41d337a9261de231ee", size = 22672, upload-time = "2026-10-02T23:05:22.922Z" }, + { url = "https://files.pythonhosted.org/packages/32/0b/72f45ce4b4efcbca4b80cf1b06703eff0be8d37e82abb78f66c85a7ead1e/markupsafe-3.0.4-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:5989cb26b2e1efc6a42216a9f6b5ee495ce5ace2e5b352a9af489976b32d1ee2", size = 24107, upload-time = "2026-10-02T23:05:24.175Z" }, + { url = "https://files.pythonhosted.org/packages/d2/03/71776e5fdcba04614b384cc102e8a4198208579d896fd1394cb7cb9aa900/markupsafe-3.0.4-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:add96447a86d205ab616665d53b2950ee81083757f56e6ea833c8b2917646b46", size = 21615, upload-time = "2026-10-02T23:05:25.215Z" }, + { url = "https://files.pythonhosted.org/packages/ab/5f/801ce02a02e7aee0f784b1ec7843026178f6adeb9c93ac67eb1992a9a84d/markupsafe-3.0.4-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:2628d3a8cb648ecebb3c5d6b0a1052d400e4d8b7ac0fb786be8d285b50040d17", size = 23076, upload-time = "2026-10-02T23:05:26.423Z" }, + { url = "https://files.pythonhosted.org/packages/4a/85/c43776625428f3bb4a61e8633940400e3efe6409e3c6f5bff26de5e45618/markupsafe-3.0.4-cp313-cp313-win32.whl", hash = "sha256:672d207103e6b16ca098611b0f9efad6bc00afd47c03d6ef62186495ca677dc0", size = 14118, upload-time = "2026-10-02T23:05:27.716Z" }, + { url = "https://files.pythonhosted.org/packages/6f/36/163da64de88a13db79214ef75fa041be7fa13bdb42261cf5b7484de14bfb/markupsafe-3.0.4-cp313-cp313-win_amd64.whl", hash = "sha256:1f1f9477e174582b0a1b583d60b66e1f2cf5d3fe12cee985e4aedf44766600e5", size = 14313, upload-time = "2026-10-02T23:05:28.749Z" }, + { url = "https://files.pythonhosted.org/packages/9f/a8/9b662783ffaa1149221432a923cee562f78b9cbbb8baa3df9b3753e63e1e/markupsafe-3.0.4-cp313-cp313-win_arm64.whl", hash = "sha256:06de8ef6331f6e822c28d577dc8bf43fe398800477c49498f38fc38b67ff33fc", size = 14189, upload-time = "2026-10-02T23:05:29.917Z" }, + { url = "https://files.pythonhosted.org/packages/5c/c3/a944f3b0df22bd129e96915b9f4e98d2eeca6516687d7618304a966c3c74/markupsafe-3.0.4-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:4ed644d75aa94a2baf7ec3a96eaa160ea58c742eb9d27c6506053c5c40fc84ed", size = 12787, upload-time = "2026-10-02T23:05:30.971Z" }, + { url = "https://files.pythonhosted.org/packages/d4/d6/a44863f69d88b6c7e27889108f70d47aed259edf89d5df3c5fca1eac87d6/markupsafe-3.0.4-cp314-cp314-android_24_x86_64.whl", hash = "sha256:6d2a9efe686f9de00d0d1ea32a4a5a86d558a2277501bd78d964214eab625e59", size = 12525, upload-time = "2026-10-02T23:05:32.263Z" }, + { url = "https://files.pythonhosted.org/packages/17/8f/168ba80e532dd6a93f96f8f706f1ad41d7990b6e1aeedc1cc0d211a33497/markupsafe-3.0.4-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:8781a792a070cf2bd1b86d3aa943894115faaba6e88122a7bf32d62072742453", size = 11589, upload-time = "2026-10-02T23:05:33.251Z" }, + { url = "https://files.pythonhosted.org/packages/32/b3/aa2c95a574d3af39403a469b295886eb9b6d448da568cbebb5a2cbfdc2e5/markupsafe-3.0.4-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:971a3bbb75d97ae4e2e8f7d4834236f86f85f0c85e04ab2e191db1123b04f80b", size = 11801, upload-time = "2026-10-02T23:05:34.315Z" }, + { url = "https://files.pythonhosted.org/packages/60/d0/34b810107d83840e768bf485de795893ebbae35b26ab061b487adfa0a692/markupsafe-3.0.4-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:8909c2f1c6dd65e054ac4b573a91c8384d1492281e55d82d159d653f7a13adf6", size = 11447, upload-time = "2026-10-02T23:05:35.302Z" }, + { url = "https://files.pythonhosted.org/packages/6c/ab/2f8488f0f817a39fca068d2b17daf446bf5cdb3eae28c3720af534d873b4/markupsafe-3.0.4-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:4cf3468d5ec187ffffcaca8e61929a37448f215dafc1386a12c750a72fe53634", size = 11722, upload-time = "2026-10-02T23:05:36.363Z" }, + { url = "https://files.pythonhosted.org/packages/ad/40/e2d117b048d47282ade906fbfd92814cbee5647afc13fda88a3406039372/markupsafe-3.0.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:52704c5d36eb6dda8866493decd61111fff86244c9b1ad225ca01b9e91e5970f", size = 12061, upload-time = "2026-10-02T23:05:37.397Z" }, + { url = "https://files.pythonhosted.org/packages/9a/a8/73a81135e85ba66217f5af7facb03bbb386807e1a729ab64532e4c802652/markupsafe-3.0.4-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1caa2fa5a6184fb233153b35f654e6687bd555476f6170f29d8ee9be1a8b0af9", size = 24434, upload-time = "2026-10-02T23:05:38.407Z" }, + { url = "https://files.pythonhosted.org/packages/ac/ca/fa9216dd01efee2dfdacafe7df32b4d0170fbac694b0c258a193d6e53999/markupsafe-3.0.4-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:387d8cd30e69b3f0a72877b9ae717033396404e19095b17fe89753a981fda44f", size = 27183, upload-time = "2026-10-02T23:05:39.581Z" }, + { url = "https://files.pythonhosted.org/packages/fa/4e/a469509e538d37af51103b17b073126973f2b1cbf197ff32c7ddf025cfe5/markupsafe-3.0.4-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:051417f74bcaaefa316276e0ff723f541616ca51043d070da00249d9bddd3e3c", size = 24813, upload-time = "2026-10-02T23:05:40.671Z" }, + { url = "https://files.pythonhosted.org/packages/8f/db/d7282caf7ab03af44d5d6fdbaa019b35c7d7f1c90588b839c07cba640d6a/markupsafe-3.0.4-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a8e9f292fcda89b324f2f5c91d13f1424a153e40fc2756f38ee23b15835ff300", size = 23049, upload-time = "2026-10-02T23:05:41.864Z" }, + { url = "https://files.pythonhosted.org/packages/30/f3/b6a425206e6964efda6acee544d0eb01d1501784d0b8e2dcc74986f33b17/markupsafe-3.0.4-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:df1ae86ff54725a01fa1a0510b914ca53a161b7050be74f6204e24aded5971d0", size = 22088, upload-time = "2026-10-02T23:05:43.014Z" }, + { url = "https://files.pythonhosted.org/packages/ea/8a/84d3582fc1f0d5bd466cdf2eebf175e172158a6e70701aacec1de1b35430/markupsafe-3.0.4-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:8965520ac587c94a4ac48b729be3d8b8de00af39699b17585dfb599babe77977", size = 23846, upload-time = "2026-10-02T23:05:44.098Z" }, + { url = "https://files.pythonhosted.org/packages/1c/65/db101cce51b7ba4864ac491a9859d297dd1adf0e55b103fee9db9c47c527/markupsafe-3.0.4-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:340cbb1957ba99929cbf19a75626d36ba1ae21d1730b287d1cf7f824a20c4fc7", size = 22674, upload-time = "2026-10-02T23:05:45.23Z" }, + { url = "https://files.pythonhosted.org/packages/e0/49/ddee9813d71db0c7a5c9d97c832125e6758a0c844777f1cf076569bb0e22/markupsafe-3.0.4-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:3a93d9616ddecfb393727a0041a562cf0b15a244e20f2bd25efc7949be4c4f17", size = 24104, upload-time = "2026-10-02T23:05:46.398Z" }, + { url = "https://files.pythonhosted.org/packages/aa/0e/7d8518d726726870a2399d69fd30d0fa36c5e57a2132c336b58d7c491073/markupsafe-3.0.4-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:d2e56fd3b00222722abfb3f5f0759ddbae4b90811b5ad4343c64030ad1bde70c", size = 21637, upload-time = "2026-10-02T23:05:47.48Z" }, + { url = "https://files.pythonhosted.org/packages/b4/b0/b505e8a361ba557dbf3b3aa7331ea39b00d2022a26e925ff8463b9714bb3/markupsafe-3.0.4-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:0d9c47709875fdb321452056622e930c52afbc07a7d780762fbb8b4d91ce6fa4", size = 23081, upload-time = "2026-10-02T23:05:48.611Z" }, + { url = "https://files.pythonhosted.org/packages/1c/ea/9cc3cea873f980c75cbdb6f4277ce30ee955de38be0b3d02f14c108e0698/markupsafe-3.0.4-cp314-cp314-win32.whl", hash = "sha256:38fc55594dab834470b6733dead2ee9e3f657fb0608c769dcafa0ba5ab52f45c", size = 14272, upload-time = "2026-10-02T23:05:49.707Z" }, + { url = "https://files.pythonhosted.org/packages/80/f0/5792ff768a410f93ee3f84fc19345295ffc352d2c936b424cb37e514714c/markupsafe-3.0.4-cp314-cp314-win_amd64.whl", hash = "sha256:c1bc67752d5f21013cfe430df4062441714eab79f65a6a05e01505957e9c35fe", size = 14501, upload-time = "2026-10-02T23:05:50.788Z" }, + { url = "https://files.pythonhosted.org/packages/5f/cf/3d074a8edffcc6899355232ff2543ae8d929733239596423b7db79698bc9/markupsafe-3.0.4-cp314-cp314-win_arm64.whl", hash = "sha256:7e1636da3d8dfc220b6dd10264db5f2b165e4888c4518594898fbe381049af8a", size = 14364, upload-time = "2026-10-02T23:05:51.857Z" }, + { url = "https://files.pythonhosted.org/packages/d9/31/87ce42159aae2163cf3bbbd0c44bc87780510eecab1ea3859099aed95dcb/markupsafe-3.0.4-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:805c8b84534fa10891890f0e4be39f3a99e94615d93e8836bf9fa1fdca2feeb2", size = 11759, upload-time = "2026-10-02T23:05:52.951Z" }, + { url = "https://files.pythonhosted.org/packages/5f/53/b047207eeb7752e960aca3eb1df5fb7eefa7dd4c62ac49bb156456c8a702/markupsafe-3.0.4-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:fa95848c929b6a75f6848d3c9793e59db365ee436776e57db835cdbfa79ba977", size = 12096, upload-time = "2026-10-02T23:05:54.066Z" }, + { url = "https://files.pythonhosted.org/packages/ee/51/4326c88a13c7b755657d44b4bb986f8c3d9843ecba7e22d98661d87f9a57/markupsafe-3.0.4-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e916035e3e9930cbdfdd10abf48861340221857f45509565898e012263f7b289", size = 25699, upload-time = "2026-10-02T23:05:55.15Z" }, + { url = "https://files.pythonhosted.org/packages/f2/bb/990581b7474bfcf2cf34bed6ba5ea23bd87adb9d671213d68e88620e7a6b/markupsafe-3.0.4-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b4d12837e0203bbace818ff4a7461afdcd78bcd782351cea148139180d7bcffe", size = 27804, upload-time = "2026-10-02T23:05:56.29Z" }, + { url = "https://files.pythonhosted.org/packages/6b/89/89491878c28e8291f5aa2fffe2c2d57230d10ae366d55dd810b840513d78/markupsafe-3.0.4-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:5086f9975abb1ab531ee6afca1761e4b59a19b446f3f6522ed776963228cfe5a", size = 25912, upload-time = "2026-10-02T23:05:57.416Z" }, + { url = "https://files.pythonhosted.org/packages/30/77/680998b54efdea06fc114565cd739b6d059f826a0279219b218dfa750d29/markupsafe-3.0.4-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b4a635a0487774f841cb1fb62e907e7195cc95bc761e053184b8acc3ceb20733", size = 23623, upload-time = "2026-10-02T23:05:58.557Z" }, + { url = "https://files.pythonhosted.org/packages/ae/75/2709f5ac5de9467b40b10e2bb8f89cc63dfb74582e09aa734b1124a217de/markupsafe-3.0.4-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cb96e6e088d6cf71c1ea977510948320234824cf226e32f6f6e044f7a9c82b34", size = 23301, upload-time = "2026-10-02T23:05:59.94Z" }, + { url = "https://files.pythonhosted.org/packages/a0/c8/39eadc6c5b14c9c7679bfb98f4d4c6a97863b5beb91839aca4d2d6e16e55/markupsafe-3.0.4-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8b5d563170ff8ba3181caa967c99a3c804d1dedb702c7cb93a6a7c32247da978", size = 24950, upload-time = "2026-10-02T23:06:01.289Z" }, + { url = "https://files.pythonhosted.org/packages/1a/5e/01037f8a43e8ccb0bffb4fbdc5212db05bf080fdd7286cd392332d58128a/markupsafe-3.0.4-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:396ec4e65cc889f69786b3b89478b471cee5a3bcf468b9d9bb03e1a30fb291fc", size = 23330, upload-time = "2026-10-02T23:06:02.441Z" }, + { url = "https://files.pythonhosted.org/packages/d4/f4/23e83ce0596bb0cbe670502d31df8f757bbd01a392aa486fa3b40d1ed399/markupsafe-3.0.4-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:15ba9e28640feef770374b116a6f019c21f52404aeabe516aa7f800587b98cfc", size = 25239, upload-time = "2026-10-02T23:06:03.579Z" }, + { url = "https://files.pythonhosted.org/packages/88/5b/3708897368073cc683d524750474f41a77d2986152c380dcc55b20fdf340/markupsafe-3.0.4-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:d920abdfa61279ba1a2ef9484aab07bf03331f8c08a10120fa332353d06e6932", size = 22767, upload-time = "2026-10-02T23:06:04.699Z" }, + { url = "https://files.pythonhosted.org/packages/c6/61/ebda1307864b409e6b3115757a3d4a09cca46cfb6cc65191b5de226b424b/markupsafe-3.0.4-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:a9f54054101545a9a9cccefddf54316aa6e4491611fcbef9e91b3b6bebec04f6", size = 23668, upload-time = "2026-10-02T23:06:05.9Z" }, + { url = "https://files.pythonhosted.org/packages/09/15/98075cceac3b5ba0dbb8e4762a847be967d2befc349a2cf2d0ac77f62c9d/markupsafe-3.0.4-cp314-cp314t-win32.whl", hash = "sha256:12a606a492de952afcb43b59a14aaaaad120e708d3663dd0fdf2d738d427a691", size = 14301, upload-time = "2026-10-02T23:06:07.109Z" }, + { url = "https://files.pythonhosted.org/packages/0b/a3/768b560fcc4156685cb563d922b217810cfa7bc135773367f62f1f9d2078/markupsafe-3.0.4-cp314-cp314t-win_amd64.whl", hash = "sha256:a18f38cafc329bac5e3c2b96c765b4c96d3d103421ed22ab7988c1e3fce27464", size = 14526, upload-time = "2026-10-02T23:06:08.276Z" }, + { url = "https://files.pythonhosted.org/packages/93/63/da554b4c97a6b0ea3229ca7fe8cbfb620be81613d517f482e85958550537/markupsafe-3.0.4-cp314-cp314t-win_arm64.whl", hash = "sha256:eba154571c16e032112afac0dc2dfe9e63c2ceb7aedd07bb7eecf2ce26d4dd4c", size = 14421, upload-time = "2026-10-02T23:06:09.402Z" }, + { url = "https://files.pythonhosted.org/packages/a9/30/54d11c8ca027114898cab97421fb39e4ffd9ddf47cdbc44df2ec76722da9/markupsafe-3.0.4-cp315-cp315-android_24_arm64_v8a.whl", hash = "sha256:737c9c3981998eba27f11786f84fddcbabc74068b72a4a1f454ea02094b57b65", size = 12791, upload-time = "2026-10-02T23:06:10.485Z" }, + { url = "https://files.pythonhosted.org/packages/10/6d/97c913e253a14bd3cd0e15a5c56d13203b823fa7ee32498342896a072dc4/markupsafe-3.0.4-cp315-cp315-android_24_x86_64.whl", hash = "sha256:489505b03f692c3f376394e49194fa7a7f9e8558d6e293a7056a0032b0c38163", size = 12527, upload-time = "2026-10-02T23:06:11.834Z" }, + { url = "https://files.pythonhosted.org/packages/26/f9/b86d032042a4d597d9e1997f0e5f63a3eedaf11258e0a05760b0a0a826ea/markupsafe-3.0.4-cp315-cp315-ios_13_0_arm64_iphoneos.whl", hash = "sha256:077293e425f28ec737dbcad442a71752e28f8ae27cde3d68acd1fb212091cd92", size = 11602, upload-time = "2026-10-02T23:06:13.122Z" }, + { url = "https://files.pythonhosted.org/packages/f2/dc/73c14c1eedf0ac5fa3292ba43435e6c49d2c2050f33cebde541f8f4807f1/markupsafe-3.0.4-cp315-cp315-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:9348cbb300d224fe3b89793262cb093504d4ae927004468463f745188a193e4a", size = 11817, upload-time = "2026-10-02T23:06:14.227Z" }, + { url = "https://files.pythonhosted.org/packages/8f/69/2c2fcaa5fcee22d72c7819c0d536fd181c74a688e6143845419579cd2863/markupsafe-3.0.4-cp315-cp315-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:b807e598953730f82e4eae3bd30f6a122cf6b31c398c6b504c0e04c13c170429", size = 11446, upload-time = "2026-10-02T23:06:15.574Z" }, + { url = "https://files.pythonhosted.org/packages/88/54/9e5ec76c62e6e2834d5a93623018c943e8b3bb41d663e3fd4c03303b9b85/markupsafe-3.0.4-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:799c39bdf5e2f1292fedd3009f7b3c9e760f10b2420cb9638d56920840ff6db8", size = 11719, upload-time = "2026-10-02T23:06:16.701Z" }, + { url = "https://files.pythonhosted.org/packages/96/24/3ec292b44064c16229e064d770b2625bd8ea941aa61f44905a9fa44942c0/markupsafe-3.0.4-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:ae9dcb8fbe244cb82f8a6458b455b927a03685e383d9bacf1ea5ce180b96dc97", size = 12075, upload-time = "2026-10-02T23:06:17.855Z" }, + { url = "https://files.pythonhosted.org/packages/aa/85/b64fdb1f304848518742136983c24e96d967bfb59a0ea160e92736901ab0/markupsafe-3.0.4-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4bced6e2a6dba6a28f7dd3c6ce14df1b2dd495923f16ea484cad03decd463b2b", size = 24656, upload-time = "2026-10-02T23:06:18.963Z" }, + { url = "https://files.pythonhosted.org/packages/9c/18/23997d4c65b355da6390d61cd56e0ab3befd6ba8dda25cb40c602bd0fa6b/markupsafe-3.0.4-cp315-cp315-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3882fb412298575bae3b9c46868251f15cc69307359f87bb1b382e53d6e5a2c9", size = 27737, upload-time = "2026-10-02T23:06:20.117Z" }, + { url = "https://files.pythonhosted.org/packages/d4/36/35998dead3c6af88c38265a56e58100211f036234ab88eb2283fd4cbce44/markupsafe-3.0.4-cp315-cp315-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:04e7902ba80ee4bac1d50a549606527a1dcf0476cd81403db41099d3b60ec653", size = 25003, upload-time = "2026-10-02T23:06:21.284Z" }, + { url = "https://files.pythonhosted.org/packages/82/96/ef49135ce260db4ca4a12b119ed468449cd248db6b1468e2112b546d7a2e/markupsafe-3.0.4-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:925f929d6b59a8b3f8b8c6ac363cd0af7eecc81efb3071770b3c6717c450a369", size = 23111, upload-time = "2026-10-02T23:06:22.524Z" }, + { url = "https://files.pythonhosted.org/packages/50/7d/83126e338bd88c17a220668235368ad719fd4638e426739858cbb8508f77/markupsafe-3.0.4-cp315-cp315-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f68edfc67aabac33708941f26f22a7b8e9f81429bc0cf249fcf7d66b23af8d19", size = 22403, upload-time = "2026-10-02T23:06:23.785Z" }, + { url = "https://files.pythonhosted.org/packages/83/dd/daf7e420de23c8206c365204e7b85e1251d8e19d34196a56336f316e5ed2/markupsafe-3.0.4-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:e5c802729725bd07e2bc3ab7b76dc7e0bbfc53129d8f1eb1c002c24cf774717e", size = 24083, upload-time = "2026-10-02T23:06:25.037Z" }, + { url = "https://files.pythonhosted.org/packages/19/3c/11eecdc06bc44ad5570350085b572ebf049e8f9a38d1ece6d76640b739cd/markupsafe-3.0.4-cp315-cp315-musllinux_1_2_armv7l.whl", hash = "sha256:55ffd6ce583d97dc71dc92e930324c8c0d25aea7e3ade6ae54ef77cedb096811", size = 22931, upload-time = "2026-10-02T23:06:26.328Z" }, + { url = "https://files.pythonhosted.org/packages/0d/9e/ac0fd77f2a726e56ecc3ca0235d095feace1358d1b822406c2a2ef26a4dc/markupsafe-3.0.4-cp315-cp315-musllinux_1_2_ppc64le.whl", hash = "sha256:2cb3dd71fc6be918ad4264346a8ed69485f9b7ed7bf35495d8e22807cd6b8bea", size = 24299, upload-time = "2026-10-02T23:06:27.742Z" }, + { url = "https://files.pythonhosted.org/packages/d7/09/c6bd842ad58ff5b3bc76eeed7e9a42a6f11adc5d090ec697b72c9672731e/markupsafe-3.0.4-cp315-cp315-musllinux_1_2_riscv64.whl", hash = "sha256:94f5407f7bc64fa6463906b896f9904beeeb7dd8dc116ee8e9056c8714ff9916", size = 21948, upload-time = "2026-10-02T23:06:29.274Z" }, + { url = "https://files.pythonhosted.org/packages/a3/46/82f586711fed61e86faa1ee1bc317d68cd45a10c8bdbe3f7d1fdf9026ad8/markupsafe-3.0.4-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:2dad610540cb2e6272855c178f08ae9a1c7ac258a7fb71660553a5f104b42741", size = 23150, upload-time = "2026-10-02T23:06:30.583Z" }, + { url = "https://files.pythonhosted.org/packages/19/2d/2dfdce99318abbfa26925195fbc17db188c46a1ec6457be121b6f9cfeb42/markupsafe-3.0.4-cp315-cp315-win32.whl", hash = "sha256:03470d1a8268e692ecf79ecd565593e59d44219377a7ead61f1f1b94c1f7ff6b", size = 14269, upload-time = "2026-10-02T23:06:31.949Z" }, + { url = "https://files.pythonhosted.org/packages/5b/ec/6000fd82e8791e58fcd0456ec20f098957e2b03d5ed02eb73241a577c0ba/markupsafe-3.0.4-cp315-cp315-win_amd64.whl", hash = "sha256:d882a373d8093c2941e01291b7ced96e9cbe4781da9a7751ca7e6c70385e5214", size = 14496, upload-time = "2026-10-02T23:06:33.258Z" }, + { url = "https://files.pythonhosted.org/packages/bc/66/e73bd5016421d5d6e2fb6de7dd609f9de020942ac8c626526bd8c6eeaf82/markupsafe-3.0.4-cp315-cp315-win_arm64.whl", hash = "sha256:353bd63081912ab8cfa6a0c7d185934cdf8426f04c618bba6bc4b394f2069b67", size = 14361, upload-time = "2026-10-02T23:06:34.539Z" }, + { url = "https://files.pythonhosted.org/packages/90/df/cb8c3dc98d313a951df2f8968f44e4cb5643df6d3cab749a530ce2f7d972/markupsafe-3.0.4-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:c61750fadcd119d0825bcb7d7d675dd264dcc89cc05292aab5be68ebdbb374ad", size = 11760, upload-time = "2026-10-02T23:06:35.807Z" }, + { url = "https://files.pythonhosted.org/packages/d6/bb/4af9b3ca0753d654ac75f9531d5bd741bb77ca6e696f36807c475ffc099a/markupsafe-3.0.4-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:1c0df495a977d10460a94941799c72d5b5ab03d3858d949b55b5a66c8f371c99", size = 12108, upload-time = "2026-10-02T23:06:37.089Z" }, + { url = "https://files.pythonhosted.org/packages/3f/d4/b56429313aee5fd59b079c3df5615299959e25e7113eb6d8caadbdd7d38a/markupsafe-3.0.4-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:02fa4acbc6a3fc5c693c34d4dd8c1130b7fe99cc915181b0ddd6f72aeb296002", size = 25762, upload-time = "2026-10-02T23:06:38.419Z" }, + { url = "https://files.pythonhosted.org/packages/65/f5/34c181e891aa4f7d59c918584672e0c5eb7fffe76c1387d1246008bf4081/markupsafe-3.0.4-cp315-cp315t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:05295589e619b9bed252a86b532b8e27350abc372d18ba89b59375325e91ec1e", size = 28400, upload-time = "2026-10-02T23:06:39.819Z" }, + { url = "https://files.pythonhosted.org/packages/ce/b5/ad14694fd0ac9a5ce30bc6498f2999378f418583dd1679cca5a1b512957e/markupsafe-3.0.4-cp315-cp315t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:be6cb0c799abb0e2ba3e618e6d28ddddf7e485f6c2ce938dfa237daf3905072c", size = 25946, upload-time = "2026-10-02T23:06:41.381Z" }, + { url = "https://files.pythonhosted.org/packages/d6/a8/26b606445387d0ceb1eb1f21840094b84e4e3c3c3983d80d10b89823b490/markupsafe-3.0.4-cp315-cp315t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:26e9867520db70d37f7fb421a7f0d8adb40171011fb84ce869afa1a83370dfa8", size = 23634, upload-time = "2026-10-02T23:06:42.748Z" }, + { url = "https://files.pythonhosted.org/packages/39/a2/b8814de672f1f0094d498bf646f2fec9d6356b503d28ef500b71c5095377/markupsafe-3.0.4-cp315-cp315t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f03460ff076f70ab595bb45a0205ccea1971443575b6920c52e755dec2b3fbfe", size = 23488, upload-time = "2026-10-02T23:06:44.176Z" }, + { url = "https://files.pythonhosted.org/packages/db/c7/287223376fb73335a3cc5d6eb22c6ab01358cf33945a9c39c06b9dac3f4b/markupsafe-3.0.4-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:436e3ffc6310d3c41878c601db29098102fe5d8a467c49da4a4125254e0980f2", size = 25037, upload-time = "2026-10-02T23:06:45.646Z" }, + { url = "https://files.pythonhosted.org/packages/f9/29/4df8355e313426d19e62ba33e0253c009ca12a0894ee77d67fa67255361c/markupsafe-3.0.4-cp315-cp315t-musllinux_1_2_armv7l.whl", hash = "sha256:4e2c4809c14559aa7ef426f27fb35afbb38104c349a903bf8f3600456764bb38", size = 23543, upload-time = "2026-10-02T23:06:47.264Z" }, + { url = "https://files.pythonhosted.org/packages/71/e5/8377731e8495668dcc768f645e717df18318c841edaf023a99395f6da9b4/markupsafe-3.0.4-cp315-cp315t-musllinux_1_2_ppc64le.whl", hash = "sha256:da2af0d7aebfc2074080d72efa6ab8317c62481ef1f896f65d9999c1c01f4494", size = 25282, upload-time = "2026-10-02T23:06:48.795Z" }, + { url = "https://files.pythonhosted.org/packages/ed/5f/373456e37ceb1478d657d6fe769cbe0a39f0a8dfc1548eeb19c471eefdd9/markupsafe-3.0.4-cp315-cp315t-musllinux_1_2_riscv64.whl", hash = "sha256:aa2c838cc024642cc04c6854232f32b43e5e22833dd11119c1766c7873b8370d", size = 22993, upload-time = "2026-10-02T23:06:50.31Z" }, + { url = "https://files.pythonhosted.org/packages/d7/93/2cbd5628435afb6f541bbaced4bce0c2edac4b09a142e6e928b8b0da9858/markupsafe-3.0.4-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:b91cc9d336957239ff200f30097e6fea2dc6d6fb3c81e853eaa09eac904fd894", size = 23712, upload-time = "2026-10-02T23:06:51.759Z" }, + { url = "https://files.pythonhosted.org/packages/81/99/157e10966b033b363aeda5263e82596ee232a0b1d082fdbf90aa417ff083/markupsafe-3.0.4-cp315-cp315t-win32.whl", hash = "sha256:e49fb0d1ce92cfa0cb198cc5b1b11cdf9d0638658e2a2db2687e39db7c87fc78", size = 14300, upload-time = "2026-10-02T23:06:53.241Z" }, + { url = "https://files.pythonhosted.org/packages/33/05/55884815414c9706a23deca150b72c25a62109e65b0b6ce232077802c719/markupsafe-3.0.4-cp315-cp315t-win_amd64.whl", hash = "sha256:4f6e0852a0283b1b1fd776eeb7b766a5f440b3e2bd31ab51af3b400585f3965c", size = 14522, upload-time = "2026-10-02T23:06:54.729Z" }, + { url = "https://files.pythonhosted.org/packages/92/f9/ecbde7149e95b8a0f18e16d5d747f7dc06049d5da2e4f77f6f5e4a1f46a8/markupsafe-3.0.4-cp315-cp315t-win_arm64.whl", hash = "sha256:39dbacefc411633db5b4378b066a9aca70a3d7e2922c9e578d825f844026eeba", size = 14417, upload-time = "2026-10-02T23:06:56.246Z" }, +] + +[[package]] +name = "mdit-py-plugins" +version = "0.6.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markdown-it-py" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/59/fc/f8d0863f8862f25602c0404d75568e89fb6b4109804645e5cdfb1be5cf56/mdit_py_plugins-0.6.1.tar.gz", hash = "sha256:a2bca0f039f39dbd35fb74ae1b5f998608c437463371f0ff7f49a19a17a114d0", size = 56114, upload-time = "2026-05-13T09:03:38.91Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a5/69/6da5581c6a7fede7dc261bf4e67d6adca4196f176b43288b55b3db395b6e/mdit_py_plugins-0.6.1-py3-none-any.whl", hash = "sha256:214c82fb2ac524472ab6a5bcab1de80f73b50443e187f401bfd77efbc7c6481d", size = 66663, upload-time = "2026-05-13T09:03:37.76Z" }, +] + +[[package]] +name = "mdurl" +version = "0.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d6/54/cfe61301667036ec958cb99bd3efefba235e65cdeb9c84d24a8293ba1d90/mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba", size = 8729, upload-time = "2022-08-14T12:40:10.846Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" }, +] + [[package]] name = "mypy" version = "2.4.1" @@ -842,6 +1239,24 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505", size = 4963, upload-time = "2025-04-22T14:54:22.983Z" }, ] +[[package]] +name = "myst-parser" +version = "5.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "docutils" }, + { name = "jinja2" }, + { name = "markdown-it-py" }, + { name = "mdit-py-plugins" }, + { name = "pyyaml" }, + { name = "sphinx", version = "9.1.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13'" }, + { name = "sphinx", version = "9.2.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/21/dc/603751677fff302f34396e206b610f556a59d7fe58b9a2145f54e96b48e8/myst_parser-5.1.0.tar.gz", hash = "sha256:ab69322dc6719dcc7f296479dbb70181b66df6ed315064f92dbc85c0e1bf2f02", size = 101182, upload-time = "2026-05-13T09:38:19.361Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/09/dc/f3dfb7488b770f3f67e6545085bf2abea5172e88f57b8ad25ef860ca704c/myst_parser-5.1.0-py3-none-any.whl", hash = "sha256:9c91c52b3cdb4d94a6506e4fab4e2f296c7623a0da0dcbe6de1565c3dad67a8a", size = 85817, upload-time = "2026-05-13T09:38:17.904Z" }, +] + [[package]] name = "ncclient" version = "0.7.1" @@ -1069,6 +1484,85 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9d/7a/d968e294073affff457b041c2be9868a40c1c71f4a35fcc1e45e5493067b/pytest_cov-7.1.0-py3-none-any.whl", hash = "sha256:a0461110b7865f9a271aa1b51e516c9a95de9d696734a2f71e3e78f46e1d4678", size = 22876, upload-time = "2026-03-21T20:11:14.438Z" }, ] +[[package]] +name = "pyyaml" +version = "6.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/33/422b98d2195232ca1826284a76852ad5a86fe23e31b009c9886b2d0fb8b2/pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196", size = 182063, upload-time = "2025-09-25T21:32:11.445Z" }, + { url = "https://files.pythonhosted.org/packages/89/a0/6cf41a19a1f2f3feab0e9c0b74134aa2ce6849093d5517a0c550fe37a648/pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0", size = 173973, upload-time = "2025-09-25T21:32:12.492Z" }, + { url = "https://files.pythonhosted.org/packages/ed/23/7a778b6bd0b9a8039df8b1b1d80e2e2ad78aa04171592c8a5c43a56a6af4/pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28", size = 775116, upload-time = "2025-09-25T21:32:13.652Z" }, + { url = "https://files.pythonhosted.org/packages/65/30/d7353c338e12baef4ecc1b09e877c1970bd3382789c159b4f89d6a70dc09/pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c", size = 844011, upload-time = "2025-09-25T21:32:15.21Z" }, + { url = "https://files.pythonhosted.org/packages/8b/9d/b3589d3877982d4f2329302ef98a8026e7f4443c765c46cfecc8858c6b4b/pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc", size = 807870, upload-time = "2025-09-25T21:32:16.431Z" }, + { url = "https://files.pythonhosted.org/packages/05/c0/b3be26a015601b822b97d9149ff8cb5ead58c66f981e04fedf4e762f4bd4/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e", size = 761089, upload-time = "2025-09-25T21:32:17.56Z" }, + { url = "https://files.pythonhosted.org/packages/be/8e/98435a21d1d4b46590d5459a22d88128103f8da4c2d4cb8f14f2a96504e1/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea", size = 790181, upload-time = "2025-09-25T21:32:18.834Z" }, + { url = "https://files.pythonhosted.org/packages/74/93/7baea19427dcfbe1e5a372d81473250b379f04b1bd3c4c5ff825e2327202/pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5", size = 137658, upload-time = "2025-09-25T21:32:20.209Z" }, + { url = "https://files.pythonhosted.org/packages/86/bf/899e81e4cce32febab4fb42bb97dcdf66bc135272882d1987881a4b519e9/pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b", size = 154003, upload-time = "2025-09-25T21:32:21.167Z" }, + { url = "https://files.pythonhosted.org/packages/1a/08/67bd04656199bbb51dbed1439b7f27601dfb576fb864099c7ef0c3e55531/pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd", size = 140344, upload-time = "2025-09-25T21:32:22.617Z" }, + { url = "https://files.pythonhosted.org/packages/d1/11/0fd08f8192109f7169db964b5707a2f1e8b745d4e239b784a5a1dd80d1db/pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8", size = 181669, upload-time = "2025-09-25T21:32:23.673Z" }, + { url = "https://files.pythonhosted.org/packages/b1/16/95309993f1d3748cd644e02e38b75d50cbc0d9561d21f390a76242ce073f/pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1", size = 173252, upload-time = "2025-09-25T21:32:25.149Z" }, + { url = "https://files.pythonhosted.org/packages/50/31/b20f376d3f810b9b2371e72ef5adb33879b25edb7a6d072cb7ca0c486398/pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c", size = 767081, upload-time = "2025-09-25T21:32:26.575Z" }, + { url = "https://files.pythonhosted.org/packages/49/1e/a55ca81e949270d5d4432fbbd19dfea5321eda7c41a849d443dc92fd1ff7/pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5", size = 841159, upload-time = "2025-09-25T21:32:27.727Z" }, + { url = "https://files.pythonhosted.org/packages/74/27/e5b8f34d02d9995b80abcef563ea1f8b56d20134d8f4e5e81733b1feceb2/pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6", size = 801626, upload-time = "2025-09-25T21:32:28.878Z" }, + { url = "https://files.pythonhosted.org/packages/f9/11/ba845c23988798f40e52ba45f34849aa8a1f2d4af4b798588010792ebad6/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6", size = 753613, upload-time = "2025-09-25T21:32:30.178Z" }, + { url = "https://files.pythonhosted.org/packages/3d/e0/7966e1a7bfc0a45bf0a7fb6b98ea03fc9b8d84fa7f2229e9659680b69ee3/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be", size = 794115, upload-time = "2025-09-25T21:32:31.353Z" }, + { url = "https://files.pythonhosted.org/packages/de/94/980b50a6531b3019e45ddeada0626d45fa85cbe22300844a7983285bed3b/pyyaml-6.0.3-cp313-cp313-win32.whl", hash = "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26", size = 137427, upload-time = "2025-09-25T21:32:32.58Z" }, + { url = "https://files.pythonhosted.org/packages/97/c9/39d5b874e8b28845e4ec2202b5da735d0199dbe5b8fb85f91398814a9a46/pyyaml-6.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c", size = 154090, upload-time = "2025-09-25T21:32:33.659Z" }, + { url = "https://files.pythonhosted.org/packages/73/e8/2bdf3ca2090f68bb3d75b44da7bbc71843b19c9f2b9cb9b0f4ab7a5a4329/pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb", size = 140246, upload-time = "2025-09-25T21:32:34.663Z" }, + { url = "https://files.pythonhosted.org/packages/9d/8c/f4bd7f6465179953d3ac9bc44ac1a8a3e6122cf8ada906b4f96c60172d43/pyyaml-6.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac", size = 181814, upload-time = "2025-09-25T21:32:35.712Z" }, + { url = "https://files.pythonhosted.org/packages/bd/9c/4d95bb87eb2063d20db7b60faa3840c1b18025517ae857371c4dd55a6b3a/pyyaml-6.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310", size = 173809, upload-time = "2025-09-25T21:32:36.789Z" }, + { url = "https://files.pythonhosted.org/packages/92/b5/47e807c2623074914e29dabd16cbbdd4bf5e9b2db9f8090fa64411fc5382/pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7", size = 766454, upload-time = "2025-09-25T21:32:37.966Z" }, + { url = "https://files.pythonhosted.org/packages/02/9e/e5e9b168be58564121efb3de6859c452fccde0ab093d8438905899a3a483/pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788", size = 836355, upload-time = "2025-09-25T21:32:39.178Z" }, + { url = "https://files.pythonhosted.org/packages/88/f9/16491d7ed2a919954993e48aa941b200f38040928474c9e85ea9e64222c3/pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5", size = 794175, upload-time = "2025-09-25T21:32:40.865Z" }, + { url = "https://files.pythonhosted.org/packages/dd/3f/5989debef34dc6397317802b527dbbafb2b4760878a53d4166579111411e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764", size = 755228, upload-time = "2025-09-25T21:32:42.084Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ce/af88a49043cd2e265be63d083fc75b27b6ed062f5f9fd6cdc223ad62f03e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35", size = 789194, upload-time = "2025-09-25T21:32:43.362Z" }, + { url = "https://files.pythonhosted.org/packages/23/20/bb6982b26a40bb43951265ba29d4c246ef0ff59c9fdcdf0ed04e0687de4d/pyyaml-6.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac", size = 156429, upload-time = "2025-09-25T21:32:57.844Z" }, + { url = "https://files.pythonhosted.org/packages/f4/f4/a4541072bb9422c8a883ab55255f918fa378ecf083f5b85e87fc2b4eda1b/pyyaml-6.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3", size = 143912, upload-time = "2025-09-25T21:32:59.247Z" }, + { url = "https://files.pythonhosted.org/packages/7c/f9/07dd09ae774e4616edf6cda684ee78f97777bdd15847253637a6f052a62f/pyyaml-6.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3", size = 189108, upload-time = "2025-09-25T21:32:44.377Z" }, + { url = "https://files.pythonhosted.org/packages/4e/78/8d08c9fb7ce09ad8c38ad533c1191cf27f7ae1effe5bb9400a46d9437fcf/pyyaml-6.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba", size = 183641, upload-time = "2025-09-25T21:32:45.407Z" }, + { url = "https://files.pythonhosted.org/packages/7b/5b/3babb19104a46945cf816d047db2788bcaf8c94527a805610b0289a01c6b/pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c", size = 831901, upload-time = "2025-09-25T21:32:48.83Z" }, + { url = "https://files.pythonhosted.org/packages/8b/cc/dff0684d8dc44da4d22a13f35f073d558c268780ce3c6ba1b87055bb0b87/pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702", size = 861132, upload-time = "2025-09-25T21:32:50.149Z" }, + { url = "https://files.pythonhosted.org/packages/b1/5e/f77dc6b9036943e285ba76b49e118d9ea929885becb0a29ba8a7c75e29fe/pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c", size = 839261, upload-time = "2025-09-25T21:32:51.808Z" }, + { url = "https://files.pythonhosted.org/packages/ce/88/a9db1376aa2a228197c58b37302f284b5617f56a5d959fd1763fb1675ce6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065", size = 805272, upload-time = "2025-09-25T21:32:52.941Z" }, + { url = "https://files.pythonhosted.org/packages/da/92/1446574745d74df0c92e6aa4a7b0b3130706a4142b2d1a5869f2eaa423c6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65", size = 829923, upload-time = "2025-09-25T21:32:54.537Z" }, + { url = "https://files.pythonhosted.org/packages/f0/7a/1c7270340330e575b92f397352af856a8c06f230aa3e76f86b39d01b416a/pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9", size = 174062, upload-time = "2025-09-25T21:32:55.767Z" }, + { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, +] + +[[package]] +name = "requests" +version = "2.34.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "charset-normalizer" }, + { name = "idna" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ac/c3/e2a2b89f2d3e2179abd6d00ebd70bff6273f37fb3e0cc209f48b39d00cbf/requests-2.34.2.tar.gz", hash = "sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed", size = 142856, upload-time = "2026-05-14T19:25:27.735Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0", size = 73075, upload-time = "2026-05-14T19:25:26.443Z" }, +] + +[[package]] +name = "roman-numerals" +version = "4.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ae/f9/41dc953bbeb056c17d5f7a519f50fdf010bd0553be2d630bc69d1e022703/roman_numerals-4.1.0.tar.gz", hash = "sha256:1af8b147eb1405d5839e78aeb93131690495fe9da5c91856cb33ad55a7f1e5b2", size = 9077, upload-time = "2025-12-17T18:25:34.381Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/54/6f679c435d28e0a568d8e8a7c0a93a09010818634c3c3907fc98d8983770/roman_numerals-4.1.0-py3-none-any.whl", hash = "sha256:647ba99caddc2cc1e55a51e4360689115551bf4476d90e8162cf8c345fe233c7", size = 7676, upload-time = "2025-12-17T18:25:33.098Z" }, +] + +[[package]] +name = "snowballstemmer" +version = "3.1.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/43/f8/0a71edf031f03c40db17503cb8ca78a69a171254e568e7db241b0ab57ea1/snowballstemmer-3.1.1.tar.gz", hash = "sha256:e07bbc54a0d798fe6010a12398422e62a8bfbba95c394fd0956ef58cb4d3e260", size = 123314, upload-time = "2026-06-03T00:56:40.194Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4c/07/2ebca9b11fb9be7340a818d8d6f63feaebb146be2c4afbd6061701d6df6e/snowballstemmer-3.1.1-py3-none-any.whl", hash = "sha256:7e207fa178741da09cdee59d3ecec3827ad5f92b1fc5c9ff3755b639f71f5752", size = 104164, upload-time = "2026-06-03T00:56:38.614Z" }, +] + [[package]] name = "sortedcontainers" version = "2.4.0" @@ -1078,6 +1572,145 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/32/46/9cb0e58b2deb7f82b84065f37f3bffeb12413f947f9388e4cac22c4621ce/sortedcontainers-2.4.0-py2.py3-none-any.whl", hash = "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0", size = 29575, upload-time = "2021-05-16T22:03:41.177Z" }, ] +[[package]] +name = "soupsieve" +version = "4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7d/5e/19b7c5d20bc2f226c44f33a9d9aab9740efed7555b96e656231fd564e514/soupsieve-4.0.tar.gz", hash = "sha256:ccd6470b256394ac44b08b86fd94b30a17d335a5335c8a3d15565f7b03d3c8cf", size = 151929, upload-time = "2026-11-03T14:28:55.388Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4f/18/d24df2f733ee0655c58717d51add4118a70e64692d0e47d403b9aac4cf98/soupsieve-4.0-py3-none-any.whl", hash = "sha256:5e7494bfce90e06676cfb4c5c2cf22243e98e5cf24646bb1ecdda46d318550a0", size = 50158, upload-time = "2026-11-03T14:28:53.988Z" }, +] + +[[package]] +name = "sphinx" +version = "9.1.1" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version < '3.13'", +] +dependencies = [ + { name = "alabaster" }, + { name = "babel" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "docutils" }, + { name = "imagesize" }, + { name = "jinja2" }, + { name = "packaging" }, + { name = "pygments" }, + { name = "requests" }, + { name = "roman-numerals" }, + { name = "snowballstemmer" }, + { name = "sphinxcontrib-applehelp" }, + { name = "sphinxcontrib-devhelp" }, + { name = "sphinxcontrib-htmlhelp" }, + { name = "sphinxcontrib-jsmath" }, + { name = "sphinxcontrib-qthelp" }, + { name = "sphinxcontrib-serializinghtml" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/fc/73/567727fc8c8d85bc3adc88a246e12937cfda8d565cf1d80e85bceaa61fe5/sphinx-9.1.1.tar.gz", hash = "sha256:48b627f02908f22c601b3ba9c7b17265b4f315dd05c8bc64cd4da41183f44fbc", size = 8725638, upload-time = "2026-10-20T00:41:13.96Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/47/18/3c66ea76da7de47e716b181c35bdbfafbd5e9af0bb247b14578e9de348e9/sphinx-9.1.1-py3-none-any.whl", hash = "sha256:f8b60013ce25017be41a509e5c2d265bc1918410061315c36b809c7f335646c0", size = 3923559, upload-time = "2026-10-20T00:41:11.288Z" }, +] + +[[package]] +name = "sphinx" +version = "9.2.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.15'", + "python_full_version >= '3.13' and python_full_version < '3.15'", +] +dependencies = [ + { name = "alabaster" }, + { name = "babel" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "docutils" }, + { name = "imagesize" }, + { name = "jinja2" }, + { name = "packaging" }, + { name = "pygments" }, + { name = "requests" }, + { name = "roman-numerals" }, + { name = "snowballstemmer" }, + { name = "sphinxcontrib-applehelp" }, + { name = "sphinxcontrib-devhelp" }, + { name = "sphinxcontrib-htmlhelp" }, + { name = "sphinxcontrib-jsmath" }, + { name = "sphinxcontrib-qthelp" }, + { name = "sphinxcontrib-serializinghtml" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/64/d4/413466f0be8b1cb0e3e43893a148b96d7462d0923d82b4166b21afc5f30c/sphinx-9.2.0.tar.gz", hash = "sha256:90470459f3cab15d9dfed146c71cf77694b0445df17b2e0e69059cb9f7fc9ee4", size = 8782438, upload-time = "2026-10-21T00:05:15.019Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c4/28/a0c50761cd215b7042f4bc617a6bc5c0cd7be62f75bc908e169e28a4d21e/sphinx-9.2.0-py3-none-any.whl", hash = "sha256:4c3ec13b612712c0d3ce4cbdd9e3eb91e68577e07a8e31f32f21cf178453578c", size = 3925388, upload-time = "2026-10-21T00:05:12.241Z" }, +] + +[[package]] +name = "sphinx-basic-ng" +version = "1.0.0b2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "sphinx", version = "9.1.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13'" }, + { name = "sphinx", version = "9.2.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/98/0b/a866924ded68efec7a1759587a4e478aec7559d8165fac8b2ad1c0e774d6/sphinx_basic_ng-1.0.0b2.tar.gz", hash = "sha256:9ec55a47c90c8c002b5960c57492ec3021f5193cb26cebc2dc4ea226848651c9", size = 20736, upload-time = "2023-07-08T18:40:54.166Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3c/dd/018ce05c532a22007ac58d4f45232514cd9d6dd0ee1dc374e309db830983/sphinx_basic_ng-1.0.0b2-py3-none-any.whl", hash = "sha256:eb09aedbabfb650607e9b4b68c9d240b90b1e1be221d6ad71d61c52e29f7932b", size = 22496, upload-time = "2023-07-08T18:40:52.659Z" }, +] + +[[package]] +name = "sphinxcontrib-applehelp" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ba/6e/b837e84a1a704953c62ef8776d45c3e8d759876b4a84fe14eba2859106fe/sphinxcontrib_applehelp-2.0.0.tar.gz", hash = "sha256:2f29ef331735ce958efa4734873f084941970894c6090408b079c61b2e1c06d1", size = 20053, upload-time = "2024-07-29T01:09:00.465Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5d/85/9ebeae2f76e9e77b952f4b274c27238156eae7979c5421fba91a28f4970d/sphinxcontrib_applehelp-2.0.0-py3-none-any.whl", hash = "sha256:4cd3f0ec4ac5dd9c17ec65e9ab272c9b867ea77425228e68ecf08d6b28ddbdb5", size = 119300, upload-time = "2024-07-29T01:08:58.99Z" }, +] + +[[package]] +name = "sphinxcontrib-devhelp" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f6/d2/5beee64d3e4e747f316bae86b55943f51e82bb86ecd325883ef65741e7da/sphinxcontrib_devhelp-2.0.0.tar.gz", hash = "sha256:411f5d96d445d1d73bb5d52133377b4248ec79db5c793ce7dbe59e074b4dd1ad", size = 12967, upload-time = "2024-07-29T01:09:23.417Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/35/7a/987e583882f985fe4d7323774889ec58049171828b58c2217e7f79cdf44e/sphinxcontrib_devhelp-2.0.0-py3-none-any.whl", hash = "sha256:aefb8b83854e4b0998877524d1029fd3e6879210422ee3780459e28a1f03a8a2", size = 82530, upload-time = "2024-07-29T01:09:21.945Z" }, +] + +[[package]] +name = "sphinxcontrib-htmlhelp" +version = "2.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/43/93/983afd9aa001e5201eab16b5a444ed5b9b0a7a010541e0ddfbbfd0b2470c/sphinxcontrib_htmlhelp-2.1.0.tar.gz", hash = "sha256:c9e2916ace8aad64cc13a0d233ee22317f2b9025b9cf3295249fa985cc7082e9", size = 22617, upload-time = "2024-07-29T01:09:37.889Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0a/7b/18a8c0bcec9182c05a0b3ec2a776bba4ead82750a55ff798e8d406dae604/sphinxcontrib_htmlhelp-2.1.0-py3-none-any.whl", hash = "sha256:166759820b47002d22914d64a075ce08f4c46818e17cfc9470a9786b759b19f8", size = 98705, upload-time = "2024-07-29T01:09:36.407Z" }, +] + +[[package]] +name = "sphinxcontrib-jsmath" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/59/02/4f334c7fc3e54c51d2364575a41cd2fae6847e7f53001949f0be5f32e18b/sphinxcontrib_jsmath-2.0.0.tar.gz", hash = "sha256:551f780cf9687f9c3c74f0cf9f18aca06e0d6958853453f3803fe4d357c5b924", size = 14511, upload-time = "2026-10-20T18:22:05.514Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/da/c2/2965c35b39587d4b7a397f65a9c7d4b942b031061e7ac1ce91782f394a0d/sphinxcontrib_jsmath-2.0.0-py3-none-any.whl", hash = "sha256:15ca92fe0eec82f0baf91a7efcc51ffacdb605924be6327683a52ebdacb6744c", size = 78095, upload-time = "2026-10-20T18:22:04.42Z" }, +] + +[[package]] +name = "sphinxcontrib-qthelp" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/68/bc/9104308fc285eb3e0b31b67688235db556cd5b0ef31d96f30e45f2e51cae/sphinxcontrib_qthelp-2.0.0.tar.gz", hash = "sha256:4fe7d0ac8fc171045be623aba3e2a8f613f8682731f9153bb2e40ece16b9bbab", size = 17165, upload-time = "2024-07-29T01:09:56.435Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/27/83/859ecdd180cacc13b1f7e857abf8582a64552ea7a061057a6c716e790fce/sphinxcontrib_qthelp-2.0.0-py3-none-any.whl", hash = "sha256:b18a828cdba941ccd6ee8445dbe72ffa3ef8cbe7505d8cd1fa0d42d3f2d5f3eb", size = 88743, upload-time = "2024-07-29T01:09:54.885Z" }, +] + +[[package]] +name = "sphinxcontrib-serializinghtml" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/3b/44/6716b257b0aa6bfd51a1b31665d1c205fb12cb5ad56de752dfa15657de2f/sphinxcontrib_serializinghtml-2.0.0.tar.gz", hash = "sha256:e9d912827f872c029017a53f0ef2180b327c3f7fd23c87229f7a8e8b70031d4d", size = 16080, upload-time = "2024-07-29T01:10:09.332Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/52/a7/d2782e4e3f77c8450f727ba74a8f12756d5ba823d81b941f1b04da9d033a/sphinxcontrib_serializinghtml-2.0.0-py3-none-any.whl", hash = "sha256:6e2cb0eef194e10c27ec0023bfeb25badbbb5868244cf5bc5bdc04e4464bf331", size = 92072, upload-time = "2024-07-29T01:10:08.203Z" }, +] + [[package]] name = "typing-extensions" version = "4.16.0" @@ -1087,6 +1720,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8", size = 45571, upload-time = "2026-07-02T08:40:04.659Z" }, ] +[[package]] +name = "urllib3" +version = "2.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e3/05/b17359e1cefb4f909b5e40b1b90a496d987258916dbbf88e842c729f510e/urllib3-2.8.0.tar.gz", hash = "sha256:63bf2ead4c879426ebf22ef2a781eeb4aa3b4ae798a0435506f8687fd5bb9b63", size = 458972, upload-time = "2026-09-15T19:29:36.253Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/92/9d/c4e665119135114480843e7ab388fa94d8480650450e6f8e26b70d323a4c/urllib3-2.8.0-py3-none-any.whl", hash = "sha256:0cf3cae568d36aa9576b28dfb35f11328f1cb974ca7647d9475ebb86c75ac6e3", size = 135717, upload-time = "2026-09-15T19:29:34.577Z" }, +] + [[package]] name = "wrapt" version = "2.5.2" From aa2aa614c3e492fb13f55963b8315c217db93165 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 3 Nov 2026 18:26:46 +0100 Subject: [PATCH 048/250] Publish tagged releases to PyPI with Trusted Publishing A tag such as v3.0.0 builds and checks the distributions, publishes them without a stored token and creates the GitHub release; a, b and rc tags become pre-releases. The environment "pypi" and a trusted publisher on pypi.org must be set up before the first release. --- .github/workflows/release.yml | 72 +++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 .github/workflows/release.yml diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000..881bb5fc --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,72 @@ +name: Release + +# Publishes a tag such as v3.0.0 or v3.0.0rc1 to PyPI and creates a GitHub release. +# Needs, once: a "pypi" environment in the repository settings, and a trusted +# publisher for this workflow and environment on pypi.org. + +on: + push: + tags: ["v*"] + +permissions: + contents: read # checkout only; the jobs below add what they need + +concurrency: + group: release-${{ github.ref }} + cancel-in-progress: false # never stop a release halfway + +jobs: + build: + name: Build distributions + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + fetch-depth: 0 # hatch-vcs derives the version from the tag + - uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3 + with: + enable-cache: false # a release does not restore caches that other runs wrote + - run: uv build + - run: uvx twine check --strict dist/* + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: dist + path: dist/ + + publish: + name: Publish to PyPI + needs: build + runs-on: ubuntu-latest + environment: + name: pypi + url: https://pypi.org/project/Exscript/ + permissions: + id-token: write # Trusted Publishing, and signing the attestations + steps: + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: dist + path: dist/ + - uses: pypa/gh-action-pypi-publish@aef6c9b6e301dc803e1bc9133e9ce67ae9b0906b # v1.14.3 + + github-release: + name: Create the GitHub release + needs: publish + runs-on: ubuntu-latest + permissions: + contents: write # creates the release and uploads the distributions + steps: + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: dist + path: dist/ + - name: Create the release with the distributions + env: + GH_TOKEN: ${{ github.token }} + run: | + prerelease=() + if [[ "$GITHUB_REF_NAME" =~ [0-9](a|b|rc)[0-9]+$ ]]; then prerelease=(--prerelease); fi + gh release create "$GITHUB_REF_NAME" dist/* --repo "$GITHUB_REPOSITORY" --verify-tag \ + --title "Exscript ${GITHUB_REF_NAME#v}" "${prerelease[@]}" \ + --notes "Changes: https://github.com/$GITHUB_REPOSITORY/blob/$GITHUB_REF_NAME/CHANGES.md" From 0232ebb39df0563302fac4ceec7921d66c3fe2a2 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 3 Nov 2026 18:29:59 +0100 Subject: [PATCH 049/250] Add issue forms and a pull request template One form is for bugs, one for devices that get the wrong driver or none. Both say what to include, such as a log, and remind reporters to remove their network data. --- .github/ISSUE_TEMPLATE/bug_report.yml | 65 +++++++++++++++++++++++ .github/ISSUE_TEMPLATE/config.yml | 8 +++ .github/ISSUE_TEMPLATE/device_support.yml | 61 +++++++++++++++++++++ .github/pull_request_template.md | 6 +++ 4 files changed, 140 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/bug_report.yml create mode 100644 .github/ISSUE_TEMPLATE/config.yml create mode 100644 .github/ISSUE_TEMPLATE/device_support.yml create mode 100644 .github/pull_request_template.md diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml new file mode 100644 index 00000000..9ffb5530 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -0,0 +1,65 @@ +name: Bug report +description: Something does not work as documented. +labels: ["bug"] +body: + - type: markdown + attributes: + value: > + Thank you for reporting a problem. Please report security problems + privately instead, as described in SECURITY.md. Remove passwords, host + names, addresses and other data of your network from everything you + paste here. + - type: input + id: version + attributes: + label: Exscript version + description: 'Output of: python -c "import Exscript; print(Exscript.__version__)"' + placeholder: 3.0.0 + validations: + required: true + - type: input + id: environment + attributes: + label: Python version and operating system + placeholder: Python 3.13.9 on Ubuntu 24.04 + validations: + required: true + - type: dropdown + id: area + attributes: + label: Area + options: + - Connecting and logging in (SSH, Telnet) + - Sending commands and waiting for responses + - NETCONF + - Template language or the exscript command + - Other + validations: + required: true + - type: input + id: device + attributes: + label: Device operating system and version + description: If a device is involved. + placeholder: Cisco IOS XR 7.11.2 + - type: textarea + id: description + attributes: + label: What happened, and what did you expect? + validations: + required: true + - type: textarea + id: reproduce + attributes: + label: Script or template that shows the problem + description: As short as possible. + render: python + - type: textarea + id: log + attributes: + label: Traceback and log + description: > + The traceback, and the conversation with the device: connect with + logfile="session.log", or with debug=5 after + logging.basicConfig(level=logging.DEBUG). + render: text diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 00000000..fd21d451 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,8 @@ +blank_issues_enabled: true +contact_links: + - name: Documentation + url: https://exscript.readthedocs.io + about: Tutorials, the template language and the API reference. + - name: Report a vulnerability + url: https://github.com/knipknap/exscript/security/advisories/new + about: Report security problems privately, not in a public issue. diff --git a/.github/ISSUE_TEMPLATE/device_support.yml b/.github/ISSUE_TEMPLATE/device_support.yml new file mode 100644 index 00000000..6005d01f --- /dev/null +++ b/.github/ISSUE_TEMPLATE/device_support.yml @@ -0,0 +1,61 @@ +name: Device not detected or not supported +description: Exscript picks the wrong driver for a device, or has none for it. +body: + - type: markdown + attributes: + value: > + Exscript picks the driver from the login banner, the prompts and the + protocol. To improve this, we need the text that the device sends until + the first command prompt. Remove passwords, host names, addresses and + other data of your network first. + - type: input + id: version + attributes: + label: Exscript version + description: 'Output of: python -c "import Exscript; print(Exscript.__version__)"' + placeholder: 3.0.0 + validations: + required: true + - type: input + id: device + attributes: + label: Device vendor, model, operating system and version + placeholder: Cisco NCS 5501, IOS XR 7.11.2 + validations: + required: true + - type: dropdown + id: protocol + attributes: + label: Protocol + options: + - SSH + - Telnet + - NETCONF + validations: + required: true + - type: input + id: guessed + attributes: + label: Driver that Exscript picks + description: Output of conn.guess_os() after conn.login(). + placeholder: generic + validations: + required: true + - type: input + id: expected + attributes: + label: Driver you expected + description: The name of an existing driver, or "new driver". + placeholder: ios_xr + - type: textarea + id: transcript + attributes: + label: Login transcript + description: > + Everything from connecting to the first command prompt, including the + banner and all prompts, for example from logfile="session.log". With + debug=3 after logging.basicConfig(level=logging.DEBUG), the log also + shows the text that the detection looked at. + render: text + validations: + required: true diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 00000000..25930783 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,6 @@ + + +- [ ] Tests cover the change (`uv run pytest`). +- [ ] `CHANGES.md` describes changes that users notice. +- [ ] Documentation and docstrings are updated. +- [ ] Recordings and logs contain no passwords, host names or addresses of real networks. From c385f4c7c4878c406580ef1b562b35b20098d911 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 3 Nov 2026 18:45:52 +0100 Subject: [PATCH 050/250] Do not echo the password in the virtual device VirtualDevice, behind the Dummy protocol and the test servers, echoed every input, so a login put the password into the log and the response, unlike a real device. --- CHANGES.md | 5 +++++ src/Exscript/emulators/vdevice.py | 5 ++++- tests/Exscript/emulators/VirtualDeviceTest.py | 12 ++++++++++++ tests/Exscript/protocols/DummyTest.py | 4 ++-- tests/Exscript/protocols/ProtocolTest.py | 4 ++-- 5 files changed, 25 insertions(+), 5 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index ebf5e8fa..d7ccdd08 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -281,6 +281,11 @@ marked with **BREAKING**; a migration guide will accompany the release. ## Emulators +- **BREAKING - Password Not Echoed:** `VirtualDevice`, which the `Dummy` protocol and + the test servers use, no longer echoes the password, like real devices. The log and + `response` of a login to it no longer contain the password; `response` after the + login is empty instead of the password. + ### Class `Exscript.emulators.CommandSet` | Old | New | Comment | diff --git a/src/Exscript/emulators/vdevice.py b/src/Exscript/emulators/vdevice.py index 57d54272..3372c826 100644 --- a/src/Exscript/emulators/vdevice.py +++ b/src/Exscript/emulators/vdevice.py @@ -72,6 +72,7 @@ def _get_prompt(self): return self.user_prompt elif self.prompt_stage == self.PROMPT_STAGE_PASSWORD: self.prompt_stage = self.PROMPT_STAGE_CUSTOM + self._hide_echo = True # like real devices, do not echo the password return self.password_prompt elif self.prompt_stage == self.PROMPT_STAGE_CUSTOM: self.logged_in = True @@ -149,6 +150,7 @@ def init(self): :return: The initial response of the virtual device. """ self.logged_in = False + self._hide_echo = False if self.login_type == self.LOGIN_TYPE_PASSWORDONLY: self.prompt_stage = self.PROMPT_STAGE_PASSWORD @@ -169,7 +171,8 @@ def do(self, command): :rtype: str :return: The response of the virtual device. """ - echo = self.echo and command or '' + echo = self.echo and not self._hide_echo and command or '' + self._hide_echo = False if not self.logged_in: return echo + '\n' + self._get_prompt() diff --git a/tests/Exscript/emulators/VirtualDeviceTest.py b/tests/Exscript/emulators/VirtualDeviceTest.py index c3fa2a94..9de5398f 100644 --- a/tests/Exscript/emulators/VirtualDeviceTest.py +++ b/tests/Exscript/emulators/VirtualDeviceTest.py @@ -53,6 +53,18 @@ def testInit(self): cs = self.cls('myhost', login_type=self.cls.LOGIN_TYPE_NONE) self.assertEqual(cs.init(), self.banner + self.prompt) + def testDoHidesPassword(self): + # Like real devices, the device echoes the user name but not the password. + cs = self.cls('myhost', strict=False) + self.assertEqual(cs.init(), self.banner + self.userprompt) + self.assertEqual(cs.do('user'), 'user\n' + self.passwdprompt) + self.assertEqual(cs.do('secret'), '\n' + self.prompt) + self.assertEqual(cs.do('ls'), 'ls\n' + self.prompt) + + cs = self.cls('myhost', strict=False, login_type=self.cls.LOGIN_TYPE_PASSWORDONLY) + self.assertEqual(cs.init(), self.banner + self.passwdprompt) + self.assertEqual(cs.do('secret'), '\n' + self.prompt) + def testDo(self): pass # See testAddCommand() diff --git a/tests/Exscript/protocols/DummyTest.py b/tests/Exscript/protocols/DummyTest.py index 9ce444af..ce9b5f6b 100644 --- a/tests/Exscript/protocols/DummyTest.py +++ b/tests/Exscript/protocols/DummyTest.py @@ -92,7 +92,7 @@ def testDummy(self): self.assertEqual(str(protocol._buffer), 'Password: ') protocol.login(self.account) self.assertEqual(str(protocol._buffer), '') - self.assertEqual(protocol.response, self.password) + self.assertEqual(protocol.response, '') # the password is not echoed protocol.close() # Test login with port number. @@ -100,7 +100,7 @@ def testDummy(self): self.assertEqual(str(protocol._buffer), 'Password: ') protocol.login(self.account) self.assertEqual(str(protocol._buffer), '') - self.assertEqual(protocol.response, self.password) + self.assertEqual(protocol.response, '') # the password is not echoed protocol.close() # Test a custom response. diff --git a/tests/Exscript/protocols/ProtocolTest.py b/tests/Exscript/protocols/ProtocolTest.py index 1fb98305..98ae7542 100644 --- a/tests/Exscript/protocols/ProtocolTest.py +++ b/tests/Exscript/protocols/ProtocolTest.py @@ -312,7 +312,7 @@ def testLogin(self): # Password login. self.doLogin(flush=False) self.assertTrue(self.protocol.response is not None) - self.assertTrue(len(self.protocol.response) > 0) + self.assertNotIn(self.password, self.protocol.response) self.assertTrue(self.protocol.is_protocol_authenticated()) self.assertTrue(self.protocol.is_app_authenticated()) self.assertTrue(self.protocol.is_app_authorized()) @@ -344,7 +344,7 @@ def testAuthenticate(self): self.assertFalse(self.protocol.is_app_authorized()) self.protocol.authenticate(self.account, flush=False) self.assertTrue(self.protocol.response is not None) - self.assertTrue(len(self.protocol.response) > 0) + self.assertNotIn(self.password, self.protocol.response) self.assertTrue(self.protocol.is_protocol_authenticated()) self.assertTrue(self.protocol.is_app_authenticated()) self.assertFalse(self.protocol.is_app_authorized()) From 3bf33f63fd6aaa18db80eefae67d5cf377e747ec Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 3 Nov 2026 18:53:04 +0100 Subject: [PATCH 051/250] Fix disable_driver() It passed the driver to list.pop(), which takes an index, so it raised a TypeError, and the OS guesser still chose the driver. --- CHANGES.md | 4 ++ src/Exscript/protocols/drivers/__init__.py | 18 ++++++- tests/Exscript/protocols/OsGuesserTest.py | 60 +++++++++++++++++++++- tests/Exscript/protocols/ProtocolTest.py | 5 -- 4 files changed, 79 insertions(+), 8 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index d7ccdd08..bc81d364 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -260,6 +260,10 @@ marked with **BREAKING**; a migration guide will accompany the release. local shell) before the connection is closed gracefully, otherwise the logout command may fail. +- **`disable_driver()`:** `Exscript.protocols.drivers.disable_driver()` works; it + raised a `TypeError` after it had removed the driver only from `driver_map`, so the + OS guesser still chose it. + #### Class `Exscript.protocols.drivers.Driver` | Old | New | Comment | diff --git a/src/Exscript/protocols/drivers/__init__.py b/src/Exscript/protocols/drivers/__init__.py index 62015d6d..0a377247 100644 --- a/src/Exscript/protocols/drivers/__init__.py +++ b/src/Exscript/protocols/drivers/__init__.py @@ -68,6 +68,13 @@ def isdriver(o): def add_driver(cls): + """ + Registers a driver class, so that the OS guesser and ``driver=`` can use + it in connections that are created afterwards. + + :type cls: type[Driver] + :param cls: The driver class; it is instantiated without arguments. + """ driver = cls() driver_classes.append(cls) drivers.append(driver) @@ -75,9 +82,16 @@ def add_driver(cls): def disable_driver(name): + """ + Unregisters the driver with the given name. Connections that are created + afterwards neither guess it nor accept its name. + + :type name: str + :param name: The name of the driver. + """ driver = driver_map.pop(name) - drivers.pop(driver) - driver_classes.pop(driver.__class__) + drivers.remove(driver) + driver_classes.remove(type(driver)) # Load built-in drivers. diff --git a/tests/Exscript/protocols/OsGuesserTest.py b/tests/Exscript/protocols/OsGuesserTest.py index bd52dbfe..25f4339e 100644 --- a/tests/Exscript/protocols/OsGuesserTest.py +++ b/tests/Exscript/protocols/OsGuesserTest.py @@ -157,10 +157,68 @@ def testLastLoginLineDoesNotBreakLogin(self): self.assertEqual(conn.get_response().strip(), 'Linux') +class AcmeDriver(drivers.Driver): + """The custom driver of the documentation (docs/drivers.rst).""" + + def __init__(self): + drivers.Driver.__init__(self, 'acme') + self.prompt_re = [re.compile(r'[\r\n]acme-[\w-]+[>#] ?$')] + + def check_head_for_os(self, string): + if 'Acme Networks OS' in string: + return 90 + return 0 + + def init_terminal(self, conn): + conn.execute('terminal length 0') + + +class DriverRegistryTest(unittest.TestCase): + def setUp(self): + self.sent = [] + self.device = VirtualDevice( + 'acme-pe1', login_type=VirtualDevice.LOGIN_TYPE_NONE, banner='Acme Networks OS 1.0\n' + ) + self.device.add_command( + 'terminal length 0', lambda command: self.sent.append(command) or '' + ) + + def tearDown(self): + if 'acme' in drivers.driver_map: + drivers.disable_driver('acme') + + def login(self): + conn = Dummy(device=self.device) + conn.connect('acme-pe1') + conn.login(Account('user', 'password', needs_lock=False)) + return conn + + def testAddDriver(self): + drivers.add_driver(AcmeDriver) + conn = self.login() + self.assertEqual(conn.guess_os(), 'acme') + self.assertIsInstance(conn.get_driver(), AcmeDriver) + conn.autoinit() + self.assertEqual(len(self.sent), 1) + + def testDisableDriver(self): + drivers.add_driver(AcmeDriver) + drivers.disable_driver('acme') + self.assertNotIn('acme', drivers.driver_map) + self.assertNotIn(AcmeDriver, drivers.driver_classes) + self.assertNotIn('acme', [driver.name for driver in drivers.drivers]) + self.assertEqual(self.login().guess_os(), 'unknown') + self.assertRaises(TypeError, Dummy().set_driver, 'acme') + + def suite(): loader = unittest.TestLoader() return unittest.TestSuite( - [loader.loadTestsFromTestCase(OsGuesserTest), loader.loadTestsFromTestCase(SignalTest)] + [ + loader.loadTestsFromTestCase(OsGuesserTest), + loader.loadTestsFromTestCase(SignalTest), + loader.loadTestsFromTestCase(DriverRegistryTest), + ] ) diff --git a/tests/Exscript/protocols/ProtocolTest.py b/tests/Exscript/protocols/ProtocolTest.py index 98ae7542..091f3183 100644 --- a/tests/Exscript/protocols/ProtocolTest.py +++ b/tests/Exscript/protocols/ProtocolTest.py @@ -68,11 +68,6 @@ def createDaemon(self): def createProtocol(self): self.protocol = Protocol(timeout=1, debug=5) - def disableDriver(self): - self.assertIn(drivers.driver_map, 'ios') - drivers.disable_driver('ios') - self.assertNotIn(drivers.driver_map, 'ios') - def doConnect(self): self.protocol.connect(self.hostname, self.port) From 5a921310eaee3e4c4cc6a5f020ccd46ec9de86ea Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 3 Nov 2026 19:13:25 +0100 Subject: [PATCH 052/250] Correct docstrings that the documentation shows wrongly get_accounts_from_file() called base64 an encryption, the examples of first_match() and any_match() were wrong, and two URL formats became broken links in the API reference. --- src/Exscript/host.py | 2 +- src/Exscript/util/file.py | 5 +++-- src/Exscript/util/match.py | 6 +++--- src/Exscript/util/url.py | 2 +- 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/src/Exscript/host.py b/src/Exscript/host.py index 4aef7e61..b2743388 100644 --- a/src/Exscript/host.py +++ b/src/Exscript/host.py @@ -66,7 +66,7 @@ def set_uri(self, uri): """ Defines the protocol, hostname/address, TCP port number, username, and password from the given URL. The hostname may be URL formatted, - so the following formats are all valid: + so the following formats are all valid:: - myhostname - myhostname.domain diff --git a/src/Exscript/util/file.py b/src/Exscript/util/file.py index 2f5c64ac..cdd324fd 100644 --- a/src/Exscript/util/file.py +++ b/src/Exscript/util/file.py @@ -49,8 +49,9 @@ def get_accounts_from_file(filename, encoding='utf-8', password_encoding='utf-8' Note that "cGFzc3dvcmQ=" is a base64 encoded password. If the input file contains extra config sections other than "account-pool", they are ignored. - Each password needs to be base64 encrypted. To encrypt a password, - you may use the following command:: + Each password needs to be base64 encoded. Base64 does not protect the + passwords: anyone who can read the file can decode them. To encode a + password, you may use the following command:: python3 -c 'import base64; print(base64.b64encode("thepassword".encode()).decode())' diff --git a/src/Exscript/util/match.py b/src/Exscript/util/match.py index 99ce27bc..cde1f874 100644 --- a/src/Exscript/util/match.py +++ b/src/Exscript/util/match.py @@ -71,7 +71,7 @@ def first_match(string, regex, flags=re.M): match = first_match(foo, r'(aaa)') # Returns None match = first_match(foo, r'(\S+)') # Returns 'my' match = first_match(foo, r'(aaa) (\S+)') # Returns (None, None) - match = first_match(foo, r'(\S+) (\S+)') # Returns ('my', 'foo') + match = first_match(foo, r'(\S+) (\S+)') # Returns ('my', 'test') :type string: string|Exscript.protocols.Protocol :param string: The string that is matched, or a Protocol object. @@ -118,10 +118,10 @@ def any_match(string, regex, flags=re.M): print(m) for one, two in any_match(foo, r'(aaa) (\S+)'): # Returns [] - print(m) + print(one, two) for one, two in any_match(foo, r'(\S+) (\S+)'): # Returns [('1', 'uno'), ('2', 'due')] - print(m) + print(one, two) :type string: string|Exscript.protocols.Protocol :param string: The string that is matched, or a Protocol object. diff --git a/src/Exscript/util/url.py b/src/Exscript/util/url.py index d822e948..029aa08c 100644 --- a/src/Exscript/util/url.py +++ b/src/Exscript/util/url.py @@ -168,7 +168,7 @@ def from_string(url, default_protocol='ssh'): - It is less strict, many more inputs are accepted. This is necessary to allow for passing a simple hostname as a URL. - - You may specify a default protocol that is used when the http:// + - You may specify a default protocol that is used when the ``http://`` portion is missing. - The port number defaults to the well-known port of the given protocol. From 81b861c91ea05fa4820023f0691a20c499c3c958 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 3 Nov 2026 19:13:25 +0100 Subject: [PATCH 053/250] Document drivers, NETCONF, security, platforms and the migration to 3.0 The command line reference and the driver list are generated at build time, because the copied help text was outdated. The tutorial uses Python 3, and CHANGES.md gets the missing entries. --- .gitignore | 1 + CHANGES.md | 25 ++++- CONTRIBUTING.md | 81 +++++++++++++++++ docs/conf.py | 29 ++++++ docs/contributing.md | 2 + docs/drivers.rst | 156 ++++++++++++++++++++++++++++++++ docs/exscript.rst | 109 ++-------------------- docs/index.rst | 22 +++-- docs/migration.rst | 191 +++++++++++++++++++++++++++++++++++++++ docs/netconf.rst | 112 +++++++++++++++++++++++ docs/platforms.rst | 51 +++++++++++ docs/python_tutorial.rst | 66 ++++++++------ docs/security.rst | 119 ++++++++++++++++++++++++ docs/templates.rst | 42 ++++++++- docs/trouble.rst | 21 +++++ 15 files changed, 891 insertions(+), 136 deletions(-) create mode 100644 CONTRIBUTING.md create mode 100644 docs/contributing.md create mode 100644 docs/drivers.rst create mode 100644 docs/migration.rst create mode 100644 docs/netconf.rst create mode 100644 docs/platforms.rst create mode 100644 docs/security.rst diff --git a/.gitignore b/.gitignore index 76cf3fde..1f536cbb 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,4 @@ htmlcov coverage.xml docs/_build docs/_autosummary +docs/_generated diff --git a/CHANGES.md b/CHANGES.md index bc81d364..41a29cea 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -15,10 +15,27 @@ marked with **BREAKING**; a migration guide will accompany the release. and log files, as UTF-8 on every platform, as Python 3.15 does by default. Before, it used the encoding of the locale, e.g. cp1252 on Windows, where writing device output to a log file could fail. -- **Python 2.7 Support:** -- **Python 3.11+ Support:** -- **Dependencies:** -- **NETCONF Protocol:** +- **BREAKING - Python 3.12 or Later:** Exscript runs on Python 3.12 to 3.15 and on the + free-threaded build of 3.14, on Linux, macOS and Windows. Python 2.7 and Python 3 + versions before 3.12 are no longer supported. +- **Dependencies:** `future` and `configparser`, backports for Python 2, are no longer + needed. paramiko 3.4 to 5.x is supported (before: 1.17 to 4.x). New: ncclient and + lxml 5.0 or later, for NETCONF. +- **NETCONF Protocol:** The new adapter `Exscript.protocols.Netconf` (scheme + `netconf://`, port 830) sends RPCs over SSH with ncclient, and returns the replies as + text or as lxml elements; an `` raises `RPCError`. The IOS XR and Junos + drivers recognize their devices by the NETCONF capabilities. +- **BREAKING - Internal Attributes and Helpers:** Attributes and helpers that were + meant for internal use are private or removed; the migration guide lists their + public replacements. Among them are the attributes `buffer`, `auto_driver`, + `manual_driver`, `manual_*_re`, `*_authenticated`, `last_account`, `cancel` and `log` + of the protocol adapters, `SSH2.client`, `shell` and `sock`, and `Telnet.tn`; + `OsGuesser.data_received()` and `protocol_info()`, now `check_received_data()` and + `check_protocol_info()`; `Exscript.util.template.test_secure()`, now + `test(secure=True)`; `InputHistory.file`, now `filename`; the `debug` argument of + `WorkQueue`, whose `set_debug()` argument is now `level`; the `type` argument of + `parselib.Lexer.expect()` and `current_is()`, now `types`; and the `dump1()` methods + of the template interpreter. - **OpenTelemetry Tracing (optional):** Install the `otel` extra (`Exscript[otel]`) and call `Exscript.contrib.otel.ExscriptInstrumentor().instrument()`. Every call of `connect()`, `login()`, `authenticate()`, `send()`, `execute()` and `close()` creates diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 00000000..b17cd1cd --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,81 @@ +# Contributing to Exscript + +Thank you for helping! Bug reports, recordings of devices, drivers, fixes and documentation +are all welcome. + +## Reporting problems + +Open an [issue](https://github.com/knipknap/exscript/issues/new/choose). There is a form for +bugs, and one for devices that get the wrong driver or none. Report vulnerabilities privately, +as described in the [security policy](https://github.com/knipknap/exscript/security/policy). + +## Development setup + +Exscript uses [uv](https://docs.astral.sh/uv/), which installs Python and all development tools +into `.venv`: + +```sh +git clone https://github.com/knipknap/exscript.git +cd exscript +uv sync +uvx pre-commit install +``` + +## Checks + +CI runs these checks on every pull request. Run them before you push: + +```sh +uv run pytest # the tests; warnings are errors +uv run mypy # the type checks +uvx pre-commit run --all-files # ruff, formatting, and checks of the workflows +uv run --group docs sphinx-build -W --keep-going docs docs/_build/html # the documentation +``` + +The tests need no devices and no network: they run against local test servers and replay +recorded sessions. To run them with another Python version, use e.g. +`uv run --python 3.15 pytest`. + +## Changes + +- A fix comes with a test that fails without it. +- Describe changes that users notice in `CHANGES.md`. Mark breaking changes with + **BREAKING**, and say how to migrate. +- Document new functions, classes and modules with docstrings in the style of the existing + code (`:param:`, `:type:`, `:return:`, `:rtype:`). The API reference is generated from + them. +- The public API and the template language stay compatible. Please open an issue before you + work on a change that breaks them. + +## Drivers and OS detection + +[Writing a driver](https://exscript.readthedocs.io/en/latest/drivers.html#writing-a-driver) +describes the driver class. Add the banner of the device (the text up to the first prompt, +sanitized) as `tests/Exscript/protocols/banners/.`, then update the expected +guesses and check that the accuracy does not drop: + +```sh +EXSCRIPT_UPDATE_GOLDEN=1 uv run pytest tests/Exscript/protocols/OsGuesserCorpusTest.py +uv run python tools/osguess_report.py --check +``` + +Review the diff of `osguess-baseline.json`: no guess that was right may become wrong. If the +accuracy improves, raise `MIN_CORRECT` in `tools/osguess_report.py`. + +## Recordings + +Recorded sessions of real devices let the tests cover what devices really send. +[tests/recordings/README.md](https://github.com/knipknap/exscript/blob/master/tests/recordings/README.md) +describes how to record, sanitize and add them. Never commit an unsanitized recording: device +output can contain passwords, addresses, host names and customer data. + +## Performance + +`uv run python tools/bench_bulk.py` measures the connection layer on loopback. CI fails if a +result exceeds four times its target. + +## Releases + +Maintainers release from `master`: update `CHANGES.md`, then push a tag such as `v3.0.0` or +`v3.0.0rc1`. The release workflow builds the distributions, publishes them to PyPI, and +creates the GitHub release; tags with `a`, `b` or `rc` become pre-releases. diff --git a/docs/conf.py b/docs/conf.py index ad98a624..4e3a1cb4 100755 --- a/docs/conf.py +++ b/docs/conf.py @@ -7,6 +7,35 @@ """ import importlib.metadata +import os +import subprocess +import sys +from pathlib import Path + +# The command line reference includes the output of "exscript --help". Sphinx +# runs this file in the docs directory. +_help = subprocess.run( + [sys.executable, "../scripts/exscript", "--help"], + capture_output=True, + check=True, + encoding="utf-8", + env=dict(os.environ, COLUMNS="80", NO_COLOR="1"), +).stdout +Path("_generated").mkdir(exist_ok=True) +Path("_generated/exscript-help.txt").write_text(_help, encoding="utf-8") + +# The list of drivers, from the docstrings of their modules. +from Exscript.protocols.drivers import driver_map # noqa: E402 + +_rows = [" * - Name\n - System\n"] +for _name, _driver in sorted(driver_map.items()): + if _name != "unknown": # an alias of generic + _doc = sys.modules[type(_driver).__module__].__doc__.strip().splitlines()[0] + _doc = _doc.removeprefix("A driver for ") + _rows.append(f" * - ``{_name}``\n - {_doc[0].upper()}{_doc[1:]}\n") +Path("_generated/drivers.inc").write_text( + ".. list-table::\n :header-rows: 1\n\n" + "".join(_rows), encoding="utf-8" +) project = "Exscript" author = "Samuel Abels and contributors" diff --git a/docs/contributing.md b/docs/contributing.md new file mode 100644 index 00000000..78caf34e --- /dev/null +++ b/docs/contributing.md @@ -0,0 +1,2 @@ +```{include} ../CONTRIBUTING.md +``` diff --git a/docs/drivers.rst b/docs/drivers.rst new file mode 100644 index 00000000..1c6cf79f --- /dev/null +++ b/docs/drivers.rst @@ -0,0 +1,156 @@ +Drivers and OS Detection +======================== + +Devices differ in their prompts, their error messages, their login, and in +the commands that make a session suitable for scripts, such as turning +off the pager. A *driver* holds this knowledge for one operating system. +Exscript chooses the driver while it connects and logs in, or uses the +one that you choose. + +Choosing a driver +----------------- + +If you know the operating system, for example from an inventory, pass the +name of its driver. This is more reliable than a guess:: + + from Exscript.protocols import SSH2 + + conn = SSH2(driver='ios_xr') + conn.set_driver('ios_xr') # the same, also after the login + +``set_driver(None)`` turns the guessing back on. A name that does not +exist makes ``set_driver()`` raise a ``TypeError``; the constructor +ignores it and guesses. + +For :class:`~Exscript.Host` objects, set the option ``driver``; the +command line tool has ``--use-driver``:: + + host.set_option('driver', 'ios_xr') + +Without a driver, Exscript guesses. ``conn.guess_os()`` returns the name +of the guessed driver, and ``conn.get_driver()`` the driver in use. + +The drivers +----------- + +.. include:: _generated/drivers.inc + +How the guess works +------------------- + +Each driver rates what the device sends with a confidence between 0 (no +sign of its system) and 100 (certain): + +- the SSH version string, for example ``SSH-2.0-Cisco-1.25`` + (``check_protocol_for_os()``); +- everything up to the end of the login, that is the banner and the + prompts (``check_head_for_os()``); +- everything received, mostly to recognize the prompt + (``check_response_for_os()``); after the login, only until a driver has + a confidence of 80 or more; +- for NETCONF, the capabilities in the ```` + (``netconf_capabilities_re``). + +The driver with the highest confidence wins, and Exscript switches to a +better driver as soon as the data shows it. If no driver recognizes the +device, the guess is ``unknown``, and the ``generic`` driver is used: its +patterns work with the prompts and error messages of most devices. + +Signals that several systems show, such as a ``Last login`` line or a +``router1 login:`` prompt, do not select a driver. A CI job measures the +accuracy of the guess on a corpus of device banners and recorded +sessions (``tools/osguess_report.py``). + +If Exscript picks the wrong driver for a device, set the driver, and +please `report it +`_ +with the login transcript. To see what the guess was based on, connect +with ``debug=3`` and enable Python's logging:: + + import logging + + logging.basicConfig(level=logging.DEBUG) + conn = SSH2(debug=3) + +Writing a driver +---------------- + +A driver is a subclass of :class:`~Exscript.protocols.drivers.driver.Driver` +that changes what differs from the generic driver. Register it with +:func:`~Exscript.protocols.drivers.add_driver`; connections that are +created afterwards can then guess it, and accept its name:: + + import re + + from Exscript.protocols.drivers import Driver, add_driver + + class AcmeDriver(Driver): + def __init__(self): + Driver.__init__(self, 'acme') + self.prompt_re = [re.compile(r'[\r\n]acme-[\w-]+[>#] ?$')] + + def check_head_for_os(self, string): + if 'Acme Networks OS' in string: + return 90 + return 0 + + def init_terminal(self, conn): + conn.execute('terminal length 0') + + add_driver(AcmeDriver) + +For the command line tool, put the class into a file without the call of +``add_driver()``, and pass the file with ``--load-driver``; the tool +registers every driver class in it. + +A driver can set these attributes in its constructor: + +``user_re``, ``password_re`` + Lists of patterns for the login prompts. + +``prompt_re`` + Patterns for the command prompt. ``execute()`` waits until one of + them matches at the end of the received data. + +``error_re`` + Patterns for lines in a response that mean that the command failed; + ``execute()`` then raises + :class:`~Exscript.protocols.exception.InvalidCommandException`. + +``login_error_re`` + Patterns for messages that mean that the login failed. + +``reconnect_between_auth_methods`` + ``True`` for devices that close the SSH connection after an + authentication method failed. + +``netconf_capabilities_re``, ``netconf_device_params`` + Patterns for the NETCONF capabilities of the system, and the + parameters of the ncclient device handler (:doc:`netconf`). + +And it can override these methods: + +``check_protocol_for_os()``, ``check_head_for_os()``, ``check_response_for_os()`` + Return the confidence for the given text, see above. Return 90 or + more only for text that only this system shows, such as a vendor + name, and less than 50 for signs that other systems may show, too. + +``init_terminal(conn)`` + Prepares the session for scripts, for example by turning off the + pager. ``conn.autoinit()`` calls it; :func:`Exscript.util.start.start` + and the queue call ``autoinit()`` after the login. + +``auto_authorize(conn, account, flush, bailout)`` + Gets the privileges that the commands need, for example with + ``enable`` on IOS. ``login()`` calls it through + ``conn.auto_app_authorize()`` after the authentication. + +``logout(conn)`` + Ends the session; ``close()`` calls it. The default sends ``exit``. + +``clean_response_for_re_match(response)`` + Removes what would keep the prompt patterns from matching, such as + terminal control sequences that the device sends around the prompt. + +The built-in drivers in ``src/Exscript/protocols/drivers`` are examples for +all of these. To contribute a driver, see :doc:`contributing`. diff --git a/docs/exscript.rst b/docs/exscript.rst index 4a18e92c..4a940e85 100644 --- a/docs/exscript.rst +++ b/docs/exscript.rst @@ -10,101 +10,11 @@ supports logging, authentication mechanisms such as TACACS and takes care of synchronizing the login procedure between multiple running connections. -These features are enabled using simple command line options. The -following options are currently provided: - -:: - - Options: - --version show program's version number and exit - -h, --help show this help message and exit - --account-pool=FILE Reads the user/password combination from the given - file instead of prompting on the command line. The - file may also contain more than one user/password - combination, in which case the accounts are used round - robin. - -c NUM, --connections=NUM - Maximum number of concurrent connections. NUM is a - number between 1 and 20, default is 1. - --csv-hosts=FILE Loads a list of hostnames and definitions from the - given file. The first line of the file must have the - column headers in the following syntax: "hostname - [variable] [variable] ...", where the fields are - separated by tabs, "hostname" is the keyword - "hostname" and "variable" is a unique name under which - the column is accessed in the script. The following - lines contain the hostname in the first column, and - the values of the variables in the following columns. - -d PAIR, --define=PAIR - Defines a variable that is passed to the script. PAIR - has the following syntax: STRING=STRING. - --default-domain=STRING - The IP domain name that is used if a given hostname - has no domain appended. - --delete-logs Delete logs of successful operations when done. - -e EXSCRIPT, --execute=EXSCRIPT - Interprets the given string as the script. - --hosts=FILE Loads a list of hostnames from the given file (one - host per line). - --insecure Allow the template to call the functions that run - local commands or access local files: sys.exec, - sys.env and file.* (except file.exists). - -i, --non-interactive - Do not ask for a username or password. - -l DIR, --logdir=DIR Logs any communication into the directory with the - given name. Each filename consists of the hostname - with "_log" appended. Errors are written to a separate - file, where the filename consists of the hostname with - ".log.error" appended. - --no-echo Turns off the echo, such that the network activity is - no longer written to stdout. This is already the - default behavior if the -c option was given with a - number greater than 1. - -n, --no-authentication - When given, the authentication procedure is skipped. - Implies -i. - --no-auto-logout Do not attempt to execute the exit or quit command at - the end of a script. - --no-prompt Do not wait for a prompt anywhere. Note that this will - also cause Exscript to disable commands that require a - prompt, such as "extract". - --no-initial-prompt Do not wait for a prompt after sending the password. - --no-strip Do not strip the first line of each response. - --overwrite-logs Instructs Exscript to overwrite existing logfiles. The - default is to append the output if a log already - exists. - -p STRING, --protocol=STRING - Specify which protocol to use to connect to the remote - host. Allowed values for STRING include: dummy, - pseudo, ssh, ssh1, ssh2, telnet. The default protocol - is telnet. - --retry=NUM Defines the number of retries per host on failure. - Default is 0. - --retry-login=NUM Defines the number of retries per host on login - failure. Default is 0. - --sleep=TIME Waits for the specified time before running the - script. TIME is a number of seconds, optionally with - the suffix m, h or d, like for the 'sleep' Unix - command; e.g. "90" or "1h 30m". - --ssh-auto-verify Disables the verification of SSH host keys, so that - any key is accepted. Highly insecure and not - recommended. - --ssh-host-key-policy={reject,warn,auto-add} - What to do with SSH hosts that are not in - ~/.ssh/known_hosts: reject the connection (the - default), warn and connect, or add the host key to - known_hosts and connect. - --ssh-key=FILE Specify a key file that is passed to the SSH client. - This is equivalent to using the "-i" parameter of the - openssh command line client. - -v NUM, --verbose=NUM - Print out debug information about the network - activity. NUM is a number between 0 (min) and 5 (max). - Default is 1. - -V NUM, --parser-verbose=NUM - Print out debug information about the Exscript - template parser. NUM is a number between 0 (min) and 5 - (max). Default is 0. +These features are enabled using command line options. ``exscript --help`` +shows them: + +.. literalinclude:: _generated/exscript-help.txt + :language: text Using Account Pooling --------------------- @@ -123,20 +33,21 @@ use the feature: other_user=another_password somebody=yet_another_password - Note that the password needs to be base64 encrypted, just putting - plain passwords there will NOT work. + The passwords are base64 encoded; plain passwords do not work. Base64 + does not protect the passwords: anyone who can read the file can + decode them. #. Save the file. It is assumed that you are aware of the security implications of saving your login passwords in a text file. -#. Start Exscript with the ``–account-pool FILE`` option. For example:: +#. Start Exscript with the ``--account-pool FILE`` option. For example:: exscript --account-pool /home/user/my_accounts my.exscript host4 Using a CSV file as input ------------------------- -By providing the –csv-hosts option you may pass a list of hosts to +By providing the ``--csv-hosts`` option you may pass a list of hosts to Exscript while at the same time providing a number of variables to the script. The CSV file should have the following format:: diff --git a/docs/index.rst b/docs/index.rst index ce05410d..edd0fbfb 100755 --- a/docs/index.rst +++ b/docs/index.rst @@ -8,7 +8,7 @@ :target: https://pypi.org/project/Exscript/ .. image:: https://img.shields.io/github/stars/knipknap/exscript.svg - :target: https://github.com/knipknap/exscript/stargazers + :target: https://github.com/knipknap/exscript .. image:: https://img.shields.io/github/license/knipknap/exscript.svg :target: https://github.com/knipknap/exscript/blob/master/COPYING @@ -17,8 +17,8 @@ What is Exscript? ================= Exscript is a Python module and a template processor for automating network -connections over protocols such as Telnet or SSH. We attempt to create the -best possible set of tools for working with Telnet and SSH. +connections over SSH, Telnet and NETCONF. We attempt to create the best +possible set of tools for working with network devices over these protocols. Exscript also provides a set of tools and functions for sysadmins, that simplify **regular expression matching**, **reporting** by email, **logging**, @@ -36,6 +36,7 @@ Using Exscript with Python Make sure to check out the :doc:`python_tutorial`. You may also want to look at the `Python examples `_. +If you upgrade from Exscript 2.x, read :doc:`migration`. Using the Exscript command line tool ------------------------------------ @@ -49,19 +50,22 @@ Main design goals - Exscript provides **high reliability** and **scalability**. Exscript is used by some of the world's largest ISPs to maintain hundreds of thousands of sessions every day. -- Exscript is **extremely well tested**. The Exscript public API has almost - 100% test coverage. +- Exscript is **well tested**: unit tests, replays of sessions recorded + from devices, and benchmarks run on Linux, macOS and Windows. - Exscript is **protocol agnostic**, so if you are migrating from Telnet to SSH later, you can switch easily by simply changing an import statement. Development ----------- -Exscript is on `GitHub `_. +Exscript is on `GitHub `_; see +:doc:`contributing`. License ------- Exscript is published under the `MIT licence `_. +``Exscript.protocols.telnetlib`` is derived from CPython and published under the +`PSF License 2.0 `_. Contents -------- @@ -74,6 +78,12 @@ Contents cli_tutorial exscript templates + drivers + netconf + security + platforms trouble + migration api changelog + contributing diff --git a/docs/migration.rst b/docs/migration.rst new file mode 100644 index 00000000..a054dc78 --- /dev/null +++ b/docs/migration.rst @@ -0,0 +1,191 @@ +Migrating from 2.x to 3.0 +========================= + +Exscript 3.0 keeps the Python API and the template language. It changes +defaults that were insecure or unreliable, and removes outdated code. +Most scripts run unchanged once they pass the checks in the first +section. The :doc:`changelog` describes every change. + +First checks +------------ + +These changes affect most scripts. + +Python 3.12 or later + Exscript 3.0 runs on Python 3.12 to 3.15. Stay with Exscript 2.x on + older versions. + +SSH is the default protocol + Hosts without a protocol, e.g. ``router1`` instead of + ``telnet://router1``, are connected with SSH. For Telnet devices, + write ``telnet://router1``, or pass ``default_protocol='telnet'`` to + the function that reads the hosts, e.g. + :func:`~Exscript.util.file.get_hosts_from_file`, + :class:`~Exscript.Host` or :func:`~Exscript.protocols.connect`. The + command line tool has ``--protocol telnet``. + +SSH host keys are verified + Exscript connects only to SSH hosts whose key is in + ``~/.ssh/known_hosts``. Add your devices, for example with + ``ssh-keyscan``, or pass ``host_key_policy='auto-add'`` (command line: + ``--ssh-host-key-policy auto-add``). See :doc:`security`. + +Templates run in secure mode + Templates that call ``sys.exec``, ``sys.env`` or ``file.*`` (except + ``file.exists``) fail unless you pass ``secure=False`` to + :func:`Exscript.util.template.eval` and the other functions of that + module, or the option ``--insecure`` to the command line tool. + ``sys.exec`` runs the command without a shell and returns text; for + pipes and redirections, use ``sys.exec("sh -c '...'")``. + +``close()`` logs out + ``close()`` sends the logout command of the driver, ``exit`` on most + devices, and closes the connection. In 2.x, it waited until the host + closed the connection, so scripts sent ``exit`` themselves; they still + work. ``close(force=True)`` closes at once, as before. + +Connections +----------- + +Terminal size + Sessions get a terminal of 511 columns and 1000 lines, instead of the + size of the terminal that runs the script, or 80×25 without one. + Devices whose driver does not set the width themselves, e.g. Linux + hosts, no longer wrap lines at 80 characters. For the old size, pass + ``term_width=80, term_height=25``. + +Closed connections + If the host closes the connection while Exscript waits for a + response, all protocols raise + :class:`~Exscript.protocols.exception.ConnectionClosedException`, + which is a ``ProtocolException`` and an ``EOFError``. Only code that + expected a ``TimeoutException`` in this case must change. + +Error messages + :class:`~Exscript.protocols.exception.InvalidCommandException` + contains only the line with the error. The full response is still in + ``conn.response``. + +Responses + ``conn.get_response()`` returns the response without the echo of the + command, control characters and terminal escape sequences, and with + ``\n`` line ends. ``conn.response`` is unchanged. + +Generic prompt + A match of the prompt pattern of the generic driver, which the + ``adtran``, ``aix`` and ``shell`` drivers use, too, counts only if no + more data arrives within 20 ms. + +OS detection + Signals that several systems show no longer select a driver, so some + devices are guessed differently, e.g. as ``unknown`` instead of a wrong + driver. Pass ``driver=`` if a device needs a specific driver. See + :doc:`drivers`. + +Templates +--------- + +``__response__`` + The functions of the ``connection`` module clean the response like + ``get_response()``. + +``connection.wait_for()`` + No longer consumes the response and the prompt; use + ``connection.expect()`` for that. + +``connection.send()``, ``connection.sendline()`` + Clear ``__response__``. + +``connection.close()`` + Logs out first; ``connection.close(true)`` closes at once. + +Command line tool +----------------- + +Besides the defaults above: a value entered at a ``-d VAR={#"PROMPT"}`` +prompt is parsed as a Python literal, such as a number or a list, instead +of being run as Python code, and ``--sleep`` accepts seconds with an +optional suffix ``m``, ``h`` or ``d`` instead of running ``sleep`` in a +shell. + +Python API +---------- + +Attributes that hold the internal state of the protocol adapters are +private. Use these instead: + +.. list-table:: + :header-rows: 1 + + * - 2.x + - 3.0 + * - ``conn.buffer`` + - ``conn.response``, ``conn.get_response()``, ``conn.expect()``, + ``conn.add_monitor()`` + * - ``conn.auto_driver``, ``conn.manual_driver`` + - ``conn.guess_os()``, ``conn.get_driver()``, ``conn.set_driver()`` + * - ``conn.manual_prompt_re``, ``manual_error_re``, + ``manual_login_error_re``, ``manual_user_re``, + ``manual_password_re`` + - ``get_prompt()``/``set_prompt()``, ``get_error_prompt()``/``set_error_prompt()``, + ``get_login_error_prompt()``/``set_login_error_prompt()``, + ``get_username_prompt()``/``set_username_prompt()``, + ``get_password_prompt()``/``set_password_prompt()`` + * - ``conn.proto_authenticated``, ``app_authenticated``, + ``app_authorized`` + - ``is_protocol_authenticated()``, ``is_app_authenticated()``, + ``is_app_authorized()`` + * - ``conn.cancel`` + - ``conn.cancel_expect()`` + * - ``conn.log`` + - ``conn.logfile``, the path of the log file; setting it switches + to another file + * - ``SSH2.client``, ``shell``, ``sock``, ``Telnet.tn``, + ``conn.last_account`` + - none; they are internal + +Other renamed or removed names: + +- ``Exscript.protocols.telnetlib`` is internal and contains only what the + Telnet adapter needs. Use :class:`Exscript.protocols.Telnet`, or the + package ``standard-telnetlib`` as a replacement for the telnetlib module + that Python 3.13 removed. +- ``OsGuesser.data_received()`` and ``protocol_info()`` are + ``check_received_data()`` and ``check_protocol_info()``. +- ``Exscript.util.template.test_secure()`` is ``test(secure=True)``. +- ``Exscript.util.interact.InputHistory.file`` is ``filename``. +- ``Exscript.util.daemonize``, ``Exscript.util.pidutil`` and + ``Exscript.util.sigintcatcher`` are removed. Run long-lived scripts + under a service manager such as systemd. +- ``WorkQueue`` has no ``debug`` argument, and ``set_debug()`` takes + ``level``. Pass the other arguments by keyword. +- ``parselib.Lexer.expect()`` and ``current_is()`` take ``types`` + instead of ``type``. +- The ``dump1()`` methods of the template interpreter are removed. +- Adapters that subclass ``Protocol`` must pass its arguments by + keyword; new ones were inserted. + +Emulators and test servers +-------------------------- + +- ``CommandSet.add()`` calls its second argument ``handler`` instead of + ``response``, and ``VirtualDevice.add_commands_from_file()`` calls + ``autoprompt`` ``prompt``. +- ``VirtualDevice`` no longer echoes the password, so ``response`` after + a login to it is empty. +- The test servers ``SSHd`` and ``Telnetd`` run as threads; stop them with + ``exit()`` and ``join()``. Their ``buf`` attribute and the ``encoding`` + argument of ``Telnetd`` are removed. + +From the 2.9 development versions +--------------------------------- + +The 2.9 versions were not released on PyPI. Besides the changes above, +their users should know: + +- The OpenTelemetry instrumentation records commands and responses only + with ``instrument(capture_payload=True)``. Its spans are children of + the current span, and their attributes and the duration histogram have + new names; see the :doc:`changelog`. +- ``Exscript.protocols.telnetlib`` lost the methods that the Telnet + adapter does not use, e.g. ``read_until()`` and ``expect()``. diff --git a/docs/netconf.rst b/docs/netconf.rst new file mode 100644 index 00000000..faac83b0 --- /dev/null +++ b/docs/netconf.rst @@ -0,0 +1,112 @@ +NETCONF +======= + +The :class:`~Exscript.protocols.Netconf` adapter speaks NETCONF over SSH +(:rfc:`6241`, :rfc:`6242`). It is based on `ncclient +`_, which is installed with +Exscript. You send RPCs as XML text, and get the replies as text or as +`lxml `_ elements. + +Connecting +---------- + +``Netconf`` is an SSH adapter, so it takes the same arguments as +:class:`~Exscript.protocols.SSH2`, and checks host keys the same way +(:doc:`security`). The default port is 830:: + + from Exscript import Account + from Exscript.protocols import Netconf + + conn = Netconf() + conn.connect('router1') + conn.login(Account('admin', 'secret')) + print(conn.get_server_capabilities()) + +In host lists and with :func:`Exscript.util.start.start`, use the scheme +``netconf://``, for example ``netconf://router1``. + +``login()`` authenticates with SSH and exchanges the ```` messages; +there is no login prompt. ``get_server_capabilities()``, +``get_client_capabilities()`` and ``get_session_id()`` return what was +exchanged. + +Sending RPCs +------------ + +``execute()`` sends the content of an ```` element and waits for the +reply; Exscript adds the ```` element with its ``message-id``. It +raises :class:`~Exscript.util.xml.XMLValidationError` before sending if +the content is not well-formed XML:: + + conn.execute('') + print(conn.get_response()) # the as text + + reply = conn.get_response_element() # the as lxml element + ns = {'ks': 'urn:ietf:params:xml:ns:yang:ietf-keystore'} + for name in reply.xpath('//ks:asymmetric-key/ks:name/text()', namespaces=ns): + print(name) + +Filters work as with any NETCONF client:: + + conn.execute(''' + + + + + + + ''') + +``close()`` sends ```` and closes the connection. + +Errors +------ + +If the reply contains an ```` with the severity ``error``, +``execute()`` raises :class:`~Exscript.protocols.exception.RPCError`. +Its attributes hold the first error, and ``errlist`` holds all errors of +the reply. Replies with only warnings do not raise:: + + from Exscript.protocols.exception import RPCError + + conn.execute('') + try: + conn.execute('') + except RPCError as error: + print(error.tag, error.message) # operation-failed Requested resource already locked + for e in error.errlist: + print(e.type, e.tag, e.severity, e.path, e.message) + conn.execute('') + +``RPCError`` is an +:class:`~Exscript.protocols.exception.InvalidCommandException`, so code +that handles failed commands of any protocol also handles it. + +Large replies +------------- + +lxml limits the size and depth of the XML documents that it parses. For +replies beyond these limits, for example the full configuration of a large +device, set ``conn.huge_tree = True``. This also removes the protection +against documents that are built to exhaust the memory, so enable it only +for devices that you trust. + +Drivers +------- + +Exscript recognizes IOS XR and Junos devices by the capabilities in their +```` (:doc:`drivers`). The driver provides the patterns that detect +the end of a reply and errors in it, and the ncclient device handler: its +``netconf_device_params`` attribute is passed to ncclient's +``make_device_handler()``. The built-in drivers use ncclient's default +handler. Because the device handler is chosen when the session starts, +before the capabilities are known, pass a driver that sets +``netconf_device_params`` with ``driver=``. + +Interactive sessions +-------------------- + +``interact()`` does not work with NETCONF. For an interactive session, +use OpenSSH:: + + ssh -s -p 830 router1 netconf diff --git a/docs/platforms.rst b/docs/platforms.rst new file mode 100644 index 00000000..f175a55e --- /dev/null +++ b/docs/platforms.rst @@ -0,0 +1,51 @@ +Platforms +========= + +Exscript runs on Linux, macOS and Windows with Python 3.12 or later. CI +runs the tests with every supported Python version on Linux, with the +latest one on macOS and Windows, and with the free-threaded build of +Python 3.14. + +Most of Exscript works the same on every platform: for example, it reads +and writes text files such as templates, host lists and logs as UTF-8 +everywhere, and gives sessions the same terminal size. These are the +differences. + +Windows +------- + +- ``interact()`` gets the keyboard input line by line, because the + Windows console passes it to Python only after Enter. Arrow keys, Tab + completion and control keys therefore do not reach the device, and a + typed line may appear twice, because both the console and the device + echo it. +- On Linux and macOS, the command line tool runs the connections in a + child process, so that Ctrl+C stops all of them at once. Windows has no + ``fork()``, so there Ctrl+C may take effect only after the running + connections end. +- The template function ``sys.exec`` passes the command line to Windows + without a shell. Commands that are built into the shell need ``cmd /c``, + for example ``sys.exec("cmd /c dir")``. +- ``pseudo://`` URLs accept Windows paths, such as + ``pseudo://C:\devices\router.py``. +- The default known_hosts file is ``%USERPROFILE%\.ssh\known_hosts`` + (:doc:`security`). + +The queue's process mode +------------------------ + +``Queue(mode='multiprocessing')`` needs the ``fork`` start method of +:mod:`multiprocessing`, which is the default only on Linux with Python +3.12 and 3.13. On other platforms and versions, use the default thread +mode, which is also what :func:`Exscript.util.start.start` uses. The +process mode warns that it is not recommended. + +Free-threaded Python +-------------------- + +Exscript works on the free-threaded build of Python 3.14 (``python3.14t``), +where threads, and so connections in a queue, run in parallel. NETCONF +uses lxml, which does not declare support for free threading yet: when +the first NETCONF session starts, Python enables the global interpreter +lock and warns, unless it runs with ``PYTHON_GIL=0``. Exscript loads lxml +only for NETCONF, so other protocols keep running without it. diff --git a/docs/python_tutorial.rst b/docs/python_tutorial.rst index f34e5d83..841a7ae7 100644 --- a/docs/python_tutorial.rst +++ b/docs/python_tutorial.rst @@ -29,29 +29,35 @@ Create a file named ``start.py`` with the following content:: from Exscript.util.interact import read_login from Exscript.protocols import SSH2 - account = read_login() # Prompt the user for his name and password + account = read_login() # Prompt the user for their name and password conn = SSH2() # We choose to use SSH2 conn.connect('localhost') # Open the SSH connection conn.login(account) # Authenticate on the remote host conn.execute('uname -a') # Execute the "uname -a" command - conn.send('exit\r') # Send the "exit" command - conn.close() # Wait for the connection to close + print(conn.get_response()) # Print the output of the command + conn.close() # Log out and close the connection Awesome fact: Just replace ``SSH2`` by ``Telnet`` and it should still work with Telnet devices. As you can see, we prompt the user for a username and a password, and connect to ``localhost`` using the entered login details. Once logged -in, we execute ``uname -a``, log out, and make sure to wait until the -remote host has closed the connection. +in, we execute ``uname -a``, print its output, log out, and close the +connection. -You can see one important difference: We used ``conn.execute`` to run -``uname -a``, but we used ``conn.send`` to execute the ``exit`` command. -The reason is that *``conn.execute`` waits until the server has -acknowledged that the command has completed*, while ``conn.send`` does -not. Since the server won't acknowledge the ``exit`` command (instead, -it just closes the connection), using ``conn.execute`` would lead to an -error. +``conn.execute()`` sends the command and waits until the remote host +shows its prompt again, which means that the command has completed. +``conn.send()`` sends text without waiting. ``conn.close()`` logs out +with the command that the device expects, ``exit`` on most devices, +before it closes the connection; ``conn.close(force=True)`` closes it +right away. + +.. note:: + + Exscript connects only to SSH hosts whose key it knows: like OpenSSH, + it checks the key against ``~/.ssh/known_hosts`` and rejects unknown + hosts. Connect once with ``ssh localhost`` and accept the key, or see + :doc:`security` for the other options. Making it easier ---------------- @@ -202,16 +208,24 @@ seen in the API documentation for Logging ------- -Exscript has built-in support for logging. In a simple case, just pass -the ``stdout`` and ``stderr`` parameters for log and errors to -``start()`` or ``quickstart()`` and you are done:: +Exscript has built-in support for logging. In a simple case, pass a +file as ``stdout`` to ``start()`` or ``quickstart()``; Exscript writes +the conversations with all hosts into it:: - with open('log.txt','w+') as fp: + with open('log.txt', 'w') as fp: start(accounts, hosts, do_something, stdout=fp) -Exscript creates one logfile per device. In the case that an error -happened on the remote device, it creates an additional file that -contains the error (including Python's traceback). +For one log file per host, decorate the function with +:func:`Exscript.util.log.log_to_file`. Exscript then writes the +conversation with each host into ``logs/.log``, and if an error +happened, the error with Python's traceback into +``logs/.log.error``:: + + from Exscript.util.log import log_to_file + + @log_to_file('logs') + def do_something(job, host, conn): + conn.execute('uname -a') Interacting with a device ------------------------- @@ -227,10 +241,10 @@ an example:: def do_something(job, host, conn): conn.execute('uname -a') - print "The response was", repr(conn.response) + print("The response was", repr(conn.get_response())) os, hostname = first_match(conn, r'^(\S+)\s+(\S+)') - print "The hostname is:", hostname - print "Operating system:", os + print("The hostname is:", hostname) + print("Operating system:", os) quickstart('ssh://localhost', do_something) @@ -243,9 +257,9 @@ containing the same number of elements as there are groups (bracket expressions) in the regular expression. This is more terse than the following typical regular idiom:: - match = re.match(r'^(\S+)\s+(\S+)', conn.response) + match = re.match(r'^(\S+)\s+(\S+)', conn.get_response()) if match: - print match.group(1) + print(match.group(1)) Similarly, the following use of :func:`Exscript.util.match.any_match` can never fail:: @@ -256,8 +270,8 @@ can never fail:: def do_something(job, host, conn): conn.execute('ls -l') for permissions, filename in any_match(conn, r'^(\S+).*\s+(\S+)$'): - print "The filename is:", filename - print "The permissions are:", permissions + print("The filename is:", filename) + print("The permissions are:", permissions) quickstart('ssh://localhost', do_something) diff --git a/docs/security.rst b/docs/security.rst new file mode 100644 index 00000000..179ddea5 --- /dev/null +++ b/docs/security.rst @@ -0,0 +1,119 @@ +Security +======== + +Exscript logs in to network devices, often with privileged accounts, so +its defaults are strict. This page explains them, and how to change them +where you need to. + +To report a vulnerability, see the `security policy +`_. + +SSH host keys +------------- + +Before Exscript sends any credentials over SSH or NETCONF, it checks the +host key of the device against ``~/.ssh/known_hosts``, like OpenSSH. On +Windows, the file is ``%USERPROFILE%\.ssh\known_hosts``. Entries for other +ports than 22 have the form ``[host]:port``, and hashed entries work, too. + +A key that differs from the known key always raises +:class:`paramiko.BadHostKeyException`: either the device was replaced, or +someone intercepts the connection. For a host without a known key, the +``host_key_policy`` option decides: + +``'reject'`` (default) + Raise :class:`paramiko.SSHException`. + +``'warn'`` + Log a warning and accept the key. + +``'auto-add'`` + Accept the key and append it to the known_hosts file, like OpenSSH's + ``StrictHostKeyChecking=accept-new``. Later connections detect a + changed key. + +The best protection is to add the keys before the first connection, and +to compare their fingerprints with the ones that the devices show:: + + ssh-keyscan router1 router2 >> ~/.ssh/known_hosts + +The options are arguments of the protocol adapters, ``Host`` options, and +command line options:: + + from Exscript import Host + from Exscript.protocols import SSH2 + + conn = SSH2(host_key_policy='auto-add', known_hosts='/srv/automation/known_hosts') + + host = Host('ssh://router1') + host.set_option('host_key_policy', 'auto-add') + +.. code-block:: text + + exscript --ssh-host-key-policy auto-add my.exscript router1 + +``verify_fingerprint=False`` (command line: ``--ssh-auto-verify``) turns +the check off completely. Anyone who can intercept the connection can +then read the credentials, so use it only in isolated labs. + +Telnet +------ + +Telnet sends passwords, commands and responses in clear text. The default +protocol is SSH; use Telnet only for devices that support nothing else, +and only in a trusted management network. + +Templates +--------- + +Templates can come from many authors, so by default they cannot run local +commands or access local files. :func:`Exscript.util.template.eval`, +``eval_file()``, ``paste()``, ``paste_file()``, ``test()`` and +``test_file()`` forbid the template functions ``sys.exec``, ``sys.env``, +and ``file.*`` except ``file.exists``. A template that calls one of them +raises :class:`Exscript.interpreter.exception.PermissionError` when it is +compiled, before anything is sent. Functions that your Python code passes +to the template as variables are always allowed. + +For templates that you trust, pass ``secure=False``, or use the command +line option ``--insecure``. ``sys.exec`` runs the command without a shell, +so template variables in the command cannot inject shell commands. + +Credentials +----------- + +Exscript keeps passwords out of its output: debug logs show them as +````, ``repr()`` of an :class:`~Exscript.Account` shows only the +name, and :class:`~Exscript.protocols.exception.InvalidCommandException` +contains only the line with the error, not the whole response. + +Where the passwords come from is up to you. In order of preference: + +- SSH keys: ``Account('admin', key=PrivateKey.from_file('/home/admin/.ssh/id_ed25519'))``. +- A secret store or the environment of a service, read at run time. +- A prompt: :func:`Exscript.util.interact.read_login`. +- An account pool file (:func:`Exscript.util.file.get_accounts_from_file`, + command line: ``--account-pool``). Its passwords are only base64 + encoded, which protects nothing: make the file readable only for you. + +Logs and recordings +------------------- + +Logs contain everything that the devices sent, which may include secrets, +for example in a ``show running-config``. Store them like the +configurations themselves. Before you share a log in an issue, remove +passwords, keys, host names and addresses. + +The OpenTelemetry instrumentation (``Exscript[otel]``) records commands +and responses only with ``instrument(capture_payload=True)``. It then +truncates them to ``max_payload_length`` characters and replaces +passwords with ````. + +XML +--- + +Exscript parses NETCONF replies without resolving entities, loading DTDs, +or accessing the network. lxml additionally limits the size and depth of +documents; ``conn.huge_tree = True`` lifts these limits for very large +replies, but also the protection against documents that are built to +exhaust the memory. Enable it only for devices that you trust. diff --git a/docs/templates.rst b/docs/templates.rst index 92b2ec0a..96415f56 100644 --- a/docs/templates.rst +++ b/docs/templates.rst @@ -395,7 +395,11 @@ For example, the following function instructs Exscript to wait for {sys.wait(10)} The supported functions are documented in the API reference of -:doc:`Exscript.stdlib <_autosummary/Exscript.stdlib>`. +:doc:`Exscript.stdlib <_autosummary/Exscript.stdlib>`. The functions +that run local commands or access local files, ``sys.exec``, +``sys.env`` and ``file.*`` except ``file.exists``, work only if the +template runs with ``secure=False`` or the command line option +``--insecure`` (:doc:`security`). Exiting A Script ~~~~~~~~~~~~~~~~ @@ -471,3 +475,39 @@ example, the following will also work:: show running-config {end} show ip int brief + +Templates And Jinja2 +-------------------- + +An Exscript template is a program that runs during the session: it sends +a line, reads the response, and can react to it with ``extract``, ``if`` +or ``loop``. `Jinja2 `_ renders text, +which suits generating commands or configurations from inventory data. +The two combine well: render the commands with Jinja2, then send them +with Exscript:: + + from jinja2.sandbox import SandboxedEnvironment + + commands = SandboxedEnvironment().from_string(""" + configure terminal + {% for vlan in vlans %} + vlan {{ vlan.id }} + name {{ vlan.name }} + {% endfor %} + end + """).render(vlans=[{"id": 10, "name": "users"}, {"id": 20, "name": "voice"}]) + + def configure(job, host, conn): + for line in commands.splitlines(): + if line.strip(): + conn.execute(line) + +``execute()`` waits for the prompt after each line, and raises +:class:`~Exscript.protocols.exception.InvalidCommandException` if the +device reports an error. The sandboxed environment keeps Jinja2 templates +from inventories or users from running Python code. + +To use Exscript template commands in the rendered text, run it with +:func:`Exscript.util.template.eval` instead. Exscript then interprets +``$``, ``{``, ``}`` and ``\`` in it, so escape them with a backslash in +the values that you insert, e.g. ``\$``. diff --git a/docs/trouble.rst b/docs/trouble.rst index 0044610d..80590fe3 100644 --- a/docs/trouble.rst +++ b/docs/trouble.rst @@ -20,9 +20,30 @@ Generally, the following kinds of errors that may happen at runtime: The following sections explain when these problems may happen and how to fix them. +Seeing What Happens +------------------- + +To see the conversation with the device, pass ``logfile='session.log'`` +to the protocol adapter, or use the command line option ``--logdir``. To +also see what Exscript waits for and which driver it chose, connect with +``debug=5`` after enabling Python's logging:: + + import logging + + logging.basicConfig(level=logging.DEBUG) + conn = SSH2(debug=5) + +The command line tool has ``--protocol-verbose 5`` for the same. + Deadlocks --------- +When Exscript misses a prompt, it waits until the timeout, 30 seconds by +default (``timeout=``), and raises +:class:`~Exscript.protocols.exception.TimeoutException`. A frequent cause +is a wrong driver: check ``conn.guess_os()``, and set the driver if it is +wrong (:doc:`drivers`). + Exscript tries to automatically detect a prompt, so generally you should not have to worry about prompt recognition. The following prompt types are supported: From 2c1ad85d97eee259dffa6f531449ccda99130207 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 3 Nov 2026 19:13:25 +0100 Subject: [PATCH 054/250] Check the documentation links weekly External links break without a change in the repository, so a scheduled workflow checks them instead of each pull request. --- .github/workflows/linkcheck.yml | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 .github/workflows/linkcheck.yml diff --git a/.github/workflows/linkcheck.yml b/.github/workflows/linkcheck.yml new file mode 100644 index 00000000..c4ed514c --- /dev/null +++ b/.github/workflows/linkcheck.yml @@ -0,0 +1,28 @@ +name: Link check + +# External links break without a change in this repository, so they are +# checked weekly instead of on every pull request. +on: + schedule: + - cron: "17 5 * * 1" # Mondays + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + linkcheck: + name: Documentation links + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + fetch-depth: 0 + - uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3 + - name: Check the links of the documentation + run: uv run --frozen --python 3.14 --group docs sphinx-build -b linkcheck docs docs/_build/linkcheck From 73e2ff35da1e62722cf5fa989177c343fbea4247 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 3 Nov 2026 19:43:31 +0100 Subject: [PATCH 055/250] Let the test servers listen in start() and choose free ports The server thread bound its port only after start() returned, so the tests had to sleep, and bind errors stayed in the thread. With port 0, a server takes a free port, so test runs do not collide. --- CHANGES.md | 4 +++ src/Exscript/servers/server.py | 34 +++++++++++++++++------- tests/Exscript/protocols/ProtocolTest.py | 4 +-- tests/Exscript/servers/ServerTest.py | 7 ++--- 4 files changed, 32 insertions(+), 17 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 41a29cea..c95e49ec 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -374,3 +374,7 @@ The names below refer to functions callable from templates, not protocol methods - **Client Handling:** `SSHd` and `Telnetd` keep listening after a client disconnects, allowing sequential connections. They share the supplied emulator device; concurrent clients are not supported. +- **Listening in `start()`:** `start()` of `SSHd` and `Telnetd` listens before it + returns, so clients can connect at once, and raises errors such as a port in use to + the caller. Before, the thread bound the port a moment later. Port 0 chooses a free + port, which the `port` attribute has after `start()`. diff --git a/src/Exscript/servers/server.py b/src/Exscript/servers/server.py index 5bc3f886..6898e189 100644 --- a/src/Exscript/servers/server.py +++ b/src/Exscript/servers/server.py @@ -34,10 +34,8 @@ _logger.addHandler(logging.NullHandler()) -# This formerly was derived from multiprocessing.Process but as POSIX Python -# does not support child processes in threaded environments/apps, we had to -# switch to threaded servers too. This will hopefully add more stability to the -# test suite. +# A thread, not a process: on POSIX, forking a program that runs threads may +# deadlock the child. class Server(threading.Thread): """ Base class of the Telnet and SSH servers. Servers are intended to be @@ -60,7 +58,8 @@ def __init__(self, host, port, device, encoding='utf8'): :type host: str :param host: The address against which the daemon binds. :type port: int - :param port: The TCP port on which to listen. + :param port: The TCP port on which to listen; 0 lets start() choose a + free one. :type device: VirtualDevice :param device: A virtual device instance. :type encoding: str @@ -109,12 +108,7 @@ def _get_server_type(self): def _run(self): self._dbg(1, 'Starting the %s server.' % self._get_server_type()) - self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - self.socket.bind((self.host, self.port)) - self.socket.listen(1) self.running = True - self._dbg(1, 'Server bound to: %s:%s' % (self.host, self.port)) try: while self.running: @@ -142,6 +136,26 @@ def _run(self): self.processes = [] self.socket.close() + def start(self): + """ + Starts listening, and then the thread that accepts the connections. + Clients can connect as soon as this returns. With port 0, the + operating system chooses a free port; the port attribute has it then. + """ + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + try: + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + sock.bind((self.host, self.port)) + sock.listen(1) + except OSError: + sock.close() + raise + self.socket = sock + self.port = sock.getsockname()[1] + self.name = '%s-%s:%s' % (self._get_server_type(), self.host, self.port) + self._dbg(1, 'Server bound to: %s:%s' % (self.host, self.port)) + threading.Thread.start(self) + def exit(self): """ Stop the daemon without waiting for the thread to terminate. diff --git a/tests/Exscript/protocols/ProtocolTest.py b/tests/Exscript/protocols/ProtocolTest.py index 091f3183..3d9f18fe 100644 --- a/tests/Exscript/protocols/ProtocolTest.py +++ b/tests/Exscript/protocols/ProtocolTest.py @@ -30,7 +30,7 @@ class ProtocolTest(unittest.TestCase): def setUp(self): self.hostname = '127.0.0.1' - self.port = 1236 + self.port = 0 # the daemon chooses a free port self.user = 'user' self.password = 'password' self.account = Account(self.user, password=self.password) @@ -40,7 +40,7 @@ def setUp(self): self.createDaemon() if self.daemon is not None: self.daemon.start() - time.sleep(0.2) + self.port = self.daemon.port # LIFO cleanup order! self.addCleanup(self.daemon.join) self.addCleanup(self.daemon.exit) diff --git a/tests/Exscript/servers/ServerTest.py b/tests/Exscript/servers/ServerTest.py index fcfa9d53..483b3b72 100644 --- a/tests/Exscript/servers/ServerTest.py +++ b/tests/Exscript/servers/ServerTest.py @@ -1,5 +1,4 @@ import re -import time import unittest from Exscript import Account @@ -12,7 +11,7 @@ class ServerTest(unittest.TestCase): def setUp(self): self.host = 'localhost' - self.port = 1235 + self.port = 0 # start() chooses a free port self.device = VirtualDevice(self.host, echo=False) self.daemon = Server(self.host, self.port, self.device) self.device.set_prompt(self.host + ':' + str(self.port) + '> ') @@ -41,7 +40,6 @@ def testConstructor(self): return self._create_daemon() self.daemon.start() - time.sleep(1) def testStart(self): # Test can not work on the abstract base. @@ -50,11 +48,10 @@ def testStart(self): self._create_daemon() self._add_commands() self.daemon.start() - time.sleep(1) client = self._create_client() client.set_prompt(re.compile(r'\w+:\d+> ?')) - client.connect(self.host, self.port) + client.connect(self.host, self.daemon.port) client.login(Account('user', 'password')) client.execute('ls') self.assertEqual(client.response, 'ok1\n') From f681ea41c8f21b10f0baeedd52ffde7dd2584985 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 3 Nov 2026 19:46:50 +0100 Subject: [PATCH 056/250] Check annotated modules strictly and measure docstring coverage mypy runs in strict mode, relaxed for the modules that do not pass yet, and pre-commit fails if the docstring coverage drops. The package __init__ modules list __all__ literally, as type checkers cannot evaluate locals(). --- .pre-commit-config.yaml | 7 ++ CONTRIBUTING.md | 5 +- pyproject.toml | 149 ++++++++++++++++++++++++++- src/Exscript/__init__.py | 14 +-- src/Exscript/emulators/__init__.py | 8 +- src/Exscript/interpreter/__init__.py | 6 +- src/Exscript/parselib/__init__.py | 7 +- src/Exscript/protocols/__init__.py | 18 +++- src/Exscript/servers/__init__.py | 8 +- src/Exscript/util/__init__.py | 8 +- src/Exscript/workqueue/__init__.py | 8 +- 11 files changed, 192 insertions(+), 46 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 49a3d9fa..34093c2a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -24,3 +24,10 @@ repos: rev: c92afdf2c6ca336aaa8f39e2bd8d48f62dbd8e1f # frozen: v1.31.1 hooks: - id: zizmor + - repo: https://github.com/econchick/interrogate + rev: a6268877efd41a21197445cc04cbf28f9c6facba # frozen: 1.7.0 + hooks: + - id: interrogate + # The coverage of the whole package, configured in pyproject.toml. + args: [src/Exscript] + pass_filenames: false diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b17cd1cd..7977b6bb 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -43,7 +43,10 @@ recorded sessions. To run them with another Python version, use e.g. **BREAKING**, and say how to migrate. - Document new functions, classes and modules with docstrings in the style of the existing code (`:param:`, `:type:`, `:return:`, `:rtype:`). The API reference is generated from - them. + them, and pre-commit fails if the share of documented functions drops. +- Annotate new code with types. mypy checks every module strictly, except those that are + still in the list of `[[tool.mypy.overrides]]` in `pyproject.toml`; remove a module from + that list when you annotate it. - The public API and the template language stay compatible. Please open an issue before you work on a change that breaks them. diff --git a/pyproject.toml b/pyproject.toml index f0c12fde..2ba55734 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -124,9 +124,154 @@ ignore = [ [tool.mypy] python_version = "3.12" files = ["src"] +strict = true ignore_missing_imports = true # paramiko, ncclient and others ship no type hints -warn_unused_ignores = true -warn_redundant_casts = true + +[[tool.mypy.overrides]] +# Modules that are not annotated yet (P8). Remove a module from this list when +# it passes the strict checks; the list only shrinks. +module = [ + "Exscript.account", + "Exscript.contrib.otel", + "Exscript.emulators.command", + "Exscript.emulators.iosemu", + "Exscript.emulators.vdevice", + "Exscript.host", + "Exscript.interpreter.append", + "Exscript.interpreter.assign", + "Exscript.interpreter.code", + "Exscript.interpreter.enter", + "Exscript.interpreter.execute", + "Exscript.interpreter.expression", + "Exscript.interpreter.extract", + "Exscript.interpreter.fail", + "Exscript.interpreter.functioncall", + "Exscript.interpreter.ifcondition", + "Exscript.interpreter.loop", + "Exscript.interpreter.number", + "Exscript.interpreter.parser", + "Exscript.interpreter.program", + "Exscript.interpreter.regex", + "Exscript.interpreter.rpc", + "Exscript.interpreter.scope", + "Exscript.interpreter.string", + "Exscript.interpreter.template", + "Exscript.interpreter.term", + "Exscript.interpreter.trying", + "Exscript.interpreter.variable", + "Exscript.key", + "Exscript.logger", + "Exscript.parselib.lexer", + "Exscript.parselib.token", + "Exscript.protocols", + "Exscript.protocols.drivers", + "Exscript.protocols.drivers.ace", + "Exscript.protocols.drivers.adtran", + "Exscript.protocols.drivers.aironet", + "Exscript.protocols.drivers.aix", + "Exscript.protocols.drivers.arbor_peakflow", + "Exscript.protocols.drivers.aruba", + "Exscript.protocols.drivers.bigip", + "Exscript.protocols.drivers.brocade", + "Exscript.protocols.drivers.cienasaos", + "Exscript.protocols.drivers.driver", + "Exscript.protocols.drivers.enterasys", + "Exscript.protocols.drivers.enterasys_wc", + "Exscript.protocols.drivers.eos", + "Exscript.protocols.drivers.ericsson_ban", + "Exscript.protocols.drivers.fortios", + "Exscript.protocols.drivers.generic", + "Exscript.protocols.drivers.hp_pro_curve", + "Exscript.protocols.drivers.icotera", + "Exscript.protocols.drivers.ios", + "Exscript.protocols.drivers.ios_xr", + "Exscript.protocols.drivers.isam", + "Exscript.protocols.drivers.junos", + "Exscript.protocols.drivers.junos_erx", + "Exscript.protocols.drivers.mrv", + "Exscript.protocols.drivers.nxos", + "Exscript.protocols.drivers.one_os", + "Exscript.protocols.drivers.rios", + "Exscript.protocols.drivers.shell", + "Exscript.protocols.drivers.smart_edge_os", + "Exscript.protocols.drivers.sp_vxoa", + "Exscript.protocols.drivers.sros", + "Exscript.protocols.drivers.vrp", + "Exscript.protocols.drivers.vxworks", + "Exscript.protocols.drivers.zte", + "Exscript.protocols.drivers.zyxel", + "Exscript.protocols.dummy", + "Exscript.protocols.exception", + "Exscript.protocols.netconf", + "Exscript.protocols.osguesser", + "Exscript.protocols.protocol", + "Exscript.protocols.ssh2", + "Exscript.protocols.telnet", + "Exscript.protocols.telnetlib", + "Exscript.queue", + "Exscript.servers.httpd", + "Exscript.servers.server", + "Exscript.servers.sshd", + "Exscript.servers.telnetd", + "Exscript.stdlib.connection", + "Exscript.stdlib.crypt", + "Exscript.stdlib.file", + "Exscript.stdlib.ipv4", + "Exscript.stdlib.list", + "Exscript.stdlib.mysys", + "Exscript.stdlib.string", + "Exscript.stdlib.util", + "Exscript.util.buffer", + "Exscript.util.cast", + "Exscript.util.collections", + "Exscript.util.crypt", + "Exscript.util.decorator", + "Exscript.util.event", + "Exscript.util.file", + "Exscript.util.impl", + "Exscript.util.interact", + "Exscript.util.ip", + "Exscript.util.ipv4", + "Exscript.util.ipv6", + "Exscript.util.log", + "Exscript.util.mail", + "Exscript.util.match", + "Exscript.util.report", + "Exscript.util.sigint", + "Exscript.util.start", + "Exscript.util.syslog", + "Exscript.util.template", + "Exscript.util.tty", + "Exscript.util.url", + "Exscript.util.weakmethod", + "Exscript.util.xml", + "Exscript.workqueue.job", + "Exscript.workqueue.mainloop", + "Exscript.workqueue.pipeline", + "Exscript.workqueue.task", + "Exscript.workqueue.workqueue", +] +disallow_any_generics = false +disallow_subclassing_any = false +disallow_untyped_calls = false +disallow_untyped_defs = false +disallow_incomplete_defs = false +check_untyped_defs = false +disallow_untyped_decorators = false +warn_return_any = false +implicit_reexport = true +strict_equality = false +extra_checks = false + +[tool.interrogate] +# Docstring coverage (P8). Private functions count; other dunder methods than +# __init__, nested functions and property setters do not. Raise fail-under as +# docstrings are added; the goal is 100. +fail-under = 49 +ignore-magic = true +ignore-nested-functions = true +ignore-setters = true +exclude = ["docs", "tests", "tools", "demos", "scripts"] [tool.pytest.ini_options] testpaths = ["tests"] diff --git a/src/Exscript/__init__.py b/src/Exscript/__init__.py index 4bbd70e6..fc54f59e 100644 --- a/src/Exscript/__init__.py +++ b/src/Exscript/__init__.py @@ -24,17 +24,19 @@ The core module. """ -import inspect - from .account import Account, AccountPool from .host import Host from .key import PrivateKey from .logger import FileLogger, Logger from .queue import Queue -from .version import __version__ +from .version import __version__ as __version__ __all__ = [ - name - for name, obj in list(locals().items()) - if not (name.startswith('_') or inspect.ismodule(obj)) + 'Account', + 'AccountPool', + 'Host', + 'PrivateKey', + 'FileLogger', + 'Logger', + 'Queue', ] diff --git a/src/Exscript/emulators/__init__.py b/src/Exscript/emulators/__init__.py index 77005dd2..d67a6708 100644 --- a/src/Exscript/emulators/__init__.py +++ b/src/Exscript/emulators/__init__.py @@ -24,14 +24,12 @@ Emulating a device for testing your scripts. """ -import inspect - from .command import CommandSet from .iosemu import IOSEmulator from .vdevice import VirtualDevice __all__ = [ - name - for name, obj in list(locals().items()) - if not (name.startswith('_') or inspect.ismodule(obj)) + 'CommandSet', + 'IOSEmulator', + 'VirtualDevice', ] diff --git a/src/Exscript/interpreter/__init__.py b/src/Exscript/interpreter/__init__.py index 94778f6e..22d341a0 100644 --- a/src/Exscript/interpreter/__init__.py +++ b/src/Exscript/interpreter/__init__.py @@ -20,12 +20,8 @@ # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -import inspect - from .parser import Parser __all__ = [ - name - for name, obj in list(locals().items()) - if not (name.startswith('_') or inspect.ismodule(obj)) + 'Parser', ] diff --git a/src/Exscript/parselib/__init__.py b/src/Exscript/parselib/__init__.py index e6b5529c..59fed4b5 100644 --- a/src/Exscript/parselib/__init__.py +++ b/src/Exscript/parselib/__init__.py @@ -20,13 +20,10 @@ # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -import inspect - from .lexer import Lexer from .token import Token __all__ = [ - name - for name, obj in list(locals().items()) - if not (name.startswith('_') or inspect.ismodule(obj)) + 'Lexer', + 'Token', ] diff --git a/src/Exscript/protocols/__init__.py b/src/Exscript/protocols/__init__.py index 3f74cf94..71326b3e 100644 --- a/src/Exscript/protocols/__init__.py +++ b/src/Exscript/protocols/__init__.py @@ -117,10 +117,18 @@ def connect(host, default_protocol='ssh', **kwargs): return conn -import inspect - __all__ = [ - name - for name, obj in list(locals().items()) - if not (name.startswith('_') or inspect.ismodule(obj)) + 'Account', + 'to_host', + 'Url', + 'Dummy', + 'Netconf', + 'Protocol', + 'SSH2', + 'Telnet', + 'protocol_map', + 'get_protocol_from_name', + 'create_protocol', + 'prepare', + 'connect', ] diff --git a/src/Exscript/servers/__init__.py b/src/Exscript/servers/__init__.py index 577b1c18..985f1e1d 100644 --- a/src/Exscript/servers/__init__.py +++ b/src/Exscript/servers/__init__.py @@ -24,14 +24,12 @@ Very simple servers, useful for emulating a device for testing. """ -import inspect - from .httpd import HTTPd from .sshd import SSHd from .telnetd import Telnetd __all__ = [ - name - for name, obj in list(locals().items()) - if not (name.startswith('_') or inspect.ismodule(obj)) + 'HTTPd', + 'SSHd', + 'Telnetd', ] diff --git a/src/Exscript/util/__init__.py b/src/Exscript/util/__init__.py index 25e87a0a..fae7955d 100644 --- a/src/Exscript/util/__init__.py +++ b/src/Exscript/util/__init__.py @@ -20,10 +20,4 @@ # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -import inspect - -__all__ = [ - name - for name, obj in list(locals().items()) - if not (name.startswith('_') or inspect.ismodule(obj)) -] +__all__: list[str] = [] diff --git a/src/Exscript/workqueue/__init__.py b/src/Exscript/workqueue/__init__.py index 0d73b47c..b06bb5b9 100644 --- a/src/Exscript/workqueue/__init__.py +++ b/src/Exscript/workqueue/__init__.py @@ -20,14 +20,12 @@ # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -import inspect - from .pipeline import Pipeline from .task import Task from .workqueue import WorkQueue __all__ = [ - name - for name, obj in list(locals().items()) - if not (name.startswith('_') or inspect.ismodule(obj)) + 'Pipeline', + 'Task', + 'WorkQueue', ] From c28bd001c2d219a185e6c8e34457a9a5434e8961 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 3 Nov 2026 19:48:31 +0100 Subject: [PATCH 057/250] Type-check against the paramiko and lxml stubs With strict checks, values from untyped libraries are Any, which hides wrong uses. ncclient has no stubs. --- pyproject.toml | 4 +-- uv.lock | 69 +++++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 70 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 2ba55734..6d29c148 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -63,7 +63,7 @@ test = [ "opentelemetry-instrumentation>=0.48b0", "hypothesis>=6.170.14", ] -types = ["mypy>=2.4"] +types = ["mypy>=2.4", "types-lxml>=2026.2", "types-paramiko>=5.1"] docs = ["sphinx>=9", "furo>=2025.12", "myst-parser>=5"] dev = [{ include-group = "test" }, { include-group = "types" }] @@ -125,7 +125,7 @@ ignore = [ python_version = "3.12" files = ["src"] strict = true -ignore_missing_imports = true # paramiko, ncclient and others ship no type hints +ignore_missing_imports = true # ncclient and others ship no type hints [[tool.mypy.overrides]] # Modules that are not annotated yet (P8). Remove a module from this list when diff --git a/uv.lock b/uv.lock index 0c7586bc..7d9b8b1a 100644 --- a/uv.lock +++ b/uv.lock @@ -588,6 +588,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f6/b6/a1faf3a27ae9405fb34b1713cc73b2d8a26b04d5c561578fa2e6ef3e5bb9/cryptography-50.0.2-cp39-abi3-win_amd64.whl", hash = "sha256:4e81d95e5bafc2d6e34e4bed780e53e4d5b9a2f928573428aa4d35fbec1eb0de", size = 3854095, upload-time = "2026-09-30T15:29:46.782Z" }, ] +[[package]] +name = "cssselect" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c8/8b/dc32df939ab541fca6ee8964d26aa231dbe231cdc2b2713228161441ba9c/cssselect-1.6.0.tar.gz", hash = "sha256:8c83a7139e97b93aa5ebdc0f46e785f7056a08a8bf201e597a6a2629d7eb11db", size = 51743, upload-time = "2026-10-09T20:05:09.484Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/08/ae/f24b3aac56ba91a29c9d3a31c07a9ad4e9eb500e5d212742bb6d348edaef/cssselect-1.6.0-py3-none-any.whl", hash = "sha256:6df6eab9b264c0f2092a6e386b33610e1684a25e27925ecebe25e3d97cbf3525", size = 22244, upload-time = "2026-10-09T20:05:08.215Z" }, +] + [[package]] name = "docutils" version = "0.22.4" @@ -621,6 +630,8 @@ dev = [ { name = "opentelemetry-sdk" }, { name = "pytest" }, { name = "pytest-cov" }, + { name = "types-lxml" }, + { name = "types-paramiko" }, ] docs = [ { name = "furo" }, @@ -637,6 +648,8 @@ test = [ ] types = [ { name = "mypy" }, + { name = "types-lxml" }, + { name = "types-paramiko" }, ] [package.metadata] @@ -658,6 +671,8 @@ dev = [ { name = "opentelemetry-sdk", specifier = ">=1.27" }, { name = "pytest", specifier = ">=9.2" }, { name = "pytest-cov", specifier = ">=7.1" }, + { name = "types-lxml", specifier = ">=2026.2" }, + { name = "types-paramiko", specifier = ">=5.1" }, ] docs = [ { name = "furo", specifier = ">=2025.12" }, @@ -671,7 +686,11 @@ test = [ { name = "pytest", specifier = ">=9.2" }, { name = "pytest-cov", specifier = ">=7.1" }, ] -types = [{ name = "mypy", specifier = ">=2.4" }] +types = [ + { name = "mypy", specifier = ">=2.4" }, + { name = "types-lxml", specifier = ">=2026.2" }, + { name = "types-paramiko", specifier = ">=5.1" }, +] [[package]] name = "furo" @@ -1711,6 +1730,54 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/52/a7/d2782e4e3f77c8450f727ba74a8f12756d5ba823d81b941f1b04da9d033a/sphinxcontrib_serializinghtml-2.0.0-py3-none-any.whl", hash = "sha256:6e2cb0eef194e10c27ec0023bfeb25badbbb5868244cf5bc5bdc04e4464bf331", size = 92072, upload-time = "2024-07-29T01:10:08.203Z" }, ] +[[package]] +name = "types-html5lib" +version = "1.1.11.20260518" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "types-webencodings" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b8/5a/0c708d1b0d35ad48b6a223c77c4a882fd016b40c25becb082a92e02a9c00/types_html5lib-1.1.11.20260518.tar.gz", hash = "sha256:4f33c087cb1119d65c4c80eca4323c2b501f9eaf8af9616b8b732ed4d8eae8fa", size = 18420, upload-time = "2026-05-18T06:07:23.662Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4d/d0/b088b9f11eb69637d6826843f06caaff60247156735a25512922d3dc2c13/types_html5lib-1.1.11.20260518-py3-none-any.whl", hash = "sha256:9baa7912224ebb37027c5ccb7e3768e43ea47b1dfdd977e7ddc4b0a4a550584d", size = 24339, upload-time = "2026-05-18T06:07:22.876Z" }, +] + +[[package]] +name = "types-lxml" +version = "2026.2.16" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "beautifulsoup4" }, + { name = "cssselect" }, + { name = "types-html5lib" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/dd/ad/c70ac8cbdc28eb58a17301c69b4925af54b614e47f9b2ebc9de5cc10f786/types_lxml-2026.2.16.tar.gz", hash = "sha256:b3a1340cc06db98d541c785732f6f68bea438daff4e2b7809ef748d545d01406", size = 161204, upload-time = "2026-02-17T02:34:50.855Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5f/5c/03ec9befbf4bb5309bfd576c6a5ac1c75633f78f6b64cf1f594e97cd3d23/types_lxml-2026.2.16-py3-none-any.whl", hash = "sha256:5dd81ffa54830e5f361988737c5f1d6a0ae48b2742790637ec560df790ea0401", size = 97040, upload-time = "2026-02-17T02:34:49.286Z" }, +] + +[[package]] +name = "types-paramiko" +version = "5.1.0.20261030" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cryptography" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/00/fe/33ac1cfe77746fa41c0e949da9bfa526023856bdbf17876768320561ebe3/types_paramiko-5.1.0.20261030.tar.gz", hash = "sha256:fbb2e7b8d901537d50e85ab5df8ab19aec4c78b5f0ec6fff13488d70c8acf44f", size = 28979, upload-time = "2026-10-30T07:50:36.442Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d2/09/78e05b9f31bde56eb2d68f699c547377c3d3d852e28386f447567507211e/types_paramiko-5.1.0.20261030-py3-none-any.whl", hash = "sha256:8e098f34f0ea29e1136beff7454bcd7c5cc942ad2323a6c6a50a20a4b8149047", size = 37757, upload-time = "2026-10-30T07:50:35.526Z" }, +] + +[[package]] +name = "types-webencodings" +version = "0.6.0.20260907" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/01/74/b83cf1d523516bc818ffe6fa7c2f5504e0eeee7c5c394ecc1b404f95fe92/types_webencodings-0.6.0.20260907.tar.gz", hash = "sha256:efa85bc5114419ed45aec227ca5051cca63fa3e2bd13fcf79017ee4107603efc", size = 7748, upload-time = "2026-09-07T06:43:22.142Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/42/e7/dc1ea506e123c437c4551c35498eaade289f52d7f7ddf3f77cf94f0675dc/types_webencodings-0.6.0.20260907-py3-none-any.whl", hash = "sha256:86dc9b5a14665b24d5d7d061149c8c3f50355243df5ef285bf816c2e2cc093d5", size = 8584, upload-time = "2026-09-07T06:43:21.177Z" }, +] + [[package]] name = "typing-extensions" version = "4.16.0" From 51820c9cebe961fb4927eb8a25c90367fea9006b Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 4 Nov 2026 14:01:06 +0100 Subject: [PATCH 058/250] Annotate the basic utility modules Every function gets annotations for mypy --strict and a docstring. Apart from two parameters that are no longer reassigned with another type, only annotations, docstrings and typing.cast() are added. --- src/Exscript/util/buffer.py | 46 ++++++++----- src/Exscript/util/cast.py | 57 ++++++++++------ src/Exscript/util/collections.py | 39 +++++++---- src/Exscript/util/crypt.py | 41 ++++++++--- src/Exscript/util/decorator.py | 53 +++++++++------ src/Exscript/util/event.py | 84 ++++++++++++----------- src/Exscript/util/impl.py | 112 +++++++++++++++++++++---------- src/Exscript/util/ip.py | 20 ++++-- src/Exscript/util/ipv4.py | 73 +++++++++++--------- src/Exscript/util/ipv6.py | 13 ++-- src/Exscript/util/syslog.py | 16 ++--- src/Exscript/util/tty.py | 2 +- src/Exscript/util/url.py | 63 +++++++++-------- src/Exscript/util/weakmethod.py | 74 +++++++++++++++----- src/Exscript/util/xml.py | 16 ++++- 15 files changed, 456 insertions(+), 253 deletions(-) diff --git a/src/Exscript/util/buffer.py b/src/Exscript/util/buffer.py index a419ba90..b37cfe8c 100644 --- a/src/Exscript/util/buffer.py +++ b/src/Exscript/util/buffer.py @@ -25,8 +25,11 @@ """ import functools +import re import threading +from collections.abc import Callable, Iterable from io import StringIO +from typing import Any from .cast import to_regexs @@ -35,9 +38,14 @@ _CONTEXT = 256 -def _locked(method): +def _locked[R](method: Callable[..., R]) -> Callable[..., R]: + """ + Wraps the given method of MonitoredBuffer such that it holds the lock + of the buffer while it runs. + """ + @functools.wraps(method) - def wrapper(self, *args, **kwargs): + def wrapper(self: 'MonitoredBuffer', *args: Any, **kwargs: Any) -> R: with self._lock: return method(self, *args, **kwargs) @@ -52,32 +60,35 @@ class MonitoredBuffer: in ncclient's thread while the caller searches. """ - def __init__(self, io=None): + def __init__(self, io: StringIO | None = None) -> None: """ Constructor. The data is stored in the given file-like object. If no object is given, or the io argument is None, a new StringIO is used. - :type io: file-like object - :param io: A file-like object that is used for storing the data. + :type io: io.StringIO + :param io: A file-like object that is used for storing the data; it + must have getvalue(), like StringIO. """ if io is None: self.io = StringIO('') else: self.io = io - self.monitors = [] + # Each monitor is a mutable list: the regular expressions, the + # callback, the end of the last match, and the limit. + self.monitors: list[list[Any]] = [] self._lock = threading.RLock() self.clear() @_locked - def __str__(self): + def __str__(self) -> str: """ Returns the content of the buffer. """ return self.io.getvalue() @_locked - def size(self): + def size(self) -> int: """ Returns the size of the buffer. @@ -87,7 +98,7 @@ def size(self): return self.io.tell() @_locked - def head(self, bytes): + def head(self, bytes: int) -> str: """ Returns the number of given bytes from the head of the buffer. The buffer remains unchanged. @@ -102,7 +113,7 @@ def head(self, bytes): return head @_locked - def tail(self, bytes): + def tail(self, bytes: int) -> str: """ Returns the number of given bytes from the tail of the buffer. The buffer remains unchanged. @@ -114,7 +125,7 @@ def tail(self, bytes): return self.io.read() @_locked - def pop(self, bytes): + def pop(self, bytes: int) -> str: """ Like :class:`head()`, but also removes the head from the buffer. @@ -132,7 +143,7 @@ def pop(self, bytes): return head @_locked - def append(self, data): + def append(self, data: str) -> None: """ Appends the given data to the buffer, and triggers all connected monitors, if any of them match the buffer content. @@ -162,7 +173,7 @@ def append(self, data): callback(i, match) @_locked - def clear(self): + def clear(self) -> None: """ Removes all data from the buffer. """ @@ -172,7 +183,12 @@ def clear(self): item[2] = 0 @_locked - def add_monitor(self, pattern, callback, limit=80): + def add_monitor( + self, + pattern: str | re.Pattern[str] | Iterable[str | re.Pattern[str]], + callback: Callable[[int, re.Match[str]], object], + limit: int = 80, + ) -> None: """ Calls the given function whenever the given pattern matches the buffer. @@ -181,7 +197,7 @@ def add_monitor(self, pattern, callback, limit=80): the match object of the regular expression. The positions in the match object refer to the searched end of the buffer. - :type pattern: str|re.RegexObject|list(str|re.RegexObject) + :type pattern: str|re.Pattern|list(str|re.Pattern) :param pattern: One or more regular expressions. :type callback: callable :param callback: The function that is called. diff --git a/src/Exscript/util/cast.py b/src/Exscript/util/cast.py index be796315..5ecfc501 100644 --- a/src/Exscript/util/cast.py +++ b/src/Exscript/util/cast.py @@ -25,29 +25,37 @@ """ import re +from collections.abc import Iterable +from typing import TYPE_CHECKING, Any, cast import Exscript +if TYPE_CHECKING: + from ..host import Host -def to_list(item): + +# Returns Any: an iterable item is returned as it is, so the type of the +# result depends on the type of the item. +def to_list(item: object) -> Any: """ If the given item is iterable, this function returns the given item. If the item is not iterable, this function returns a list with only the - item in it. + item in it. A string counts as not iterable. :type item: object :param item: Any object. - :rtype: list - :return: A list with the item in it. + :rtype: list|Iterable + :return: The item, or a list with the item in it. """ if hasattr(item, '__iter__') and not isinstance(item, str): return item return [item] -def to_host(host, default_protocol='ssh', default_domain=''): +def to_host(host: 'str | Host', default_protocol: str = 'ssh', default_domain: str = '') -> 'Host': """ Given a string or a Host object, this function returns a Host object. + Raises TypeError if the host is None. :type host: string|Host :param host: A hostname (may be URL formatted) or a Host object. @@ -61,13 +69,17 @@ def to_host(host, default_protocol='ssh', default_domain=''): if host is None: raise TypeError('None can not be cast to Host') if hasattr(host, 'get_address'): - return host + return cast('Host', host) # any object with get_address() counts as a Host if default_domain and '.' not in host: host += '.' + default_domain return Exscript.Host(host, default_protocol=default_protocol) -def to_hosts(hosts, default_protocol='ssh', default_domain=''): +def to_hosts( + hosts: 'str | Host | Iterable[str | Host]', + default_protocol: str = 'ssh', + default_domain: str = '', +) -> 'list[Host]': """ Given a string or a Host object, or a list of strings or Host objects, this function returns a list of Host objects. @@ -84,33 +96,36 @@ def to_hosts(hosts, default_protocol='ssh', default_domain=''): return [to_host(h, default_protocol, default_domain) for h in to_list(hosts)] -def to_regex(regex, flags=0): +def to_regex(regex: str | re.Pattern[str], flags: int = 0) -> re.Pattern[str]: """ - Given a string, this function returns a new re.RegexObject. - Given a re.RegexObject, this function just returns the same object. + Given a string, this function returns a new re.Pattern. + Given a re.Pattern, this function just returns the same object. + Raises TypeError if the regex is None. - :type regex: string|re.RegexObject - :param regex: A regex or a re.RegexObject + :type regex: string|re.Pattern + :param regex: A regex or a re.Pattern :type flags: int :param flags: See Python's re.compile(). - :rtype: re.RegexObject + :rtype: re.Pattern :return: The Python regex object. """ if regex is None: raise TypeError('None can not be cast to re.RegexObject') if hasattr(regex, 'match'): - return regex + return cast('re.Pattern[str]', regex) # any object with match() counts as compiled return re.compile(regex, flags) -def to_regexs(regexs): +def to_regexs( + regexs: str | re.Pattern[str] | Iterable[str | re.Pattern[str]], +) -> list[re.Pattern[str]]: """ - Given a string or a re.RegexObject, or a list of strings or - re.RegexObjects, this function returns a list of re.RegexObjects. + Given a string or a re.Pattern, or a list of strings or + re.Patterns, this function returns a list of re.Patterns. - :type regexs: str|re.RegexObject|list(str|re.RegexObject) - :param regexs: One or more regexs or re.RegexObjects. - :rtype: list(re.RegexObject) - :return: A list of re.RegexObjects. + :type regexs: str|re.Pattern|list(str|re.Pattern) + :param regexs: One or more regexs or re.Patterns. + :rtype: list(re.Pattern) + :return: A list of re.Patterns. """ return [to_regex(r) for r in to_list(regexs)] diff --git a/src/Exscript/util/collections.py b/src/Exscript/util/collections.py index dc571746..36c58bb9 100644 --- a/src/Exscript/util/collections.py +++ b/src/Exscript/util/collections.py @@ -1,46 +1,61 @@ +""" +Collection types. +""" + import copy from collections import OrderedDict from collections.abc import Callable +from typing import Any, Self, cast -class OrderedDefaultDict(OrderedDict): +class OrderedDefaultDict[K, V](OrderedDict[K, V]): """ A fusion of Python's defaultdict and Python's OrderedDict. """ - def __init__(self, default_factory=None, *a, **kw): - if default_factory is not None and not isinstance(default_factory, Callable): + def __init__(self, default_factory: Callable[[], V] | None = None, *a: Any, **kw: Any) -> None: + """ + Constructor. The other arguments are passed to OrderedDict. + + :param default_factory: Called without arguments to create the value + of a missing key; if it is None, a missing key raises KeyError. + """ + # mypy rejects the Callable ABC in isinstance(). + if default_factory is not None and not isinstance(default_factory, cast(Any, Callable)): raise TypeError('first argument must be callable') OrderedDict.__init__(self, *a, **kw) - self.default_factory = default_factory + self.default_factory: Callable[[], V] | None = default_factory - def __getitem__(self, key): + def __getitem__(self, key: K) -> V: try: return OrderedDict.__getitem__(self, key) except KeyError: return self.__missing__(key) - def __missing__(self, key): + def __missing__(self, key: K) -> V: if self.default_factory is None: raise KeyError(key) self[key] = value = self.default_factory() return value - def __reduce__(self): + def __reduce__(self) -> tuple[Any, ...]: if self.default_factory is None: - args = tuple() + args: tuple[Callable[[], V], ...] = tuple() else: args = (self.default_factory,) return type(self), args, None, None, self.items() - def copy(self): + def copy(self) -> Self: + """ + Returns a shallow copy, with the same default factory. + """ return self.__copy__() - def __copy__(self): + def __copy__(self) -> Self: return type(self)(self.default_factory, self) - def __deepcopy__(self, memo): + def __deepcopy__(self, memo: dict[int, Any]) -> Self: return type(self)(self.default_factory, copy.deepcopy(self.items())) - def __repr__(self): + def __repr__(self) -> str: return 'OrderedDefaultDict(%s, %s)' % (self.default_factory, OrderedDict.__repr__(self)) diff --git a/src/Exscript/util/crypt.py b/src/Exscript/util/crypt.py index 1f9884a5..08188b1a 100644 --- a/src/Exscript/util/crypt.py +++ b/src/Exscript/util/crypt.py @@ -2083,7 +2083,11 @@ ] -def _checksummed_long(key): +def _checksummed_long(key: int) -> int: + """ + Appends the two-bit checksum of the given 64-bit key to it, as the + six-word format (RFC 2289) requires, and returns the result. + """ thesum, k = 0, key for i in range(0, 32): thesum = thesum + (k % 4) @@ -2091,17 +2095,23 @@ def _checksummed_long(key): return (key << 2) | (thesum % 4) -def _sixword_from_long(key): +def _sixword_from_long(key: int) -> str: + """ + Converts the given 64-bit key and its checksum to six words from the + dictionary, as described in RFC 2289. + """ key = _checksummed_long(key) - words = [] + words: list[str] = [] for i in range(0, 6): words = [_DICTIONARY[key % 2048]] + words key = key >> 11 return ' '.join(words) -def _long_from_raw(thehash): - """Fold to a long, a digest supplied as a string.""" +def _long_from_raw(thehash: bytes) -> int: + """ + Converts the given digest to an integer, most significant byte first. + """ hashnum = 0 for h in thehash: hashnum <<= 8 @@ -2109,11 +2119,19 @@ def _long_from_raw(thehash): return hashnum -def _sixword_from_raw(key): +def _sixword_from_raw(key: bytes) -> str: + """ + Converts the given 8-byte key to six words from the dictionary. + """ return _sixword_from_long(_long_from_raw(key)) -def _fold_md4_or_md5(digest): +def _fold_md4_or_md5(digest: bytes) -> bytes: + """ + Folds the given 16-byte MD4 or MD5 digest to 8 bytes by XORing its two + halves, and returns the result. Raises ValueError if the digest is + shorter than 16 bytes. + """ if len(digest) < 16: raise ValueError('digest is too short') result = b'' @@ -2125,11 +2143,12 @@ def _fold_md4_or_md5(digest): return result -def otp(password, seed, sequence): +def otp(password: str, seed: str, sequence: int) -> str: """ Calculates a one-time password hash using the given password, seed, and sequence number and returns it. Uses the MD4/sixword algorithm as supported by TACACS+ servers. + Raises ValueError if an argument is out of range. :type password: str :param password: A password. @@ -2151,11 +2170,11 @@ def otp(password, seed, sequence): raise ValueError('sequence') # Pycryptodome only supports byte strings. - seed = seed.encode('utf-8') - password = password.encode('utf-8') + seed_bytes = seed.encode('utf-8') + password_bytes = password.encode('utf-8') # Discard the first keys - thehash = MD4.new(seed + password).digest() + thehash = MD4.new(seed_bytes + password_bytes).digest() thehash = _fold_md4_or_md5(thehash) for i in range(0, sequence): thehash = _fold_md4_or_md5(MD4.new(thehash).digest()) diff --git a/src/Exscript/util/decorator.py b/src/Exscript/util/decorator.py index fee1973f..dcee2cd8 100644 --- a/src/Exscript/util/decorator.py +++ b/src/Exscript/util/decorator.py @@ -24,11 +24,20 @@ Decorators for callbacks passed to Queue.run(). """ +from collections.abc import Callable, Mapping +from typing import TYPE_CHECKING, Any + from ..protocols.exception import LoginFailure -from .impl import add_label, copy_labels, get_label # noqa: F401 +from .impl import add_label as add_label # re-exported for compatibility +from .impl import copy_labels +from .impl import get_label as get_label # re-exported for compatibility + +if TYPE_CHECKING: + from ..host import Host + from ..protocols import Protocol -def bind(function, *args, **kwargs): +def bind[R](function: Callable[..., R], *args: Any, **kwargs: Any) -> Callable[..., R]: """ Wraps the given function such that when it is called, the given arguments are passed in addition to the connection argument. @@ -43,7 +52,7 @@ def bind(function, *args, **kwargs): :return: The wrapped function. """ - def decorated(*inner_args, **inner_kwargs): + def decorated(*inner_args: Any, **inner_kwargs: Any) -> R: kwargs.update(inner_kwargs) return function(*(inner_args + args), **kwargs) @@ -51,7 +60,7 @@ def decorated(*inner_args, **inner_kwargs): return decorated -def os_function_mapper(map): +def os_function_mapper[R](map: Mapping[str, Callable[..., R]]) -> Callable[..., R]: """ When called with an open connection, this function uses the conn.guess_os() function to guess the operating system @@ -72,19 +81,15 @@ def shell(job, host, conn): An exception is raised if a matching function is not found in the map. - :type conn: Exscript.protocols.Protocol - :param conn: The open connection. :type map: dict :param map: A dictionary mapping operating system name to a function. - :type args: list - :param args: Passed on to the called function. - :type kwargs: dict - :param kwargs: Passed on to the called function. - :rtype: object - :return: The return value of the called function. + :rtype: function + :return: A function that takes the arguments (job, host, conn, ...), + passes them to the function for the operating system of the + connection, and returns its result. """ - def decorated(job, host, conn, *args, **kwargs): + def decorated(job: object, host: 'Host', conn: 'Protocol', *args: Any, **kwargs: Any) -> R: os = conn.guess_os() func = map.get(os) if func is None: @@ -94,7 +99,9 @@ def decorated(job, host, conn, *args, **kwargs): return decorated -def _decorate(flush=True, attempts=1, only_authenticate=False): +def _decorate[R]( + flush: bool = True, attempts: int = 1, only_authenticate: bool = False +) -> Callable[[Callable[..., R]], Callable[..., R]]: """ Wraps the given function such that conn.login() or conn.authenticate() is executed. @@ -108,11 +115,11 @@ def _decorate(flush=True, attempts=1, only_authenticate=False): :type only_authenticate: bool :param only_authenticate: login or only authenticate (don't authorize)? :rtype: function - :return: The wrapped function. + :return: A decorator that wraps the function. """ - def decorator(function): - def decorated(job, host, conn, *args, **kwargs): + def decorator(function: Callable[..., R]) -> Callable[..., R]: + def decorated(job: object, host: 'Host', conn: 'Protocol', *args: Any, **kwargs: Any) -> R: failed = 0 while True: try: @@ -134,7 +141,9 @@ def decorated(job, host, conn, *args, **kwargs): return decorator -def autologin(flush=True, attempts=1): +def autologin[R]( + flush: bool = True, attempts: int = 1 +) -> Callable[[Callable[..., R]], Callable[..., R]]: """ Wraps the given function such that conn.login() is executed before calling it. Example:: @@ -149,12 +158,14 @@ def my_func(job, host, conn): :type attempts: int :param attempts: The number of login attempts if login fails. :rtype: function - :return: The wrapped function. + :return: A decorator that wraps the function. """ return _decorate(flush, attempts) -def autoauthenticate(flush=True, attempts=1): +def autoauthenticate[R]( + flush: bool = True, attempts: int = 1 +) -> Callable[[Callable[..., R]], Callable[..., R]]: """ Wraps the given function such that conn.authenticate() is executed before calling it. Example:: @@ -169,6 +180,6 @@ def my_func(job, host, conn): :type attempts: int :param attempts: The number of login attempts if login fails. :rtype: function - :return: The wrapped function. + :return: A decorator that wraps the function. """ return _decorate(flush, attempts, only_authenticate=True) diff --git a/src/Exscript/util/event.py b/src/Exscript/util/event.py index 5060b496..e483678c 100644 --- a/src/Exscript/util/event.py +++ b/src/Exscript/util/event.py @@ -24,7 +24,9 @@ A simple signal/event mechanism. """ +from collections.abc import Callable from threading import Lock +from typing import Any, cast from . import weakmethod @@ -42,26 +44,28 @@ def mycallback(arg, **kwargs): # Or just: myevent('test', foo = 'bar') """ - def __init__(self): + def __init__(self) -> None: """ Constructor. """ - # To save memory, we do NOT init the subscriber attributes with - # lists. Unfortunately this makes a lot of the code in this class - # more messy than it should be, but events are used so widely in - # Exscript that this change makes a huge difference to the memory - # footprint. - self.lock = None - self.weak_subscribers = None - self.hard_subscribers = None + # To save memory, the lock and the subscriber lists are created only + # when needed: events are used so widely in Exscript that this makes + # a big difference to the memory footprint. + self.lock: Lock | None = None + self.weak_subscribers: ( + list[tuple[weakmethod.WeakMethod, tuple[Any, ...], dict[str, Any]]] | None + ) = None + self.hard_subscribers: ( + list[tuple[Callable[..., Any], tuple[Any, ...], dict[str, Any]]] | None + ) = None - def __call__(self, *args, **kwargs): + def __call__(self, *args: Any, **kwargs: Any) -> Any: """ Like emit(). """ return self.emit(*args, **kwargs) - def connect(self, callback, *args, **kwargs): + def connect(self, callback: Callable[..., Any], *args: Any, **kwargs: Any) -> None: """ Connects the event with the given callback. When the signal is emitted, the callback is invoked. @@ -71,7 +75,7 @@ def connect(self, callback, *args, **kwargs): need to make sure to call :class:`disconnect()` if you want the handler to be garbage collected. - :type callback: object + :type callback: callable :param callback: The callback function. :type args: tuple :param args: Optional arguments passed to the callback. @@ -84,7 +88,9 @@ def connect(self, callback, *args, **kwargs): self.hard_subscribers = [] self.hard_subscribers.append((callback, args, kwargs)) - def listen(self, callback, *args, **kwargs): + def listen( + self, callback: Callable[..., Any], *args: Any, **kwargs: Any + ) -> weakmethod.WeakMethod: """ Like :class:`connect()`, but uses a weak reference instead of a normal reference. @@ -96,7 +102,7 @@ def listen(self, callback, *args, **kwargs): your handler is a local function, it may be garbage collected. To prevent this, use :class:`connect()` instead. - :type callback: object + :type callback: callable :param callback: The callback function. :type args: tuple :param args: Optional arguments passed to the callback. @@ -116,7 +122,7 @@ def listen(self, callback, *args, **kwargs): self.weak_subscribers.append((ref, args, kwargs)) return ref - def n_subscribers(self): + def n_subscribers(self) -> int: """ Returns the number of connected subscribers. @@ -127,10 +133,18 @@ def n_subscribers(self): weak = self.weak_subscribers and len(self.weak_subscribers) or 0 return hard + weak - def _hard_callbacks(self): - return [s[0] for s in self.hard_subscribers] + def _hard_callbacks(self) -> list[Callable[..., Any]]: + """ + Returns the callbacks that are connected with a hard reference. + Only called if there are hard subscribers. + """ + return [s[0] for s in cast('list[Any]', self.hard_subscribers)] - def _weakly_connected_index(self, callback): + def _weakly_connected_index(self, callback: Callable[..., Any]) -> int | None: + """ + Returns the index of the given callback in the list of weak + subscribers, or None if it is not connected with a weak reference. + """ if self.weak_subscribers is None: return None weak = [s[0].get_function() for s in self.weak_subscribers] @@ -139,11 +153,11 @@ def _weakly_connected_index(self, callback): except ValueError: return None - def is_connected(self, callback): + def is_connected(self, callback: Callable[..., Any]) -> bool: """ Returns True if the event is connected to the given function. - :type callback: object + :type callback: callable :param callback: The callback function. :rtype: bool :return: Whether the signal is connected to the given function. @@ -155,7 +169,7 @@ def is_connected(self, callback): return False return callback in self._hard_callbacks() - def emit(self, *args, **kwargs): + def emit(self, *args: Any, **kwargs: Any) -> Any: """ Emits the signal, passing the given arguments to the callbacks. If one of the callbacks returns a value other than None, no further @@ -181,15 +195,9 @@ def emit(self, *args, **kwargs): for callback, user_args, user_kwargs in self.weak_subscribers: kwargs.update(user_kwargs) - # Even though WeakMethod notifies us when the underlying - # function is destroyed, and we remove the item from the - # the list of subscribers, there is no guarantee that - # this notification has already happened because the garbage - # collector may run while this loop is executed. - # Disabling the garbage collector temporarily also does - # not work, because other threads may be trying to do - # the same, causing yet another race condition. - # So the only solution is to skip such functions. + # The garbage collector may destroy the function during this + # loop, before WeakMethod's notification removes it; disabling + # the collector would race with other threads. So skip it. function = callback.get_function() if function is None: continue @@ -197,31 +205,31 @@ def emit(self, *args, **kwargs): if result is not None: return result - def _try_disconnect(self, ref): + def _try_disconnect(self, ref: weakmethod.WeakMethod) -> None: """ Called by the weak reference when its target dies. In other words, we can assert that self.weak_subscribers is not None at this time. """ - with self.lock: - weak = [s[0] for s in self.weak_subscribers] + with cast(Lock, self.lock): + weak = [s[0] for s in cast('list[Any]', self.weak_subscribers)] try: index = weak.index(ref) except ValueError: # subscriber was already removed by a call to disconnect() pass else: - self.weak_subscribers.pop(index) + cast('list[Any]', self.weak_subscribers).pop(index) - def disconnect(self, callback): + def disconnect(self, callback: Callable[..., Any]) -> None: """ Disconnects the signal from the given function. - :type callback: object + :type callback: callable :param callback: The callback function. """ if self.weak_subscribers is not None: - with self.lock: + with cast(Lock, self.lock): # listen() creates the lock first index = self._weakly_connected_index(callback) if index is not None: self.weak_subscribers.pop(index)[0] @@ -233,7 +241,7 @@ def disconnect(self, callback): else: self.hard_subscribers.pop(index) - def disconnect_all(self): + def disconnect_all(self) -> None: """ Disconnects all connected functions from all signals. """ diff --git a/src/Exscript/util/impl.py b/src/Exscript/util/impl.py index 9db5e4b6..0ac829ec 100644 --- a/src/Exscript/util/impl.py +++ b/src/Exscript/util/impl.py @@ -27,10 +27,13 @@ import sys import traceback import warnings +from collections.abc import Callable from functools import wraps +from types import TracebackType +from typing import Any, Self -def add_label(obj, name, **kwargs): +def add_label[T](obj: T, name: str, **kwargs: Any) -> T: """ Labels an object such that it can later be checked with :class:`get_label()`. @@ -49,7 +52,7 @@ def add_label(obj, name, **kwargs): return obj -def get_label(obj, name): +def get_label(obj: object, name: str) -> dict[str, Any] | None: """ Checks whether an object has the given label attached (see :class:`add_label()`) and returns the associated options. @@ -61,13 +64,13 @@ def get_label(obj, name): :rtype: dict or None :return: The optional values if the label is attached, None otherwise. """ - labels = obj.__dict__.get('_labels') + labels: dict[str, dict[str, Any]] | None = obj.__dict__.get('_labels') if labels is None: return None return labels.get(name) -def copy_labels(src, dst): +def copy_labels(src: object, dst: object) -> None: """ Copies all labels of one object to another object. @@ -82,16 +85,19 @@ def copy_labels(src, dst): dst.__dict__['_labels'] = labels.copy() -def serializeable_exc_info(thetype, ex, tb): +def serializeable_exc_info( + thetype: type[BaseException] | None, ex: BaseException | None, tb: TracebackType | None +) -> tuple[type[BaseException] | None, BaseException | None, str]: """ - Since traceback objects can not be pickled, this function manipulates - exception info tuples before they are passed accross process - boundaries. + Returns the given exception info with the traceback formatted as a + string, so that it can be pickled, unlike a traceback object. + + :return: The exception info, with the traceback formatted as a string. """ return thetype, ex, ''.join(traceback.format_exception(thetype, ex, tb)) -def serializeable_sys_exc_info(): +def serializeable_sys_exc_info() -> tuple[type[BaseException] | None, BaseException | None, str]: """ Convenience wrapper around serializeable_exc_info, equivalent to serializeable_exc_info(sys.exc_info()). @@ -99,38 +105,37 @@ def serializeable_sys_exc_info(): return serializeable_exc_info(*sys.exc_info()) -def format_exception(thetype, ex, tb): +def format_exception( + thetype: type[BaseException] | None, ex: BaseException | None, tb: TracebackType | str | None +) -> str: """ - This function is a drop-in replacement for Python's - traceback.format_exception(). - - Since traceback objects can not be pickled, Exscript is forced to - manipulate them before they are passed accross process boundaries. - This leads to the fact the Python's traceback.format_exception() - no longer works for those objects. + Formats the given exception info like Python's + traceback.format_exception(), but also accepts a traceback that is + formatted as a string, as serializeable_exc_info() returns it. - This function works with any traceback object, regardless of whether - or not Exscript manipulated it. + :return: The formatted exception; unlike Python's function, a string. """ if isinstance(tb, str): return tb return ''.join(traceback.format_exception(thetype, ex, tb)) -def deprecation(msg): +def deprecation(msg: str) -> None: """ - Prints a deprecation warning. + Issues a DeprecationWarning with the given message. Call it in the + deprecated function: the warning points to the code that called that + function. """ warnings.warn(msg, category=DeprecationWarning, stacklevel=3) -def deprecated(func): +def deprecated[**P, R](func: Callable[P, R]) -> Callable[P, R]: """ - A decorator for marking functions as deprecated. Results in - a printed warning message when the function is used. + A decorator for marking functions as deprecated: each call of the + function issues a DeprecationWarning that points to the calling code. """ - def decorated(*args, **kwargs): + def decorated(*args: P.args, **kwargs: P.kwargs) -> R: deprecation('Call to deprecated function %s.' % func.__name__) return func(*args, **kwargs) @@ -140,13 +145,13 @@ def decorated(*args, **kwargs): return decorated -def synchronized(func): +def synchronized[R](func: Callable[..., R]) -> Callable[..., R]: """ Decorator for synchronizing method access. """ @wraps(func) - def wrapped(self, *args, **kwargs): + def wrapped(self: Any, *args: Any, **kwargs: Any) -> R: try: rlock = self._sync_lock except AttributeError: @@ -159,13 +164,13 @@ def wrapped(self, *args, **kwargs): return wrapped -def debug(func): +def debug[**P, R](func: Callable[P, R]) -> Callable[P, R]: """ Decorator that prints a message whenever a function is entered or left. """ @wraps(func) - def wrapped(*args, **kwargs): + def wrapped(*args: P.args, **kwargs: P.kwargs) -> R: arg = repr(args) + ' ' + repr(kwargs) sys.stdout.write('Entering ' + func.__name__ + arg + '\n') try: @@ -182,32 +187,67 @@ def wrapped(*args, **kwargs): class Decorator: - def __init__(self, obj): + """ + Wraps an object: the attributes of the wrapped object can be read and + written through the wrapper, except for the attributes of the wrapper + itself. + """ + + def __init__(self, obj: object) -> None: + """ + Constructor. + + :param obj: The wrapped object. + """ self.__dict__['obj'] = obj - def __setattr__(self, name, value): + def __setattr__(self, name: str, value: object) -> None: if name in list(self.__dict__.keys()): self.__dict__[name] = value else: setattr(self.obj, name, value) - def __getattr__(self, name): + def __getattr__(self, name: str) -> Any: if name in list(self.__dict__.keys()): return self.__dict__[name] return getattr(self.obj, name) class _Context(Decorator): - def __enter__(self): + """ + Wraps an object for a with statement that does nothing when it ends. + """ + + def __enter__(self) -> Self: return self - def __exit__(self, thetype, value, traceback): + def __exit__( + self, + thetype: type[BaseException] | None, + value: BaseException | None, + traceback: TracebackType | None, + ) -> None: pass class Context(_Context): - def __exit__(self, thetype, value, traceback): + """ + Wraps an object for a with statement that calls release() of the object + when it ends, e.g. to release an account. + """ + + # Any: the result of release() of the wrapped object. + def __exit__( + self, + thetype: type[BaseException] | None, + value: BaseException | None, + traceback: TracebackType | None, + ) -> Any: return self.release() - def context(self): + def context(self) -> _Context: + """ + Returns a context for the wrapped object that does not call + release() when it ends. + """ return _Context(self) diff --git a/src/Exscript/util/ip.py b/src/Exscript/util/ip.py index bea08d64..7f6d82cd 100644 --- a/src/Exscript/util/ip.py +++ b/src/Exscript/util/ip.py @@ -24,10 +24,12 @@ Wrapper around the ipv4 and ipv6 modules to handle both, ipv4 and ipv6. """ +from typing import Any, cast + from Exscript.util import ipv4, ipv6 -def is_ip(string): +def is_ip(string: str) -> bool: """ Returns True if the given string is an IPv4 or IPv6 address, False otherwise. @@ -40,7 +42,13 @@ def is_ip(string): return ipv4.is_ip(string) or ipv6.is_ip(string) -def _call_func(funcname, ip, *args): +def _call_func(funcname: str, ip: str, *args: Any) -> Any: + """ + Calls the function with the given name from the ipv4 or the ipv6 module, + depending on the version of the given IP address, and returns its result. + Raises ValueError if the address is neither an IPv4 nor an IPv6 address. + """ + # Any: the result is that of the function with the given name. if ipv4.is_ip(ip): return ipv4.__dict__[funcname](ip, *args) elif ipv6.is_ip(ip): @@ -48,7 +56,7 @@ def _call_func(funcname, ip, *args): raise ValueError('neither ipv4 nor ipv6: ' + repr(ip)) -def normalize_ip(ip): +def normalize_ip(ip: str) -> str: """ Transform the address into a fixed-length form, such as: @@ -60,10 +68,10 @@ def normalize_ip(ip): :rtype: string :return: The normalized IP. """ - return _call_func('normalize_ip', ip) + return cast(str, _call_func('normalize_ip', ip)) -def clean_ip(ip): +def clean_ip(ip: str) -> str: """ Cleans the ip address up, useful for removing leading zeros, e.g.:: @@ -75,4 +83,4 @@ def clean_ip(ip): :rtype: string :return: The cleaned up IP. """ - return _call_func('clean_ip', ip) + return cast(str, _call_func('clean_ip', ip)) diff --git a/src/Exscript/util/ipv4.py b/src/Exscript/util/ipv4.py index 208a5772..25351976 100644 --- a/src/Exscript/util/ipv4.py +++ b/src/Exscript/util/ipv4.py @@ -28,16 +28,26 @@ import re import socket import struct +from collections.abc import Iterable +from typing import cast -def _least_bit(number): +def _least_bit(number: int) -> int: + """ + Returns the position of the lowest bit that is set in the given 32-bit + number, or 32 if no bit is set. + """ for i in range(0, 32): if number & (0x00000001 << i) != 0: return i return 32 -def _highest_bit(number): +def _highest_bit(number: int) -> float: + """ + Rounds the given 32-bit number up to the next power of two and returns + the square root of the result; returns 0 for 0. + """ if number == 0: return 0 number -= 1 @@ -50,7 +60,7 @@ def _highest_bit(number): return math.sqrt(number) -def is_ip(string): +def is_ip(string: str) -> bool: """ Returns True if the given string is an IPv4 address, False otherwise. @@ -68,7 +78,7 @@ def is_ip(string): return True -def normalize_ip(ip): +def normalize_ip(ip: str) -> str: """ Transform the address into a fixed-length form, such as:: @@ -85,7 +95,7 @@ def normalize_ip(ip): return '.'.join(str(int(l)).rjust(3, '0') for l in theip) -def clean_ip(ip): +def clean_ip(ip: str) -> str: """ Cleans the ip address up, useful for removing leading zeros, e.g.:: @@ -99,25 +109,25 @@ def clean_ip(ip): return '.'.join(str(int(i)) for i in ip.split('.')) -def ip2int(ip): +def ip2int(ip: str) -> int: """ Converts the given IP address to a 4 byte integer value. :type ip: string :param ip: An IP address. - :rtype: long + :rtype: int :return: The IP, converted to a number. """ if ip == '255.255.255.255': return 0xFFFFFFFF - return struct.unpack('!L', socket.inet_aton(ip))[0] + return cast(int, struct.unpack('!L', socket.inet_aton(ip))[0]) -def int2ip(number): +def int2ip(number: int) -> str: """ Converts the given integer value to an IP address. - :type number: long + :type number: int :param number: An IP as a number. :rtype: string :return: The IP address. @@ -126,19 +136,19 @@ def int2ip(number): return socket.inet_ntoa(struct.pack('!L', number)) -def pfxlen2mask_int(pfxlen): +def pfxlen2mask_int(pfxlen: int) -> int: """ Converts the given prefix length to an IP mask value. :type pfxlen: int :param pfxlen: A prefix length. - :rtype: long - :return: The mask, as a long value. + :rtype: int + :return: The mask, as an integer. """ return 0xFFFFFFFF << (32 - int(pfxlen)) -def pfxlen2mask(pfxlen): +def pfxlen2mask(pfxlen: int) -> str: """ Converts the given prefix length to an IP mask. @@ -150,31 +160,32 @@ def pfxlen2mask(pfxlen): return int2ip(pfxlen2mask_int(pfxlen)) -def mask2pfxlen(mask): +def mask2pfxlen(mask: str) -> int: """ Converts the given IP mask to a prefix length. :type mask: string :param mask: An IP mask. - :rtype: long - :return: The mask, as a long value. + :rtype: int + :return: The prefix length. """ return 32 - _least_bit(ip2int(mask)) -def parse_prefix(prefix, default_length=24): +def parse_prefix(prefix: str, default_length: int = 24) -> tuple[str, int]: """ Splits the given IP prefix into a network address and a prefix length. If the prefix does not have a length (i.e., it is a simple IP address), it is presumed to have the given default length. :type prefix: string - :param prefix: An IP mask. - :type default_length: long + :param prefix: An IP prefix. + :type default_length: int :param default_length: The default ip prefix length. :rtype: string, int :return: A tuple containing the IP address and prefix length. """ + pfxlen: str | int if '/' in prefix: network, pfxlen = prefix.split('/') else: @@ -183,14 +194,14 @@ def parse_prefix(prefix, default_length=24): return network, int(pfxlen) -def network(prefix, default_length=24): +def network(prefix: str, default_length: int = 24) -> str: """ Given a prefix, this function returns the corresponding network address. :type prefix: string :param prefix: An IP prefix. - :type default_length: long + :type default_length: int :param default_length: The default ip prefix length. :rtype: string :return: The IP network address. @@ -200,14 +211,14 @@ def network(prefix, default_length=24): return int2ip(ip & pfxlen2mask_int(pfxlen)) -def broadcast(prefix, default_length=24): +def broadcast(prefix: str, default_length: int = 24) -> str: """ Given a prefix, this function returns the corresponding broadcast address. :type prefix: string :param prefix: An IP prefix. - :type default_length: long + :type default_length: int :param default_length: The default ip prefix length. :rtype: string :return: The IP broadcast address. @@ -217,7 +228,7 @@ def broadcast(prefix, default_length=24): return int2ip(ip | ~pfxlen2mask_int(pfxlen)) -def remote_ip(local_ip): +def remote_ip(local_ip: str) -> str: """ Given an IP address, this function calculates the remaining available IP address under the assumption that it is a /30 network. @@ -229,12 +240,12 @@ def remote_ip(local_ip): :rtype: string :return: The other IP address of the link address pair. """ - local_ip = ip2int(local_ip) - network = local_ip & pfxlen2mask_int(30) - return int2ip(network + 3 - (local_ip - network)) + ip_int = ip2int(local_ip) + network = ip_int & pfxlen2mask_int(30) + return int2ip(network + 3 - (ip_int - network)) -def matches_prefix(ip, prefix): +def matches_prefix(ip: str, prefix: str) -> bool: """ Returns True if the given IP address is part of the given network, returns False otherwise. @@ -253,7 +264,7 @@ def matches_prefix(ip, prefix): return ip_int & mask_int == network_int & mask_int -def is_private(ip): +def is_private(ip: str) -> bool: """ Returns True if the given IP address is private, returns False otherwise. @@ -272,7 +283,7 @@ def is_private(ip): return False -def sort(iterable): +def sort(iterable: Iterable[str]) -> list[str]: """ Given an IP address list, this function sorts the list. diff --git a/src/Exscript/util/ipv6.py b/src/Exscript/util/ipv6.py index 4e3b81db..6ca3aec2 100644 --- a/src/Exscript/util/ipv6.py +++ b/src/Exscript/util/ipv6.py @@ -25,7 +25,7 @@ """ -def is_ip(string): +def is_ip(string: str) -> bool: """ Returns True if the given string is an IPv6 address, False otherwise. @@ -41,7 +41,7 @@ def is_ip(string): return True -def normalize_ip(ip): +def normalize_ip(ip: str) -> str: """ Transform the address into a standard, fixed-length form, such as: @@ -81,7 +81,7 @@ def normalize_ip(ip): return ':'.join(result).lower() -def clean_ip(ip): +def clean_ip(ip: str) -> str: """ Cleans the ip address up, useful for removing leading zeros, e.g.:: @@ -132,19 +132,20 @@ def clean_ip(ip): return ':'.join(result) -def parse_prefix(prefix, default_length=128): +def parse_prefix(prefix: str, default_length: int = 128) -> tuple[str, int]: """ Splits the given IP prefix into a network address and a prefix length. If the prefix does not have a length (i.e., it is a simple IP address), it is presumed to have the given default length. :type prefix: string - :param prefix: An IP mask. - :type default_length: long + :param prefix: An IP prefix. + :type default_length: int :param default_length: The default ip prefix length. :rtype: string, int :return: A tuple containing the IP address and prefix length. """ + pfxlen: str | int if '/' in prefix: network, pfxlen = prefix.split('/') else: diff --git a/src/Exscript/util/syslog.py b/src/Exscript/util/syslog.py index e7698143..615fdf89 100644 --- a/src/Exscript/util/syslog.py +++ b/src/Exscript/util/syslog.py @@ -34,13 +34,13 @@ def netlog( - message, - source=None, - host='localhost', - port=514, - priority=LOG_DEBUG, - facility=LOG_USER, -): + message: str, + source: str | None = None, + host: str = 'localhost', + port: int = 514, + priority: int = LOG_DEBUG, + facility: int = LOG_USER, +) -> None: """ Sends a message to a syslog server over UDP, like the logger -n command. Python's built in syslog module does not support networking, so @@ -54,7 +54,7 @@ def netlog( :type message: str :param message: The message. :type source: str - :param source: The source address. + :param source: The message source, e.g. "scriptname[pid]". :type host: str :param host: The IP address or hostname of the receiving server. :type port: int diff --git a/src/Exscript/util/tty.py b/src/Exscript/util/tty.py index 88067564..ca5368f6 100644 --- a/src/Exscript/util/tty.py +++ b/src/Exscript/util/tty.py @@ -27,7 +27,7 @@ import shutil -def get_terminal_size(default_rows=25, default_cols=80): +def get_terminal_size(default_rows: int = 25, default_cols: int = 80) -> tuple[int, int]: """ Returns the number of lines and columns of the terminal: from the LINES and COLUMNS environment variables if they are set, else from the terminal diff --git a/src/Exscript/util/url.py b/src/Exscript/util/url.py index 029aa08c..7a66594f 100644 --- a/src/Exscript/util/url.py +++ b/src/Exscript/util/url.py @@ -25,12 +25,17 @@ """ import re +from typing import Any, cast from urllib.parse import quote, urlencode, urlparse, urlsplit from .collections import OrderedDefaultDict -def _make_hexmap(): +def _make_hexmap() -> dict[str, str]: + """ + Returns a dictionary that maps each two-digit hexadecimal number, in + lower and in upper case, to the character with that code. + """ hexmap = dict() for i in range(256): hexmap['%02x' % i] = chr(i) @@ -54,8 +59,12 @@ def _make_hexmap(): } -def _unquote(string): - """_unquote('abc%20def') -> 'abc def'.""" +def _unquote(string: str) -> str: + """ + Replaces each %xx escape in the given string by the character with that + code, e.g. _unquote('abc%20def') -> 'abc def'. A % that is not followed + by two hexadecimal digits is kept. + """ result = string.split('%') for i, item in enumerate(result[1:]): i += 1 @@ -68,15 +77,12 @@ def _unquote(string): return ''.join(result) -def _urlparse_qs(url): +def _urlparse_qs(url: str) -> OrderedDefaultDict[str, list[str]]: """ - Parse a URL query string and return the components as a dictionary. - - Based on the cgi.parse_qs method.This is a utility function provided - with urlparse so that users need not use cgi module for - parsing the url query string. - - Arguments: + Parses the query string of the given URL and returns the variables as + an ordered dictionary that maps each name to the list of its values. + The pairs are separated by & or ;, a + stands for a space, and pairs + without a value are skipped. :type url: str :param url: URL with query string to be parsed @@ -88,7 +94,7 @@ def _urlparse_qs(url): pairs = [s2 for s1 in querystring.split('&') for s2 in s1.split(';')] # Split the name/value pairs. - result = OrderedDefaultDict(list) + result: OrderedDefaultDict[str, list[str]] = OrderedDefaultDict(list) for name_value in pairs: pair = name_value.split('=', 1) if len(pair) != 2: @@ -107,17 +113,20 @@ class Url: Represents a URL. """ - def __init__(self): - self.protocol = None - self.username = None - self.password1 = None - self.password2 = None - self.hostname = None - self.port = None - self.path = None - self.vars = None - - def __str__(self): + def __init__(self) -> None: + """ + Constructor. All attributes are None; see :class:`from_string()`. + """ + self.protocol: str | None = None + self.username: str | None = None + self.password1: str | None = None + self.password2: str | None = None + self.hostname: str | None = None + self.port: int | None = None + self.path: str | None = None + self.vars: dict[str, list[Any]] | None = None + + def __str__(self) -> str: """ Like :class:`to_string()`. @@ -137,7 +146,7 @@ def __str__(self): if self.password2 is not None: url += ':' + quote(self.password2, '') url += '@' - url += self.hostname + url += cast(str, self.hostname) # a hostname is required; None raises TypeError if self.port: url += ':' + str(self.port) if self.path: @@ -151,7 +160,7 @@ def __str__(self): url += '?' + urlencode(pairs) return url - def to_string(self): + def to_string(self) -> str: """ Returns the URL, including all attributes, as a string. @@ -161,7 +170,7 @@ def to_string(self): return str(self) @staticmethod - def from_string(url, default_protocol='ssh'): + def from_string(url: str, default_protocol: str = 'ssh') -> 'Url': """ Parses the given URL and returns an URL object. There are some differences to Python's built-in URL parser: @@ -210,7 +219,7 @@ def from_string(url, default_protocol='ssh'): netloc = parsed[1] # Parse username and password. - auth = '' + auth: str | list[str] = '' if '@' in netloc: auth, netloc = netloc.split('@') auth = auth.split(':') diff --git a/src/Exscript/util/weakmethod.py b/src/Exscript/util/weakmethod.py index bb2e8fba..a18a5ff1 100644 --- a/src/Exscript/util/weakmethod.py +++ b/src/Exscript/util/weakmethod.py @@ -25,6 +25,9 @@ """ import weakref +from collections.abc import Callable +from types import MethodType +from typing import Any, cast class DeadMethodCalled(Exception): @@ -43,18 +46,22 @@ class WeakMethod: __slots__ = 'name', 'callback' - def __init__(self, name, callback): + def __init__(self, name: str, callback: 'Callable[[WeakMethod], object] | None') -> None: """ Constructor. Do not use directly, use :class:`ref()` instead. """ self.name = name self.callback = callback - def _dead(self, ref): + def _dead(self, ref: weakref.ref[Any]) -> None: + """ + Called when the referenced object dies; passes this weak reference + to the callback, if any. + """ if self.callback is not None: self.callback(self) - def get_function(self): + def get_function(self) -> Callable[..., Any] | None: """ Returns the referenced method/function if it is still alive. Returns None otherwise. @@ -64,7 +71,7 @@ def get_function(self): """ raise NotImplementedError() - def isalive(self): + def isalive(self) -> bool: """ Returns True if the referenced function is still alive, False otherwise. @@ -74,13 +81,11 @@ def isalive(self): """ return self.get_function() is not None - def __call__(self, *args, **kwargs): + def __call__(self, *args: Any, **kwargs: Any) -> None: """ Proxied to the underlying function or method. Raises :class:`DeadMethodCalled` - if the referenced function is dead. - - :rtype: object - :return: Whatever the referenced function returned. + if the referenced function is dead. The result of the function is + discarded. """ method = self.get_function() if method is None: @@ -89,33 +94,67 @@ def __call__(self, *args, **kwargs): class _WeakMethodBound(WeakMethod): + """ + A weak reference to a bound method. Only the object of the method is + referenced weakly; the method is looked up by its name when needed. + """ + __slots__ = 'name', 'callback', 'f', 'c' - def __init__(self, f, callback): + def __init__(self, f: MethodType, callback: Callable[[WeakMethod], object] | None) -> None: + """ + Constructor. + + :param f: The bound method. + :param callback: Called with this object when the object of the + method dies. + """ name = f.__self__.__class__.__name__ + '.' + f.__func__.__name__ WeakMethod.__init__(self, name, callback) self.f = f.__func__ self.c = weakref.ref(f.__self__, self._dead) - def get_function(self): + def get_function(self) -> Callable[..., Any] | None: + """ + Returns the method, bound to the object if it is still alive. + Returns None otherwise. + """ cls = self.c() if cls is None: return None - return getattr(cls, self.f.__name__) + return cast('Callable[..., Any]', getattr(cls, self.f.__name__)) class _WeakMethodFree(WeakMethod): + """ + A weak reference to a function or another callable object that is not + a bound method. + """ + __slots__ = 'name', 'callback', 'f' - def __init__(self, f, callback): + def __init__( + self, f: Callable[..., Any], callback: Callable[[WeakMethod], object] | None + ) -> None: + """ + Constructor. + + :param f: The function. + :param callback: Called with this object when the function dies. + """ WeakMethod.__init__(self, f.__class__.__name__, callback) self.f = weakref.ref(f, self._dead) - def get_function(self): + def get_function(self) -> Callable[..., Any] | None: + """ + Returns the function if it is still alive, None otherwise. + """ return self.f() -def ref(function, callback=None): +def ref( + function: Callable[..., Any], callback: Callable[[WeakMethod], object] | None = None +) -> WeakMethod: """ Returns a weak reference to the given method or function. If the callback argument is not None, it is called as soon @@ -126,8 +165,9 @@ def ref(function, callback=None): :type callback: callable :param callback: Called when the function dies. """ + # A bound method is recognized by its __func__ attribute. try: - function.__func__ + cast(MethodType, function).__func__ except AttributeError: return _WeakMethodFree(function, callback) - return _WeakMethodBound(function, callback) + return _WeakMethodBound(cast(MethodType, function), callback) diff --git a/src/Exscript/util/xml.py b/src/Exscript/util/xml.py index 2eb670a3..58ebab43 100644 --- a/src/Exscript/util/xml.py +++ b/src/Exscript/util/xml.py @@ -27,8 +27,13 @@ free-threaded Python. """ +from typing import TYPE_CHECKING -def _parse(string, huge_tree=False): +if TYPE_CHECKING: + from lxml.etree import _Element + + +def _parse(string: str, huge_tree: bool = False) -> '_Element': """ Parses an XML string and returns the root element, safe from XXE attacks: the parser never resolves entities or loads DTDs or other @@ -50,12 +55,17 @@ class XMLValidationError(Exception): Provides the XML string that failed in its `xmlstring` attribute. """ - def __init__(self, *args: object, xmlstring=None) -> None: + def __init__(self, *args: object, xmlstring: str | None = None) -> None: + """ + Constructor. The positional arguments are passed to Exception. + + :param xmlstring: The XML string that failed the validation. + """ super().__init__(*args) self.xmlstring = xmlstring -def validate_xml(string, *, silent=False): +def validate_xml(string: str, *, silent: bool = False) -> bool: """ Try to parse the given string as XML to verify its syntax. From 8be7106400b955f2f21af2d98c26484b8d62aae8 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 4 Nov 2026 14:01:07 +0100 Subject: [PATCH 059/250] Annotate Account, Host, PrivateKey and the loggers Every function gets annotations for mypy --strict and a docstring. Apart from a new local name in Host.set_uri(), only annotations, docstrings and typing.cast() are added. --- src/Exscript/account.py | 264 ++++++++++++++++++++++++-------------- src/Exscript/host.py | 104 ++++++++------- src/Exscript/key.py | 38 +++--- src/Exscript/logger.py | 275 +++++++++++++++++++++++++++++++++------- 4 files changed, 477 insertions(+), 204 deletions(-) diff --git a/src/Exscript/account.py b/src/Exscript/account.py index b43267bb..092147e4 100644 --- a/src/Exscript/account.py +++ b/src/Exscript/account.py @@ -26,18 +26,37 @@ import multiprocessing from collections import defaultdict, deque +from collections.abc import Callable, Hashable, Iterable +from types import TracebackType +from typing import TYPE_CHECKING, Self, cast +from .key import PrivateKey +from .logger import Log from .util.cast import to_list from .util.event import Event from .util.impl import Context +if TYPE_CHECKING: + # _ConnectionBase is the common base class of the pipe connections on + # POSIX (Connection) and on Windows (PipeConnection). + from multiprocessing.connection import _ConnectionBase + + from .host import Host + class Account: """ This class represents a user account. """ - def __init__(self, name, password='', password2=None, key=None, needs_lock=True): + def __init__( + self, + name: str, + password: str | None = '', + password2: str | None = None, + key: PrivateKey | None = None, + needs_lock: bool = True, + ) -> None: """ Constructor. @@ -62,8 +81,8 @@ def __init__(self, name, password='', password2=None, key=None, needs_lock=True) :type name: str :param name: A username. - :type password: str - :param password: The authentication password. + :type password: str|None + :param password: The authentication password, or None if it is unknown. :type password2: str :param password2: The authorization password, if required. :type key: Exscript.PrivateKey @@ -82,25 +101,30 @@ def __init__(self, name, password='', password2=None, key=None, needs_lock=True) self.lock = multiprocessing.Lock() self.needs_lock = needs_lock - def __repr__(self): + def __repr__(self) -> str: return f'{type(self).__name__}({self.name!r})' # never the passwords - def __enter__(self): + def __enter__(self) -> Self: if self.needs_lock: self.acquire() return self - def __exit__(self, thetype, value, traceback): + def __exit__( + self, + thetype: type[BaseException] | None, + value: BaseException | None, + traceback: TracebackType | None, + ) -> None: if self.needs_lock: self.release() - def context(self): + def context(self) -> Context: """ When you need a 'with' context for an already-acquired account. """ return Context(self) - def acquire(self, signal=True): + def acquire(self, signal: bool = True) -> None: """ Locks the account. Method has no effect if the constructor argument `needs_lock` @@ -118,7 +142,7 @@ def acquire(self, signal=True): self.acquired_event(self) self.synclock.notify_all() - def release(self, signal=True): + def release(self, signal: bool = True) -> None: """ Unlocks the account. Method has no effect if the constructor argument `needs_lock` @@ -135,7 +159,7 @@ def release(self, signal=True): self.released_event(self) self.synclock.notify_all() - def set_name(self, name): + def set_name(self, name: str) -> None: """ Changes the name of the account. @@ -145,7 +169,7 @@ def set_name(self, name): self.name = name self.changed_event.emit(self) - def get_name(self): + def get_name(self) -> str: """ Returns the name of the account. @@ -154,45 +178,47 @@ def get_name(self): """ return self.name - def set_password(self, password): + def set_password(self, password: str | None) -> None: """ Changes the password of the account. - :type password: string - :param password: The account password. + :type password: string|None + :param password: The account password, or None if it is unknown. """ self.password = password self.changed_event.emit(self) - def get_password(self): + def get_password(self) -> str | None: """ Returns the password of the account. - :rtype: string - :return: The account password. + :rtype: string|None + :return: The account password, or None if it is unknown. """ return self.password - def set_authorization_password(self, password): + def set_authorization_password(self, password: str | None) -> None: """ Changes the authorization password of the account. - :type password: string - :param password: The new authorization password. + :type password: string|None + :param password: The new authorization password, or None to use the + authentication password. """ self.authorization_password = password self.changed_event.emit(self) - def get_authorization_password(self): + def get_authorization_password(self) -> str | None: """ - Returns the authorization password of the account. + Returns the authorization password of the account, or the + authentication password if no authorization password is set. - :rtype: string + :rtype: string|None :return: The account password. """ return self.authorization_password or self.password - def get_key(self): + def get_key(self) -> PrivateKey | None: """ Returns the key of the account, if any. @@ -208,7 +234,7 @@ class AccountProxy: process. """ - def __init__(self, parent): + def __init__(self, parent: '_ConnectionBase') -> None: """ Constructor. @@ -216,19 +242,20 @@ def __init__(self, parent): :param parent: A pipe to the associated account manager. """ self.parent = parent - self.account_hash = None - self.host = None - self.user = None - self.password = None - self.authorization_password = None - self.key = None + self.account_hash: int | None = None + self.host: Host | None = None + self.user: str | None = None + self.password: str | None = None + self.authorization_password: str | None = None + self.key: PrivateKey | None = None @staticmethod - def for_host(parent, host): + def for_host(parent: '_ConnectionBase', host: 'Host') -> 'AccountProxy | None': """ Returns a new AccountProxy that has an account acquired. The account is chosen based on what the connected AccountManager - selects for the given host. + selects for the given host. Returns None if the account manager + returns no account. """ account = AccountProxy(parent) account.host = host @@ -237,11 +264,11 @@ def for_host(parent, host): return None @staticmethod - def for_account_hash(parent, account_hash): + def for_account_hash(parent: '_ConnectionBase', account_hash: int) -> 'AccountProxy | None': """ Returns a new AccountProxy that acquires the account with the given hash, if such an account is known to the account manager. - It is an error if the account manager does not have such an + Returns None if the account manager does not have such an account. """ account = AccountProxy(parent) @@ -251,41 +278,51 @@ def for_account_hash(parent, account_hash): return None @staticmethod - def for_random_account(parent): + def for_random_account(parent: '_ConnectionBase') -> 'AccountProxy | None': """ Returns a new AccountProxy that has an account acquired. The - account is chosen by the connected AccountManager. + account is chosen by the connected AccountManager. Returns None + if the account manager returns no account. """ account = AccountProxy(parent) if account.acquire(): return account return None - def __hash__(self): + def __hash__(self) -> int: """ - Returns the hash of the currently acquired account. + Returns the hash of the currently acquired account. Until an + account was acquired, the hash is None, and hash() raises a + TypeError. """ - return self.account_hash + # account_hash is None until an account is acquired; the cast keeps + # the int return type that hash() requires. + return cast(int, self.account_hash) - def __enter__(self): + def __enter__(self) -> bool: """ Like :class:`acquire()`. """ return self.acquire() - def __exit__(self, thetype, value, traceback): + def __exit__( + self, + thetype: type[BaseException] | None, + value: BaseException | None, + traceback: TracebackType | None, + ) -> None: """ Like :class:`release()`. """ return self.release() - def context(self): + def context(self) -> Context: """ When you need a 'with' context for an already-acquired account. """ return Context(self) - def acquire(self): + def acquire(self) -> bool: """ Locks the account. Returns True on success, False if the account is thread-local and must not be locked. @@ -308,7 +345,7 @@ def acquire(self): ) return True - def release(self): + def release(self) -> None: """ Unlocks the account. """ @@ -321,34 +358,35 @@ def release(self): if response != 'ok': raise ValueError('unexpected response: ' + repr(response)) - def get_name(self): + def get_name(self) -> str | None: """ Returns the name of the account. - :rtype: string - :return: The account name. + :rtype: string|None + :return: The account name, or None if no account was acquired. """ return self.user - def get_password(self): + def get_password(self) -> str | None: """ Returns the password of the account. - :rtype: string - :return: The account password. + :rtype: string|None + :return: The account password, or None if it is unknown. """ return self.password - def get_authorization_password(self): + def get_authorization_password(self) -> str | None: """ - Returns the authorization password of the account. + Returns the authorization password of the account, or the + authentication password if no authorization password is set. - :rtype: string + :rtype: string|None :return: The account password. """ return self.authorization_password or self.password - def get_key(self): + def get_key(self) -> PrivateKey | None: """ Returns the key of the account, if any. @@ -364,30 +402,53 @@ class LoggerProxy: process. """ - def __init__(self, parent, logger_id): + def __init__(self, parent: '_ConnectionBase', logger_id: int) -> None: """ Constructor. :type parent: multiprocessing.Connection :param parent: A pipe to the associated pipe handler. + :param logger_id: The id() of the Logger in the other process. """ self.parent = parent self.logger_id = logger_id - def add_log(self, job_id, name, attempt): + def add_log(self, job_id: int, name: str, attempt: int) -> Log | None: + """ + Asks the logger in the other process to create a log for a new + attempt of the given job. + + :return: A copy of the new log, or None if the logger no longer + exists. + """ self.parent.send(('log-add', (self.logger_id, job_id, name, attempt))) response = self.parent.recv() if isinstance(response, Exception): raise response - return response + # The pipe returns unpickled objects without a static type. + return cast(Log | None, response) - def log(self, job_id, message): + def log(self, job_id: int, message: str) -> None: + """ + Sends the message to the log of the given job in the other process. + """ self.parent.send(('log-message', (self.logger_id, job_id, message))) - def log_aborted(self, job_id, exc_info): + def log_aborted( + self, job_id: int, exc_info: tuple[type[BaseException], BaseException, str] + ) -> None: + """ + Tells the logger in the other process that the given job failed. + + :param exc_info: The exception, as returned by + :func:`Exscript.util.impl.serializeable_sys_exc_info()`. + """ self.parent.send(('log-aborted', (self.logger_id, job_id, exc_info))) - def log_succeeded(self, job_id): + def log_succeeded(self, job_id: int) -> None: + """ + Tells the logger in the other process that the given job succeeded. + """ self.parent.send(('log-succeeded', (self.logger_id, job_id))) @@ -396,22 +457,26 @@ class AccountPool: This class manages a collection of available accounts. """ - def __init__(self, accounts=None): + def __init__(self, accounts: Account | Iterable[Account] | None = None) -> None: """ Constructor. :type accounts: Account|list[Account] :param accounts: Passed to add_account() """ - self.accounts = set() - self.unlocked_accounts = deque() - self.owner2account = defaultdict(list) - self.account2owner = dict() + self.accounts: set[Account] = set() + self.unlocked_accounts: deque[Account] = deque() + self.owner2account: defaultdict[Hashable, list[Account]] = defaultdict(list) + self.account2owner: dict[Account, Hashable] = dict() self.unlock_cond = multiprocessing.Condition(multiprocessing.RLock()) if accounts: self.add_account(accounts) - def _on_account_acquired(self, account): + def _on_account_acquired(self, account: Account) -> Account: + """ + Called when the given account was acquired outside of the pool; + marks it as locked. + """ with self.unlock_cond: if account not in self.accounts: msg = 'attempt to acquire unknown account %s' % account @@ -422,7 +487,11 @@ def _on_account_acquired(self, account): self.unlock_cond.notify_all() return account - def _on_account_released(self, account): + def _on_account_released(self, account: Account) -> Account: + """ + Called when the given account was released; marks it as unlocked + and forgets its owner. + """ with self.unlock_cond: if account not in self.accounts: msg = 'attempt to acquire unknown account %s' % account @@ -437,7 +506,7 @@ def _on_account_released(self, account): self.unlock_cond.notify_all() return account - def get_account_from_hash(self, account_hash): + def get_account_from_hash(self, account_hash: int) -> Account | None: """ Returns the account with the given hash, or None if no such account is included in the account pool. @@ -447,7 +516,7 @@ def get_account_from_hash(self, account_hash): return account return None - def has_account(self, account): + def has_account(self, account: Account) -> bool: """ Returns True if the given account exists in the pool, returns False otherwise. @@ -457,7 +526,7 @@ def has_account(self, account): """ return account in self.accounts - def add_account(self, accounts): + def add_account(self, accounts: Account | Iterable[Account]) -> None: """ Adds one or more account instances to the pool. @@ -472,8 +541,11 @@ def add_account(self, accounts): self.unlocked_accounts.append(account) self.unlock_cond.notify_all() - def _remove_account(self, accounts): + def _remove_account(self, accounts: Account | Iterable[Account]) -> None: """ + Removes one or more unlocked accounts from the pool and stops + listening to their events. + :type accounts: Account|list[Account] :param accounts: The accounts to be removed. """ @@ -488,7 +560,7 @@ def _remove_account(self, accounts): self.accounts.remove(account) self.unlocked_accounts.remove(account) - def reset(self): + def reset(self) -> None: """ Removes all accounts. """ @@ -498,9 +570,10 @@ def reset(self): self._remove_account(self.accounts.copy()) self.unlock_cond.notify_all() - def get_account_from_name(self, name): + def get_account_from_name(self, name: str) -> Account | None: """ - Returns the account with the given name. + Returns the account with the given name, or None if no such + account is included in the account pool. :type name: string :param name: The name of the account. @@ -510,13 +583,15 @@ def get_account_from_name(self, name): return account return None - def n_accounts(self): + def n_accounts(self) -> int: """ Returns the number of accounts that are currently in the pool. """ return len(self.accounts) - def acquire_account(self, account=None, owner=None): + def acquire_account( + self, account: Account | None = None, owner: Hashable | None = None + ) -> Account: """ Waits until an account becomes available, then locks and returns it. If an account is not passed, the next available account is returned. @@ -550,7 +625,7 @@ def acquire_account(self, account=None, owner=None): self.unlock_cond.notify_all() return account - def release_accounts(self, owner): + def release_accounts(self, owner: Hashable) -> None: """ Releases all accounts that were acquired by the given owner. @@ -572,22 +647,25 @@ class AccountManager: worker threads. """ - def __init__(self): + def __init__(self) -> None: """ Constructor. """ - self.default_pool = None - self.pools = None + # reset() sets both attributes; the casts are for the type checker. + self.default_pool: AccountPool = cast(AccountPool, None) + self.pools: list[tuple[Callable[[Host], bool], AccountPool]] = cast( + 'list[tuple[Callable[[Host], bool], AccountPool]]', None + ) self.reset() - def reset(self): + def reset(self) -> None: """ Removes all account pools. """ self.default_pool = AccountPool() self.pools = [] - def add_pool(self, pool, match=None): + def add_pool(self, pool: AccountPool, match: 'Callable[[Host], bool] | None' = None) -> None: """ Adds a new account pool. If the given match argument is None, the pool is the default pool. Otherwise, the match argument is @@ -650,22 +728,22 @@ def use_this_pool(host): else: self.pools.append((match, pool)) - def add_account(self, account): + def add_account(self, account: Account | Iterable[Account]) -> None: """ Adds the given account to the default account pool that Exscript uses to log into all hosts that have no specific :class:`Account` attached. - :type account: Account - :param account: The account that is added. + :type account: Account|list[Account] + :param account: The account or accounts that are added. """ self.default_pool.add_account(account) - def get_account_from_hash(self, account_hash): + def get_account_from_hash(self, account_hash: int) -> Account | None: """ Returns the account with the given hash, if it is contained in any of the pools. Returns None otherwise. - :type account_hash: str + :type account_hash: int :param account_hash: The hash of an account object. """ for _, pool in self.pools: @@ -674,13 +752,15 @@ def get_account_from_hash(self, account_hash): return account return self.default_pool.get_account_from_hash(account_hash) - def acquire_account(self, account=None, owner=None): + def acquire_account( + self, account: Account | None = None, owner: Hashable | None = None + ) -> Account: """ Acquires the given account. If no account is given, one is chosen from the default pool. :type account: Account - :param account: The account that is added. + :param account: The account to be acquired, or None. :type owner: object :param owner: An optional descriptor for the owner. :rtype: :class:`Account` @@ -698,7 +778,7 @@ def acquire_account(self, account=None, owner=None): return self.default_pool.acquire_account(account, owner) - def acquire_account_for(self, host, owner=None): + def acquire_account_for(self, host: 'Host', owner: Hashable | None = None) -> Account: """ Acquires an account for the given host and returns it. The host is passed to each of the match functions that were @@ -720,7 +800,7 @@ def acquire_account_for(self, host, owner=None): # Else, choose an account from the default account pool. return self.default_pool.acquire_account(owner=owner) - def release_accounts(self, owner): + def release_accounts(self, owner: Hashable) -> None: """ Releases all accounts that were acquired by the given owner. diff --git a/src/Exscript/host.py b/src/Exscript/host.py index b2743388..f4bc702e 100644 --- a/src/Exscript/host.py +++ b/src/Exscript/host.py @@ -24,6 +24,9 @@ Representing a device to connect with. """ +from collections.abc import Mapping +from typing import Any, cast + from .account import Account from .util.cast import to_list from .util.ipv4 import clean_ip, is_ip @@ -37,7 +40,7 @@ class Host: __slots__ = ('protocol', 'vars', 'account', 'name', 'address', 'tcp_port', 'options') - def __init__(self, uri, default_protocol='ssh'): + def __init__(self, uri: str, default_protocol: str = 'ssh') -> None: """ Constructor. The given uri is passed to Host.set_uri(). The default_protocol argument defines the protocol that is used @@ -49,20 +52,23 @@ def __init__(self, uri, default_protocol='ssh'): :param default_protocol: The protocol name. """ self.protocol = default_protocol - self.vars = None # To save memory, do not init with a dict. - self.account = None - self.name = None - self.address = None - self.tcp_port = None - self.options = None + # Variables and options hold values of any type: variables are lists + # of strings in templates, and each option has its own type. + self.vars: dict[str, Any] | None = None # To save memory, do not init with a dict. + self.account: Account | None = None + # set_uri() sets the name and the address; the casts are for the type checker. + self.name: str = cast(str, None) + self.address: str = cast(str, None) + self.tcp_port: int | None = None + self.options: dict[str, Any] | None = None self.set_uri(uri) - def __copy__(self): + def __copy__(self) -> 'Host': host = Host(self.get_uri()) host.set_name(self.get_name()) return host - def set_uri(self, uri): + def set_uri(self, uri: str) -> None: """ Defines the protocol, hostname/address, TCP port number, username, and password from the given URL. The hostname may be URL formatted, @@ -83,24 +89,27 @@ def set_uri(self, uri): :param uri: An URL formatted hostname. """ try: - uri = Url.from_string(uri, self.protocol) + url = Url.from_string(uri, self.protocol) except ValueError: raise ValueError('Hostname parse error: ' + repr(uri)) - hostname = uri.hostname or '' - name = uri.path and hostname + uri.path or hostname - self.set_protocol(uri.protocol) - self.set_tcp_port(uri.port) + # The casts are for the type checker: from_string() always sets the + # protocol and the variables, and it sets the user name if it sets a + # password. + hostname = url.hostname or '' + name = url.path and hostname + url.path or hostname + self.set_protocol(cast(str, url.protocol)) + self.set_tcp_port(url.port) self.set_name(name) self.set_address(name) - if uri.username is not None or uri.password1 is not None or uri.password2: - account = Account(uri.username, uri.password1, uri.password2) + if url.username is not None or url.password1 is not None or url.password2: + account = Account(cast(str, url.username), url.password1, url.password2) self.set_account(account) - for key, val in list(uri.vars.items()): + for key, val in list(cast(dict[str, list[Any]], url.vars).items()): self.set(key, val) - def get_uri(self): + def get_uri(self) -> str: """ Returns a URI formatted representation of the host, including all of its attributes except for the name. Uses the @@ -126,7 +135,7 @@ def get_uri(self): return str(url) - def get_dict(self): + def get_dict(self) -> dict[str, str | int | None]: """ Returns a dict containing the host's attributes. The following keys are contained: @@ -146,7 +155,7 @@ def get_dict(self): 'port': self.get_tcp_port(), } - def set_name(self, name): + def set_name(self, name: str) -> None: """ Set the hostname of the remote host without changing username, password, protocol, and TCP port number. @@ -156,16 +165,16 @@ def set_name(self, name): """ self.name = name - def get_name(self): + def get_name(self) -> str: """ Returns the name. :rtype: string - :return: The hostname excluding the name. + :return: The name of the host. """ return self.name - def set_address(self, address): + def set_address(self, address: str) -> None: """ Set the address of the remote host that is contacted, without changing hostname, username, password, protocol, and TCP port @@ -180,7 +189,7 @@ def set_address(self, address): else: self.address = address - def get_address(self): + def get_address(self) -> str: """ Returns the address that is used to open the connection. @@ -189,7 +198,7 @@ def get_address(self): """ return self.address - def set_protocol(self, protocol): + def set_protocol(self, protocol: str) -> None: """ Defines the protocol. The following protocols are currently supported: @@ -197,6 +206,7 @@ def set_protocol(self, protocol): - telnet: Telnet - ssh: SSH version 2 - ssh2: SSH version 2 + - netconf: NETCONF over SSH - dummy: A virtual device that accepts any command, but that does not respond anything useful. - pseudo: A virtual device that loads a file with the given @@ -211,7 +221,7 @@ def set_protocol(self, protocol): """ self.protocol = protocol - def get_protocol(self): + def get_protocol(self) -> str: """ Returns the name of the protocol. @@ -220,7 +230,7 @@ def get_protocol(self): """ return self.protocol - def set_option(self, name, value): + def set_option(self, name: str, value: Any) -> None: """ Defines a (possibly protocol-specific) option for the host. Possible options include: @@ -271,7 +281,7 @@ def set_option(self, name, value): self.options = {} self.options[name] = value - def get_option(self, name, default=None): + def get_option(self, name: str, default: Any = None) -> Any: """ Returns the value of the given option if it is defined, returns the given default value otherwise. @@ -285,7 +295,7 @@ def get_option(self, name, default=None): return default return self.options.get(name, default) - def get_options(self): + def get_options(self) -> dict[str, Any]: """ Return a dictionary containing all defined options. @@ -296,46 +306,46 @@ def get_options(self): return {} return self.options - def set_tcp_port(self, tcp_port): + def set_tcp_port(self, tcp_port: int | None) -> None: """ Defines the TCP port number. - :type tcp_port: int - :param tcp_port: The TCP port number. + :type tcp_port: int|None + :param tcp_port: The TCP port number, or None. """ if tcp_port is None: self.tcp_port = None return self.tcp_port = int(tcp_port) - def get_tcp_port(self): + def get_tcp_port(self) -> int | None: """ Returns the TCP port number. - :rtype: string - :return: The TCP port number. + :rtype: int|None + :return: The TCP port number, or None if it is not known. """ return self.tcp_port - def set_account(self, account): + def set_account(self, account: Account | None) -> None: """ Defines the account that is used to log in. :type account: :class:`Exscript.Account` - :param account: The account. + :param account: The account, or None. """ self.account = account - def get_account(self): + def get_account(self) -> Account | None: """ Returns the account that is used to log in. - :rtype: Account - :return: The account. + :rtype: Account|None + :return: The account, or None if no account is attached. """ return self.account - def set(self, name, value): + def set(self, name: str, value: Any) -> None: """ Stores the given variable/value in the object for later retrieval. @@ -348,7 +358,7 @@ def set(self, name, value): self.vars = {} self.vars[name] = value - def set_all(self, variables): + def set_all(self, variables: Mapping[str, Any]) -> None: """ Like set(), but replaces all variables by using the given dictionary. In other words, passing an empty dictionary @@ -359,7 +369,7 @@ def set_all(self, variables): """ self.vars = dict(variables) - def append(self, name, value): + def append(self, name: str, value: Any) -> None: """ Appends the given value to the list variable with the given name. @@ -375,7 +385,7 @@ def append(self, name, value): else: self.vars[name] = [value] - def set_default(self, name, value): + def set_default(self, name: str, value: Any) -> None: """ Like set(), but only sets the value if the variable is not already defined. @@ -390,7 +400,7 @@ def set_default(self, name, value): if name not in self.vars: self.vars[name] = value - def has_key(self, name): + def has_key(self, name: str) -> bool: """ Returns True if the variable with the given name is defined, False otherwise. @@ -404,7 +414,7 @@ def has_key(self, name): return False return name in self.vars - def get(self, name, default=None): + def get(self, name: str, default: Any = None) -> Any: """ Returns the value of the given variable, or the given default value if the variable is not defined. @@ -420,7 +430,7 @@ def get(self, name, default=None): return default return self.vars.get(name, default) - def get_all(self): + def get_all(self) -> dict[str, Any]: """ Returns a dictionary containing all variables. diff --git a/src/Exscript/key.py b/src/Exscript/key.py index 844aecfc..3178d296 100644 --- a/src/Exscript/key.py +++ b/src/Exscript/key.py @@ -36,7 +36,7 @@ class PrivateKey: keytypes: set[str] = set() - def __init__(self, keytype='rsa'): + def __init__(self, keytype: str = 'rsa') -> None: """ Constructor. Supported key types are provided by their respective protocol adapters and can be retrieved from the PrivateKey.keytypes @@ -48,19 +48,21 @@ class attribute. if keytype not in self.keytypes: raise TypeError('unsupported key type: ' + repr(keytype)) self.keytype = keytype - self.filename = None - self.password = None + self.filename: str | None = None + self.password: str | None = None @staticmethod - def from_file(filename, password='', keytype=None): + def from_file( + filename: str, password: str | None = '', keytype: str | None = None + ) -> 'PrivateKey': """ Returns a new PrivateKey instance with the given attributes. If keytype is None, we attempt to automatically detect the type. :type filename: string :param filename: The key file name. - :type password: string - :param password: The key password. + :type password: string|None + :param password: The key password, or None if the key is not encrypted. :type keytype: string :param keytype: The key type. :rtype: PrivateKey @@ -86,16 +88,16 @@ def from_file(filename, password='', keytype=None): key.password = password return key - def get_type(self): + def get_type(self) -> str: """ - Returns the type of the key, e.g. RSA or DSA. + Returns the type of the key, e.g. 'rsa' or 'ed25519'. :rtype: string :return: The key type """ return self.keytype - def set_filename(self, filename): + def set_filename(self, filename: str) -> None: """ Sets the name of the key file to use. @@ -104,29 +106,29 @@ def set_filename(self, filename): """ self.filename = filename - def get_filename(self): + def get_filename(self) -> str | None: """ Returns the name of the key file. - :rtype: string - :return: The key password. + :rtype: string|None + :return: The key filename, or None if none was set. """ return self.filename - def set_password(self, password): + def set_password(self, password: str | None) -> None: """ Defines the password used for decrypting the key. - :type password: string - :param password: The key password. + :type password: string|None + :param password: The key password, or None if the key is not encrypted. """ self.password = password - def get_password(self): + def get_password(self) -> str | None: """ Returns the password for the key. - :rtype: string - :return: The key password. + :rtype: string|None + :return: The key password, or None if none was set. """ return self.password diff --git a/src/Exscript/logger.py b/src/Exscript/logger.py index 2b4bdbfc..aa2fdd63 100644 --- a/src/Exscript/logger.py +++ b/src/Exscript/logger.py @@ -31,11 +31,19 @@ import time import weakref from collections import defaultdict +from collections.abc import Callable from io import StringIO from itertools import chain +from types import TracebackType +from typing import TYPE_CHECKING, cast from .util.impl import format_exception +if TYPE_CHECKING: + # _ConnectionBase is the common base class of the pipe connections on + # POSIX (Connection) and on Windows (PipeConnection). + from multiprocessing.connection import _ConnectionBase + # ----------------------------------------------------------------------------- # Python Logging @@ -43,11 +51,18 @@ LOGGING_LEVEL = logging.DEBUG -def getConsoleLogger(name, *, level=LOGGING_LEVEL, fmt=LOGGING_FORMAT, utc=True): +def getConsoleLogger( + name: str, *, level: int | str = LOGGING_LEVEL, fmt: str = LOGGING_FORMAT, utc: bool = True +) -> logging.Logger: """Get the console logger with the given name. This function is not related to the host logging mechanics of Exscript. Instead it configures the logging for the particular Python logger instance. + If the logger has a console handler already, it is returned unchanged. + + :param level: The level of the console handler. + :param fmt: The format of the console handler. + :param utc: Whether the timestamps are in UTC instead of local time. """ logger = logging.getLogger(name) @@ -69,19 +84,42 @@ def getConsoleLogger(name, *, level=LOGGING_LEVEL, fmt=LOGGING_FORMAT, utc=True) return logger -def setupLogging(*, package=False, parser=False, protocol=False, init=getConsoleLogger): +def setupLogging( + *, + package: bool = False, + parser: bool = False, + protocol: bool = False, + init: Callable[[str], logging.Logger] = getConsoleLogger, +) -> None: """Setup the package logging. By default the setup adds a console/terminal handler to make the logs visible. The relevant loggers get filtered according to the boolean flags. + + :param package: Whether to show the logs of the Exscript package. + :param parser: Whether to show the logs of the template parser and lexer. + :param protocol: Whether to show the logs of the protocols, the servers, + ncclient and paramiko. + :param init: A function that returns the configured logger with the given + name; the filters are added to the handlers of that logger. """ class _LogFilter: - def __init__(self, name, enabled): + """ + Passes the records of the named logger and its children if enabled. + """ + + def __init__(self, name: str, enabled: bool) -> None: + """ + Constructor. + """ self.name = name self.enabled = enabled - def filter(self, record): + def filter(self, record: logging.LogRecord) -> bool: + """ + Returns whether the given record passes the filter. + """ if record.name == self.name: return self.enabled if record.name.startswith(self.name + "."): @@ -122,25 +160,46 @@ def filter(self, record): class Log: - def __init__(self, name): + """ + Records the output of one attempt of a job in memory, and whether and + how the attempt ended. + """ + + def __init__(self, name: str) -> None: + """ + Creates an empty log with the given name. + """ self.name = name self.data = StringIO('') - self.exc_info = None + self.exc_info: tuple[type[BaseException], BaseException, TracebackType | str] | None = None self.did_end = False - def __str__(self): + def __str__(self) -> str: return self.data.getvalue() - def __len__(self): + def __len__(self) -> int: return len(str(self)) - def get_name(self): + def get_name(self) -> str: + """ + Returns the name of the log. + """ return self.name - def write(self, *data): + def write(self, *data: str) -> None: + """ + Appends the given strings to the log, separated by spaces. + """ self.data.write(' '.join(data)) - def get_error(self, include_tb=True): + def get_error(self, include_tb: bool = True) -> str | None: + """ + Returns the error that ended the log, or None if there was none. + + :param include_tb: Whether to return the formatted exception with + the traceback. Otherwise the exception message is returned, or + the name of the exception class if the message is empty. + """ if self.exc_info is None: return None if include_tb: @@ -149,30 +208,41 @@ def get_error(self, include_tb=True): return str(self.exc_info[1]) return self.exc_info[0].__name__ - def started(self): + def started(self) -> None: """ Called by a logger to inform us that logging may now begin. """ self.did_end = False - def aborted(self, exc_info): + def aborted( + self, exc_info: tuple[type[BaseException], BaseException, TracebackType | str] + ) -> None: """ Called by a logger to log an exception. + + :param exc_info: The exception, as returned by sys.exc_info(), or + with the traceback formatted as a string. """ self.exc_info = exc_info self.did_end = True self.write(format_exception(*self.exc_info)) - def succeeded(self): + def succeeded(self) -> None: """ Called by a logger to inform us that logging is complete. """ self.did_end = True - def has_error(self): + def has_error(self) -> bool: + """ + Returns True if the log ended with an error, False otherwise. + """ return self.exc_info is not None - def has_ended(self): + def has_ended(self) -> bool: + """ + Returns True if the log ended, with or without an error. + """ return self.did_end @@ -182,7 +252,16 @@ class Logfile(Log): log containing the error message with a traceback. """ - def __init__(self, name, filename, mode='a', delete=False): + def __init__(self, name: str, filename: str, mode: str = 'a', delete: bool = False) -> None: + """ + Creates a log that writes to the given file, and the error with the + traceback to the same file name with the suffix '.error'. The + directory of the file is created if it does not exist. + + :param mode: The mode in which the files are opened for each write. + :param delete: Whether to delete the file when the log succeeds + without an error. + """ Log.__init__(self, name) self.filename = filename self.errorname = filename + '.error' @@ -197,7 +276,7 @@ def __init__(self, name, filename, mode='a', delete=False): if e.errno != errno.EEXIST: raise - def __str__(self): + def __str__(self) -> str: data = '' if os.path.isfile(self.filename): with open(self.filename, encoding="utf-8") as thefile: @@ -207,7 +286,12 @@ def __str__(self): data += thefile.read() return data - def _write_file(self, filename, *data): + def _write_file(self, filename: str, *data: str) -> None: + """ + Writes the given strings to the given file, separated by spaces. + If writing fails, the error is raised, and later writes are + skipped. + """ if not self.do_log: return try: @@ -218,22 +302,41 @@ def _write_file(self, filename, *data): self.do_log = False raise - def write(self, *data): + def write(self, *data: str) -> None: + """ + Writes the given strings to the log file, separated by spaces. + """ return self._write_file(self.filename, *data) - def _write_error(self, *data): + def _write_error(self, *data: str) -> None: + """ + Writes the given strings to the error file, separated by spaces. + """ return self._write_file(self.errorname, *data) - def started(self): + def started(self) -> None: + """ + Creates the log file. + """ self.write('') # Creates the file. - def aborted(self, exc_info): + def aborted( + self, exc_info: tuple[type[BaseException], BaseException, TracebackType | str] + ) -> None: + """ + Writes the error message to the log file, and the error with the + traceback to the error file. + """ self.exc_info = exc_info self.did_end = True self.write('ERROR:', str(exc_info[1]), '\n') self._write_error(format_exception(*self.exc_info)) - def succeeded(self): + def succeeded(self) -> None: + """ + Marks the log as ended. If the log was created with delete=True and + has no error, deletes the log file instead. + """ if self.delete and not self.has_error(): os.remove(self.filename) return @@ -246,48 +349,70 @@ class Logger: job. Logs are kept in memory, and not written to the disk. """ - def __init__(self): + def __init__(self) -> None: """ Creates a new logger instance. Use the :class:`Exscript.util.log.log_to` decorator to send messages to the logger. """ logger_registry[id(self)] = self self._lock = threading.Lock() - self.logs = defaultdict(list) + self.logs: defaultdict[int, list[Log]] = defaultdict(list) self.started = 0 self.success = 0 self.failed = 0 - def _reset(self): + def _reset(self) -> None: + """ + Forgets all logs. + """ self.logs = defaultdict(list) - def get_succeeded_actions(self): + def get_succeeded_actions(self) -> int: """ Returns the number of jobs that were completed successfully. """ return self.success - def get_aborted_actions(self): + def get_aborted_actions(self) -> int: """ Returns the number of jobs that were aborted. """ return self.failed - def get_logs(self): + def get_logs(self) -> list[Log]: + """ + Returns the logs of all attempts of all jobs. + """ return list(chain.from_iterable(iter(self.logs.values()))) - def get_succeeded_logs(self): + def get_succeeded_logs(self) -> list[Log]: + """ + Returns the logs that ended without an error. + """ func = lambda x: x.has_ended() and not x.has_error() return list(filter(func, self.get_logs())) - def get_aborted_logs(self): + def get_aborted_logs(self) -> list[Log]: + """ + Returns the logs that ended with an error. + """ func = lambda x: x.has_ended() and x.has_error() return list(filter(func, self.get_logs())) - def _get_log(self, job_id): + def _get_log(self, job_id: int) -> Log: + """ + Returns the log of the last attempt of the given job. + """ return self.logs[job_id][-1] - def add_log(self, job_id, name, attempt): + def add_log(self, job_id: int, name: str, attempt: int) -> Log: + """ + Creates the log for a new attempt of the given job, and returns it. + + :param job_id: The id() of the job. + :param name: The name of the log. + :param attempt: The number of the attempt, starting at 1. + """ log = Log(name) log.started() self.logs[job_id].append(log) @@ -295,19 +420,31 @@ def add_log(self, job_id, name, attempt): self.started += 1 return log - def log(self, job_id, message): + def log(self, job_id: int, message: str) -> None: + """ + Appends the message to the log of the last attempt of the given job. + """ # This method is called whenever a sub thread sends a log message # via a pipe. (See LoggerProxy and Queue.PipeHandler) log = self._get_log(job_id) log.write(message) - def log_aborted(self, job_id, exc_info): + def log_aborted( + self, job_id: int, exc_info: tuple[type[BaseException], BaseException, TracebackType | str] + ) -> None: + """ + Records that the last attempt of the given job failed with the + given exception. + """ log = self._get_log(job_id) log.aborted(exc_info) with self._lock: self.failed += 1 - def log_succeeded(self, job_id): + def log_succeeded(self, job_id: int) -> None: + """ + Records that the last attempt of the given job succeeded. + """ log = self._get_log(job_id) log.succeeded() with self._lock: @@ -320,34 +457,57 @@ class LoggerProxy: process. """ - def __init__(self, parent, logger_id): + def __init__(self, parent: '_ConnectionBase', logger_id: int) -> None: """ Constructor. :type parent: multiprocessing.Connection :param parent: A pipe to the associated pipe handler. + :param logger_id: The id() of the Logger in the other process. """ self.parent = parent self.logger_id = logger_id self._lock = threading.Lock() - def add_log(self, job_id, name, attempt): + def add_log(self, job_id: int, name: str, attempt: int) -> Log | None: + """ + Asks the logger in the other process to create a log for a new + attempt of the given job. + + :return: A copy of the new log, or None if the logger no longer + exists. + """ with self._lock: self.parent.send(('log-add', (self.logger_id, job_id, name, attempt))) response = self.parent.recv() if isinstance(response, Exception): raise response - return response + # The pipe returns unpickled objects without a static type. + return cast(Log | None, response) - def log(self, job_id, message): + def log(self, job_id: int, message: str) -> None: + """ + Sends the message to the log of the given job in the other process. + """ with self._lock: self.parent.send(('log-message', (self.logger_id, job_id, message))) - def log_aborted(self, job_id, exc_info): + def log_aborted( + self, job_id: int, exc_info: tuple[type[BaseException], BaseException, str] + ) -> None: + """ + Tells the logger in the other process that the given job failed. + + :param exc_info: The exception, as returned by + :func:`Exscript.util.impl.serializeable_sys_exc_info()`. + """ with self._lock: self.parent.send(('log-aborted', (self.logger_id, job_id, exc_info))) - def log_succeeded(self, job_id): + def log_succeeded(self, job_id: int) -> None: + """ + Tells the logger in the other process that the given job succeeded. + """ with self._lock: self.parent.send(('log-succeeded', (self.logger_id, job_id))) @@ -357,7 +517,9 @@ class FileLogger(Logger): A Logger that stores logs into files. """ - def __init__(self, logdir, mode='a', delete=False, clearmem=True): + def __init__( + self, logdir: str, mode: str = 'a', delete: bool = False, clearmem: bool = True + ) -> None: """ The logdir argument specifies the location where the logs are stored. The mode specifies whether to append the existing logs @@ -375,7 +537,16 @@ def __init__(self, logdir, mode='a', delete=False, clearmem=True): if not os.path.exists(self.logdir): os.mkdir(self.logdir) - def add_log(self, job_id, name, attempt): + def add_log(self, job_id: int, name: str, attempt: int) -> Logfile: + """ + Creates the log file for a new attempt of the given job, and returns + the log. The file of a retry gets the suffix '_retryN', where N is + the number of the retry. + + :param job_id: The id() of the job. + :param name: The name of the log, also used for the file name. + :param attempt: The number of the attempt, starting at 1. + """ if attempt > 1: name += '_retry%d' % (attempt - 1) filename = os.path.join(self.logdir, name + '.log') @@ -384,12 +555,22 @@ def add_log(self, job_id, name, attempt): self.logs[job_id].append(log) return log - def log_aborted(self, job_id, exc_info): + def log_aborted( + self, job_id: int, exc_info: tuple[type[BaseException], BaseException, TracebackType | str] + ) -> None: + """ + Like :meth:`Logger.log_aborted`, but forgets the logs of the job if + clearmem is True. + """ Logger.log_aborted(self, job_id, exc_info) if self.clearmem: self.logs.pop(job_id) - def log_succeeded(self, job_id): + def log_succeeded(self, job_id: int) -> None: + """ + Like :meth:`Logger.log_succeeded`, but forgets the logs of the job if + clearmem is True. + """ Logger.log_succeeded(self, job_id) if self.clearmem: self.logs.pop(job_id) From 3d96c990c65281d67ed97fafda346fe53898a035 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 4 Nov 2026 14:01:08 +0100 Subject: [PATCH 060/250] Annotate the protocol core with types and docstrings Protocol, its exceptions, the OS guesser, the Driver base class, the driver registry and util.match are annotated for mypy --strict. Only annotations, docstrings and typing.cast() are added. --- src/Exscript/protocols/drivers/__init__.py | 22 +- src/Exscript/protocols/drivers/driver.py | 128 ++++- src/Exscript/protocols/exception.py | 27 +- src/Exscript/protocols/osguesser.py | 47 +- src/Exscript/protocols/protocol.py | 585 +++++++++++++++------ src/Exscript/util/match.py | 16 +- 6 files changed, 608 insertions(+), 217 deletions(-) diff --git a/src/Exscript/protocols/drivers/__init__.py b/src/Exscript/protocols/drivers/__init__.py index 0a377247..d373ee85 100644 --- a/src/Exscript/protocols/drivers/__init__.py +++ b/src/Exscript/protocols/drivers/__init__.py @@ -20,7 +20,14 @@ # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" +The drivers for the supported operating systems, and the registry of drivers +that the OS guesser and the driver arguments of the protocols use. +""" + import inspect +from collections.abc import Callable +from typing import TypeGuard, cast from .ace import ACEDriver as ACEDriver from .adtran import AdtranDriver as AdtranDriver @@ -31,7 +38,7 @@ from .bigip import BigIPDriver as BigIPDriver from .brocade import BrocadeDriver as BrocadeDriver from .cienasaos import CienaSAOSDriver as CienaSAOSDriver -from .driver import Driver +from .driver import Driver as Driver from .enterasys import EnterasysDriver as EnterasysDriver from .enterasys_wc import EnterasysWCDriver as EnterasysWCDriver from .eos import EOSDriver as EOSDriver @@ -63,11 +70,15 @@ driver_map: dict[str, Driver] = {} -def isdriver(o): +def isdriver(o: object) -> TypeGuard[type[Driver]]: + """ + Returns whether the given object is a subclass of Driver, other than + Driver itself. + """ return inspect.isclass(o) and issubclass(o, Driver) and o is not Driver -def add_driver(cls): +def add_driver(cls: type[Driver]) -> None: """ Registers a driver class, so that the OS guesser and ``driver=`` can use it in connections that are created afterwards. @@ -75,13 +86,14 @@ def add_driver(cls): :type cls: type[Driver] :param cls: The driver class; it is instantiated without arguments. """ - driver = cls() + # Unlike Driver itself, driver classes take no constructor arguments. + driver = cast(Callable[[], Driver], cls)() driver_classes.append(cls) drivers.append(driver) driver_map[driver.name] = driver -def disable_driver(name): +def disable_driver(name: str) -> None: """ Unregisters the driver with the given name. Connections that are created afterwards neither guess it nor accept its name. diff --git a/src/Exscript/protocols/drivers/driver.py b/src/Exscript/protocols/drivers/driver.py index 182351f1..4be6af20 100644 --- a/src/Exscript/protocols/drivers/driver.py +++ b/src/Exscript/protocols/drivers/driver.py @@ -26,6 +26,12 @@ import re import string +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from ...account import Account, AccountProxy + from ...util.impl import _Context + from ..protocol import Protocol _flags = re.I _printable = re.escape(string.printable) @@ -105,7 +111,22 @@ class Driver: - def __init__(self, name): + """ + The base class of all drivers. A driver holds what is specific to one + operating system: the patterns of its prompts and error messages, how it + is recognized, and how a session is prepared, authorized and ended. The + generic driver uses this class unchanged; its patterns work with many + devices. + """ + + def __init__(self, name: str) -> None: + """ + Constructor. Sets the generic patterns; subclasses replace those that + differ. + + :param name: The name of the driver, which is also the name of the + operating system that guess_os() returns and set_driver() accepts. + """ self.name = name self.user_re = _user_re self.password_re = _pass_re @@ -115,28 +136,47 @@ def __init__(self, name): # SSH2 related: self.reconnect_between_auth_methods = False # Netconf related: - self.netconf_device_params = {} # see ncclient documentation - self.netconf_capabilities_re = [] # used to determine vendor/OS driver + self.netconf_device_params: dict[str, Any] = {} # see ncclient documentation + # used to determine vendor/OS driver + self.netconf_capabilities_re: list[re.Pattern[str]] = [] self.netconf_prompt_re = _netconf_prompt_re self.netconf_error_re = _netconf_error_re - def __repr__(self): + def __repr__(self) -> str: return f"{type(self).__name__}('{self.name}')" - def supports_os_guesser(self): + def supports_os_guesser(self) -> bool: + """ + Returns whether the driver overrides one of check_protocol_for_os(), + check_head_for_os() and check_response_for_os(), i.e. whether the OS + guesser can recognize its operating system. + """ return ( self.check_protocol_for_os.__code__ is not Driver.check_protocol_for_os.__code__ or self.check_head_for_os.__code__ is not Driver.check_head_for_os.__code__ or self.check_response_for_os.__code__ is not Driver.check_response_for_os.__code__ ) - def check_protocol_for_os(self, string): + def check_protocol_for_os(self, string: str) -> int: + """ + Returns the confidence, from 0 to 100, that the given protocol + information, e.g. the SSH version string, comes from the operating + system of this driver. The base class returns 0. + """ return 0 - def _check_protocol(self, string): + def _check_protocol(self, string: str) -> tuple[str, int]: + """ + Returns the name of the driver and the result of + check_protocol_for_os() for the given string. + """ return self.name, self.check_protocol_for_os(string) - def check_netconf_capabilities(self, string): + def check_netconf_capabilities(self, string: str) -> int: + """ + Returns 100 if one of the netconf_capabilities_re patterns matches the + given NETCONF capabilities (one per line), else 0. + """ if not self.netconf_capabilities_re: return 0 for regex in self.netconf_capabilities_re: @@ -144,32 +184,84 @@ def check_netconf_capabilities(self, string): return 100 return 0 - def _check_netconf(self, string): + def _check_netconf(self, string: str) -> tuple[str, int]: + """ + Returns the name of the driver and the result of + check_netconf_capabilities() for the given string. + """ return self.name, self.check_netconf_capabilities(string) - def check_head_for_os(self, string): + def check_head_for_os(self, string: str) -> int: + """ + Returns the confidence, from 0 to 100, that the given data, which the + host sent up to the end of the login, comes from the operating system + of this driver. The base class returns 0. + """ return 0 - def _check_head(self, string): + def _check_head(self, string: str) -> tuple[str, int]: + """ + Returns the name of the driver and the result of check_head_for_os() + for the given string. + """ return self.name, self.check_head_for_os(string) - def check_response_for_os(self, string): + def check_response_for_os(self, string: str) -> int: + """ + Returns the confidence, from 0 to 100, that the given data, which the + host sent, e.g. a prompt, comes from the operating system of this + driver. The base class returns 0. + """ return 0 - def _check_response(self, string): + def _check_response(self, string: str) -> tuple[str, int]: + """ + Returns the name of the driver and the result of + check_response_for_os() for the given string. + """ return self.name, self.check_response_for_os(string) - def clean_response_for_re_match(self, response): + def clean_response_for_re_match(self, response: str) -> tuple[str, str]: + """ + Prepares the end of the received data for matching the prompt + patterns that end with $ or \\Z, e.g. by removing control sequences. + The base class returns the data unchanged. + + :param response: The end of the received data. + :return: The cleaned data, and the incomplete rest at its end that is + not matched, e.g. the start of an escape sequence. + """ return response, '' - def init_terminal(self, conn): + def init_terminal(self, conn: 'Protocol') -> None: + """ + Prepares the session on the given connection for scripts, e.g. turns + off the pager. The base class does nothing. + """ pass - def supports_auto_authorize(self): + def supports_auto_authorize(self) -> bool: + """ + Returns whether the driver overrides auto_authorize(). + """ return self.__class__.auto_authorize is not Driver.auto_authorize - def auto_authorize(self, conn, account, flush, bailout): + def auto_authorize( + self, + conn: 'Protocol', + account: 'Account | AccountProxy | _Context | None', + flush: bool, + bailout: bool, + ) -> None: + """ + Gets the privileges that commands need, e.g. with "enable". The base + class calls conn.app_authorize(); the arguments are those of + Protocol.app_authorize(). + """ conn.app_authorize(account, flush, bailout) - def logout(self, conn): + def logout(self, conn: 'Protocol') -> None: + """ + Ends the session on the given connection. The base class sends "exit". + """ conn.send('exit\r') diff --git a/src/Exscript/protocols/exception.py b/src/Exscript/protocols/exception.py index 77fef248..08807c68 100644 --- a/src/Exscript/protocols/exception.py +++ b/src/Exscript/protocols/exception.py @@ -24,6 +24,12 @@ Network related error types. """ +from collections.abc import Iterable +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from lxml.etree import _Element + class ProtocolException(Exception): """ @@ -94,8 +100,22 @@ class RPCError(InvalidCommandException): more rpc-error elements with error severity. """ - def __init__(self, message, *, element, type, tag, severity, app_tag='', path='', info=''): + def __init__( + self, + message: str | None, + *, + element: '_Element', + type: str | None, + tag: str | None, + severity: str | None, + app_tag: str | None = '', + path: str | None = '', + info: str | None = '', + ) -> None: """ + Creates the exception for one rpc-error. The text fields may be None + if the rpc-error has no such element; the attributes are '' then. + :param message: The primary rpc-error message (error-message). :param element: The XML element of the rpc-error node. :param type: The NETCONF error-type (e.g. 'application', 'protocol'). @@ -121,11 +141,14 @@ def __init__(self, message, *, element, type, tag, severity, app_tag='', path='' self.info = info or '' self.errlist = [self] + # The errors are Any: ncclient has no type hints. @classmethod - def _from_ncclient(cls, rpc_errors): + def _from_ncclient(cls, rpc_errors: Iterable[Any]) -> 'RPCError': """ Translates a list of ncclient `RPCError` objects (from RPCReply.errors) into Exscript :class:`RPCError` instances. + + :return: The first error; its errlist holds all of them. """ all_errors = [ cls( diff --git a/src/Exscript/protocols/osguesser.py b/src/Exscript/protocols/osguesser.py index c484f133..444dca5b 100644 --- a/src/Exscript/protocols/osguesser.py +++ b/src/Exscript/protocols/osguesser.py @@ -25,6 +25,8 @@ """ import logging +from collections.abc import Iterable +from typing import Any, cast from .drivers import drivers @@ -42,7 +44,11 @@ class OsGuesser: based on the information provided. """ - def __init__(self): + def __init__(self) -> None: + """ + Constructor. Gets the checks of all drivers that are registered at + this time. + """ self._logger = _logger self._buffer_limit = 512 self._auth_os_map = [d._check_head for d in drivers] @@ -52,18 +58,27 @@ def __init__(self): self.debug = 0 self._init() - def _init(self): + def _init(self) -> None: + """ + Clears the buffers and sets the guess to 'unknown' with a confidence + of 0. + """ self._auth_buffer = '' self._protocol_buffer = '' self._response_buffer = '' self._info = {"os": ("unknown", 0)} - def _dbg(self, level, msg, exc_info=None): + def _dbg(self, level: int, msg: str, exc_info: bool | BaseException | None = None) -> None: + """ + Logs the given message if the debug level is at least the given level. + """ if level > self.debug: return self._logger.debug(msg, exc_info=exc_info) - def reset(self, auth_buffer='', protocol_buffer='', response_buffer=''): + def reset( + self, auth_buffer: str = '', protocol_buffer: str = '', response_buffer: str = '' + ) -> None: """ Reset the guesser state and optionally pre-populate the monitoring buffers. """ @@ -74,7 +89,7 @@ def reset(self, auth_buffer='', protocol_buffer='', response_buffer=''): self._protocol_buffer = protocol_buffer self._response_buffer = response_buffer - def set(self, key, value, confidence=100): + def set(self, key: str, value: str | None, confidence: int = 100) -> None: """ Defines the given value with the given confidence, unless the same value is already defined with a higher confidence level. @@ -91,7 +106,9 @@ def set(self, key, value, confidence=100): ) self._info[key] = (value, confidence) - def set_from_match(self, key, regex_list, string): + # The items are Any: mypy does not tell the functions from the tuples by + # the hasattr() check below. + def set_from_match(self, key: str, regex_list: Iterable[Any], string: str) -> None: """ Given a list of functions or three-tuples (regex, value, confidence), this function walks through them and checks whether any of the @@ -115,24 +132,26 @@ def set_from_match(self, key, regex_list, string): if regex.search(string): self.set(key, value, confidence) - def get(self, key, confidence=0): + def get(self, key: str, confidence: int = 0) -> str | None: """ Returns the info with the given key, if it has at least the given confidence. Returns None otherwise. """ if key not in self._info: return None - value, conf = self._info.get(key) + value, conf = cast(tuple[str, int], self._info.get(key)) if conf >= confidence: return value return None - def check_received_data(self, data, authenticated): + def check_received_data(self, data: str, authenticated: bool) -> None: """ Callback to search the received data for matching patterns. - The `authenticated` flag controls whether to also evaluate received data - from the authentication phase. + Before the authentication, the data is matched with the head and the + response checks of the drivers. Afterwards (`authenticated` is True), + only the response checks are used, and only until the guess has a + confidence of 80 or more. """ if not data: return @@ -151,7 +170,7 @@ def check_received_data(self, data, authenticated): self.set_from_match('os', self._response_os_map, self._auth_buffer) self._auth_buffer = self._auth_buffer[-self._buffer_limit :] - def check_protocol_info(self, data): + def check_protocol_info(self, data: str | None) -> None: """ Callback to search the received protocol information for matching patterns. """ @@ -162,9 +181,11 @@ def check_protocol_info(self, data): self.set_from_match('os', self._protocol_os_map, self._protocol_buffer) self._protocol_buffer = self._protocol_buffer[-self._buffer_limit :] - def check_netconf_capabilities(self, capabilities): + def check_netconf_capabilities(self, capabilities: str | Iterable[str] | None) -> None: """ Callback to search the received Netconf capabilities for matching patterns. + + :param capabilities: The capabilities, as a list or one per line. """ if not capabilities: return diff --git a/src/Exscript/protocols/protocol.py b/src/Exscript/protocols/protocol.py index e3de3991..df8fcf2b 100644 --- a/src/Exscript/protocols/protocol.py +++ b/src/Exscript/protocols/protocol.py @@ -34,9 +34,11 @@ import sys import threading import time +from collections.abc import Callable, Iterable, Iterator, Mapping from functools import partial from io import StringIO -from typing import IO +from types import FrameType, TracebackType +from typing import IO, TYPE_CHECKING, Any, NoReturn, Self, cast from ..util.buffer import MonitoredBuffer from ..util.cast import to_regexs @@ -58,6 +60,12 @@ ) from .osguesser import OsGuesser +if TYPE_CHECKING: + from _typeshed import FileDescriptorLike + + from ..account import Account, AccountProxy + from ..key import PrivateKey + try: import termios import tty @@ -96,7 +104,7 @@ ) -def _strip_escapes(text): +def _strip_escapes(text: str) -> tuple[str, list[tuple[int, int]]]: """ Removes terminal escape sequences. Returns the remaining text and, for each removed sequence, its position in the remaining text and the number @@ -112,7 +120,7 @@ def _strip_escapes(text): return ''.join(parts), cuts -def _unstrip(cuts, pos, after): +def _unstrip(cuts: list[tuple[int, int]], pos: int, after: bool) -> int: """ Maps a position in the text from _strip_escapes() back to the original text. Sequences at the position itself are skipped if after is true. @@ -125,7 +133,7 @@ def _unstrip(cuts, pos, after): return pos + removed -def _ends_with_anchor(regex): +def _ends_with_anchor(regex: re.Pattern[str]) -> bool: """Whether the regular expression ends with an unescaped $ or \\Z.""" pattern = regex.pattern.rstrip(')') if pattern.endswith('$') and not regex.flags & re.MULTILINE: @@ -218,27 +226,29 @@ class Protocol: def __init__( self, - driver=None, - stdout=None, - stderr=None, - debug=0, - connect_rate_limit=60, - connect_timeout=30, - timeout=30, - logfile=None, - termtype='dumb', - disabled_algorithms=None, # SSH2 only - preferred_authentications=None, # SSH2 only - verify_fingerprint=True, # SSH2 only - account_factory=None, - banner_timeout=20, # SSH2 only - encoding='latin-1', + driver: Driver | str | None = None, + stdout: Any = None, # any object with write() and flush(), see the stdout property + stderr: IO[str] | None = None, + debug: int = 0, + connect_rate_limit: int = 60, + connect_timeout: int = 30, + timeout: int = 30, + logfile: str | None = None, + termtype: str = 'dumb', + disabled_algorithms: dict[str, list[str]] | None = None, # SSH2 only + preferred_authentications: list[str] | None = None, # SSH2 only + verify_fingerprint: bool = True, # SSH2 only + account_factory: ( + Callable[['Account | AccountProxy | None'], 'Account | AccountProxy'] | None + ) = None, + banner_timeout: float = 20, # SSH2 only + encoding: str = 'latin-1', *, - host_key_policy='reject', # SSH2 only - known_hosts=None, # SSH2 only - term_width=511, - term_height=1000, - ): + host_key_policy: str = 'reject', # SSH2 only + known_hosts: str | None = None, # SSH2 only + term_width: int = 511, + term_height: int = 1000, + ) -> None: """ Constructor. @@ -251,9 +261,11 @@ def __init__( :type driver: Driver()|str|None :keyword driver: The manually picked device driver. - :keyword stdout: Where to write the device response. Defaults to - an in-memory buffer. - :keyword stderr: Where to write debug info. Defaults to stderr. + :keyword stdout: Where to write the device response: any object with + write() and flush(). Defaults to an in-memory buffer. + :keyword stderr: The stream that the stderr property returns; + defaults to sys.stderr. The adapter writes nothing to it: its + debug output goes to the logging module (see debug). :keyword debug: An integer between 0 (no debugging) and 5 (very verbose debugging) that specifies the amount of debug output, which the loggers under Exscript.protocols log at the DEBUG @@ -296,9 +308,12 @@ def __init__( :keyword known_hosts: The known_hosts file. Defaults to ~/.ssh/known_hosts. Entries for other ports than 22 have the form [host]:port, like in OpenSSH. - :keyword account_factory: A function that produces a new :class:`Account`. - :type banner_timeout: bool - :keyword banner_timeout: The time to wait for the banner. + :keyword account_factory: A function that returns the account to use, + locked already, whenever an account is needed. It gets the account + that was passed to login() or to the authentication methods, else + the one of the last call, or None. + :type banner_timeout: float + :keyword banner_timeout: The time to wait for the banner, in seconds. :type encoding: str :keyword encoding: The encoding of data received from the remote host. """ @@ -312,12 +327,12 @@ def __init__( self._os_guesser = OsGuesser() self._os_guesser.debug = debug self._default_port = _WELL_KNOWN_PORTS.get(self._name) - self._manual_user_re = None - self._manual_password_re = None - self._manual_prompt_re = None - self._manual_error_re = None - self._manual_login_error_re = None - self._manual_driver = None + self._manual_user_re: list[re.Pattern[str]] | None = None + self._manual_password_re: list[re.Pattern[str]] | None = None + self._manual_prompt_re: list[re.Pattern[str]] | None = None + self._manual_error_re: list[re.Pattern[str]] | None = None + self._manual_login_error_re: list[re.Pattern[str]] | None = None + self._manual_driver: Driver | None = None self._receives_echo = True # does the remote shell echo sent commands? self._sending_secret = False # set by _secret() while a password is sent self._write_lock = threading.Lock() @@ -327,19 +342,19 @@ def __init__( self.data_received_event = Event() self.data_sent_event = Event() self.otp_requested_event = Event() - self.host = None - self.port = None + self.host: str | None = None + self.port: int | None = None self.account_factory = account_factory self.connect_rate_limit = connect_rate_limit self.connect_timeout = connect_timeout self.debug = debug self.encoding = encoding self.logfile = logfile - self.response = None + self.response: str | None = None # FIXME: Remove self.send_data in favor of self.data_sent_event! # Right now this "buffer" is only used in interactive sessions but grows # indefinitely, which is a memory leak. - self.send_data = None + self.send_data: IO[str] | None = None self.stdout = stdout self.stderr = stderr self.termtype = termtype @@ -369,7 +384,7 @@ def __init__( else: self._dbg(1, 'Invalid driver given. Ignoring.') - def __copy__(self): + def __copy__(self) -> Self: """ Overwritten to return the very same object instead of copying the stream, because copying a network connection is impossible. @@ -379,25 +394,30 @@ def __copy__(self): """ return self - def __deepcopy__(self, memo): + def __deepcopy__(self, memo: dict[int, Any]) -> Self: """ Overwritten to return the very same object instead of copying the stream, because copying a network connection is impossible. - :type memo: object + :type memo: dict[int, Any] :param memo: Please refer to Python's standard library documentation. :rtype: Protocol :return: self """ return self - def __enter__(self): + def __enter__(self) -> Self: """ Enter the runtime context related to this object. """ return self - def __exit__(self, exc_type, exc_value, traceback): + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + traceback: TracebackType | None, + ) -> None: """ Ensures to close the connection when context is left. @@ -406,7 +426,7 @@ def __exit__(self, exc_type, exc_value, traceback): """ self.close(force=exc_type is not None) - def __repr__(self): + def __repr__(self) -> str: clsname = type(self).__name__ host = self.host or 'unknown' port = self.port or self._default_port @@ -414,7 +434,7 @@ def __repr__(self): return f'<{clsname}({host}:{port} [{state}]) at {hex(id(self))}>' @property - def connected(self): + def connected(self) -> bool: """ Whether the connection to the remote host is currently open (read-only). @@ -424,7 +444,7 @@ def connected(self): return self._connected @property - def logfile(self): + def logfile(self) -> str | None: """ The path of the current logfile, or None if logging to file is disabled. This also maintains any open logfile descriptor internally, especially @@ -436,7 +456,7 @@ def logfile(self): return self._logfile @logfile.setter - def logfile(self, value) -> None: + def logfile(self, value: str | None) -> None: with self._write_lock: old_log = getattr(self, '_log', None) if old_log: @@ -446,11 +466,13 @@ def logfile(self, value) -> None: pass self._logfile = value self._log = ( - open(self._logfile, 'a', encoding='utf-8') if value and self.connected else None + open(cast(str, self._logfile), 'a', encoding='utf-8') + if value and self.connected + else None ) @property - def os_guesser(self): + def os_guesser(self) -> OsGuesser: """ The OS guesser used to auto-detect the remote host's OS/driver (read-only). @@ -463,8 +485,10 @@ def os_guesser(self): """ return self._os_guesser + # The stream is Any: any object with write() and flush() works, and + # Exscript.queue passes one that is not an IO[str]. @property - def stdout(self): + def stdout(self) -> Any: """ The stream to which device responses are written. Defaults to an internal StringIO buffer which will be cleared when the connection is closed. @@ -475,17 +499,17 @@ def stdout(self): return self._stdout @stdout.setter - def stdout(self, value): + def stdout(self, value: Any) -> None: with self._write_lock: if value is None: - self._stdout = StringIO() + self._stdout: Any = StringIO() self._stdout_owner = True else: self._stdout = value self._stdout_owner = False @property - def stderr(self): + def stderr(self) -> IO[str]: """ The error stream of the adapter, `sys.stderr` by default. The adapter writes nothing to it: its debug output goes to the logging module @@ -497,31 +521,42 @@ def stderr(self): return self._stderr @stderr.setter - def stderr(self, value): + def stderr(self, value: IO[str] | None) -> None: with self._write_lock: self._stderr = sys.stderr if value is None else value - def _dbg(self, level, msg, exc_info=None): + def _dbg(self, level: int, msg: str, exc_info: bool | BaseException | None = None) -> None: + """ + Logs the given message, with the name of the driver, if the debug + level is at least the given level. + """ if level > self.debug: return msg = '[%s] %s' % (self.get_driver().name, msg) self._logger.debug(msg, exc_info=exc_info) - def _init(self): + def _init(self) -> None: + """ + Sets the state of a new session: the constructor and close() call it. + """ # Init all per-session state settings. self._app_authenticated = False self._app_authorized = False self._proto_authenticated = False self._connected = False - self._device_context_stack = [] + self._device_context_stack: list[dict[str, Any]] = [] self._auto_driver = driver_map[self.guess_os()] self._driver_replaced = False self._expect_canceled = False - self._last_account = None + self._last_account: Account | AccountProxy | None = None # The buffer position at the last send(); None before the first one. - self._sent_at = None + self._sent_at: int | None = None - def _reset(self): + def _reset(self) -> None: + """ + Clears the buffer, the OS guess and stdout (if it is the internal + buffer), closes the logfile, and sets the state of a new session. + """ # Reset all per-session state settings when close() is called. self._buffer.clear() self._os_guesser.reset() @@ -536,7 +571,12 @@ def _reset(self): self.stdout.seek(0) self._init() - def _receive_cb(self, data): + def _receive_cb(self, data: str) -> None: + """ + Handles data that was received from the host: writes it to stdout and + to the logfile, updates the OS guess and the driver, and emits + data_received_event. + """ with self._write_lock: self._write_to_streams(data) self._write_to_logs(data) @@ -548,7 +588,11 @@ def _receive_cb(self, data): # Send signals to subscribers. self.data_received_event(data) - def _send_cb(self, data): + def _send_cb(self, data: str) -> None: + """ + Handles data that was sent to the host: writes it to stdout and to the + logfile if the host does not echo it, and emits data_sent_event. + """ if not self._receives_echo: # The sent commands are either logged via the _receive_cb() when # they are echoed back, or they are logged here explicitly. @@ -559,17 +603,23 @@ def _send_cb(self, data): # Send signals to subscribers. self.data_sent_event(data) - def _write_to_logs(self, data): + def _write_to_logs(self, data: str) -> None: + """ + Writes the given data to the logfile, if one is open. + """ if self._log is None: return self._log.write(data) self._log.flush() - def _write_to_streams(self, data): + def _write_to_streams(self, data: str) -> None: + """ + Writes the given data to stdout. + """ self.stdout.write(data) self.stdout.flush() - def is_dummy(self): + def is_dummy(self) -> bool: """ Returns True if the adapter implements a virtual device, i.e. it isn't an actual network connection. @@ -580,7 +630,12 @@ def is_dummy(self): return False # OPTIMIZE: OsGuesser should take this method as a callback on OS change. - def _update_auto_driver(self): + def _update_auto_driver(self) -> None: + """ + Sets the automatically chosen driver to the one of the guessed OS. If + this changes the driver in use while connected, cancels a running + expect() or waitfor(), which then restarts with the new driver. + """ old_driver = self.get_driver() self._auto_driver = driver_map[self.guess_os()] new_driver = self.get_driver() @@ -591,7 +646,7 @@ def _update_auto_driver(self): self._driver_replaced = True self.cancel_expect() - def set_driver(self, driver=None): + def set_driver(self, driver: Driver | str | None = None) -> None: """ Defines the driver that is used to recognize prompts and implement behavior depending on the remote system. The driver argument may be an @@ -623,7 +678,7 @@ def set_driver(self, driver=None): self._driver_replaced = True self.cancel_expect() - def get_driver(self): + def get_driver(self) -> Driver: """ Returns the currently used driver. @@ -634,7 +689,7 @@ def get_driver(self): return self._manual_driver return self._auto_driver - def get_banner(self): + def get_banner(self) -> str | None: """ Returns the banner that was received upon login. Only supported on SSH2/Netconf. Returns `None` on all other protocols. @@ -645,7 +700,7 @@ def get_banner(self): """ return None - def get_remote_version(self): + def get_remote_version(self) -> str | None: """ Returns the remote version id-string that was received upon login. Only supported on SSH2/Netconf. Returns `None` on all other protocols. @@ -656,7 +711,7 @@ def get_remote_version(self): """ return None - def autoinit(self): + def autoinit(self) -> None: """ Make the remote host more script-friendly by automatically executing one or more commands on it to prepare the session/environment. @@ -671,57 +726,68 @@ def autoinit(self): self._dbg(4, 'Calling %s.init_terminal().' % repr(driver)) driver.init_terminal(self) - def set_username_prompt(self, regex=None): + def set_username_prompt( + self, regex: str | re.Pattern[str] | Iterable[str | re.Pattern[str]] | None = None + ) -> None: """ Defines a pattern that is used to monitor the response of the connected host for a username prompt. - :type regex: RegEx - :param regex: The pattern that, when matched, causes an error. + :type regex: str|re.RegexObject|list(str|re.RegexObject)|None + :param regex: One or more patterns that match the username prompt, or + None to use the patterns of the driver. """ if regex is None: self._manual_user_re = regex else: self._manual_user_re = to_regexs(regex) - def get_username_prompt(self): + def get_username_prompt(self) -> list[re.Pattern[str]]: """ - Returns the regular expression that is used to monitor the response + Returns the regular expressions that are used to monitor the response of the connected host for a username prompt. - :rtype: regex - :return: A regular expression. + :rtype: list(re.RegexObject) + :return: A list of regular expression objects. """ if self._manual_user_re: return self._manual_user_re return self.get_driver().user_re - def set_password_prompt(self, regex=None): + def set_password_prompt( + self, regex: str | re.Pattern[str] | Iterable[str | re.Pattern[str]] | None = None + ) -> None: """ Defines a pattern that is used to monitor the response of the connected host for a password prompt. - :type regex: RegEx - :param regex: The pattern that, when matched, causes an error. + :type regex: str|re.RegexObject|list(str|re.RegexObject)|None + :param regex: One or more patterns that match the password prompt, or + None to use the patterns of the driver. """ if regex is None: self._manual_password_re = regex else: self._manual_password_re = to_regexs(regex) - def get_password_prompt(self): + def get_password_prompt(self) -> list[re.Pattern[str]]: """ - Returns the regular expression that is used to monitor the response + Returns the regular expressions that are used to monitor the response of the connected host for a password prompt. - :rtype: regex - :return: A regular expression. + :rtype: list(re.RegexObject) + :return: A list of regular expression objects. """ if self._manual_password_re: return self._manual_password_re return self.get_driver().password_re - def set_prompt(self, prompt=None, *, extend=False): + def set_prompt( + self, + prompt: str | re.Pattern[str] | Iterable[str | re.Pattern[str]] | None = None, + *, + extend: bool = False, + ) -> None: """ Defines a pattern that is waited for when calling the expect_prompt() method. If the set_prompt() method is not called, or if it is called @@ -730,7 +796,7 @@ def set_prompt(self, prompt=None, *, extend=False): others. Netconf uses the EOM or EOC markers to emulate prompt matching. This should *NOT* be changed as its RFC is strict about this. - :type prompt: RegEx + :type prompt: str|re.RegexObject|list(str|re.RegexObject)|None :param prompt: The pattern that matches the prompt of the remote host. :type extend: bool :param extend: Whether to extend the default pattern instead of setting @@ -745,7 +811,7 @@ def set_prompt(self, prompt=None, *, extend=False): else: self._manual_prompt_re = to_regexs(prompt) - def get_prompt(self): + def get_prompt(self) -> list[re.Pattern[str]]: """ Returns the regular expressions that is matched against the host response when calling the expect_prompt() method. @@ -757,7 +823,12 @@ def get_prompt(self): return self._manual_prompt_re return self.get_driver().prompt_re - def set_error_prompt(self, error=None, *, extend=False): + def set_error_prompt( + self, + error: str | re.Pattern[str] | Iterable[str | re.Pattern[str]] | None = None, + *, + extend: bool = False, + ) -> None: """ Defines a pattern that is used to monitor the response of the connected host. If the set_error_prompt() method is not called, or if it is called @@ -770,7 +841,7 @@ def set_error_prompt(self, error=None, *, extend=False): are used, but not waitfor()), an InvalidCommandException is raised. Netconf raises the RPCError exception (subclass) with structured details. - :type error: RegEx + :type error: str|re.RegexObject|list(str|re.RegexObject)|None :param error: The pattern that, when matched, causes an error. :type extend: bool :param extend: Whether to extend the default pattern instead of setting @@ -785,25 +856,27 @@ def set_error_prompt(self, error=None, *, extend=False): else: self._manual_error_re = to_regexs(error) - def get_error_prompt(self): + def get_error_prompt(self) -> list[re.Pattern[str]]: """ - Returns the regular expression that is used to monitor the response + Returns the regular expressions that are used to monitor the response of the connected host for errors. - :rtype: regex - :return: A regular expression. + :rtype: list(re.RegexObject) + :return: A list of regular expression objects. """ if self._manual_error_re: return self._manual_error_re return self.get_driver().error_re - def set_login_error_prompt(self, error=None): + def set_login_error_prompt( + self, error: str | re.Pattern[str] | Iterable[str | re.Pattern[str]] | None = None + ) -> None: """ Defines a pattern that is used to monitor the response of the connected host during the authentication procedure. If the pattern matches a LoginFailure exception is raised. - :type error: RegEx + :type error: str|re.RegexObject|list(str|re.RegexObject)|None :param error: The pattern that, when matched, causes an error. """ if error is None: @@ -811,20 +884,20 @@ def set_login_error_prompt(self, error=None): else: self._manual_login_error_re = to_regexs(error) - def get_login_error_prompt(self): + def get_login_error_prompt(self) -> list[re.Pattern[str]]: """ - Returns the regular expression that is used to monitor the response + Returns the regular expressions that are used to monitor the response of the connected host for login errors; this is only used during the login procedure, i.e. app_authenticate() or app_authorize(). - :rtype: regex - :return: A regular expression. + :rtype: list(re.RegexObject) + :return: A list of regular expression objects. """ if self._manual_login_error_re: return self._manual_login_error_re return self.get_driver().login_error_re - def set_connect_timeout(self, timeout): + def set_connect_timeout(self, timeout: int) -> None: """ Defines the maximum time that the adapter waits for initial connection. @@ -833,7 +906,7 @@ def set_connect_timeout(self, timeout): """ self.connect_timeout = int(timeout) - def get_connect_timeout(self): + def get_connect_timeout(self) -> int: """ Returns the current connect timeout in seconds. @@ -842,7 +915,7 @@ def get_connect_timeout(self): """ return self.connect_timeout - def set_timeout(self, timeout): + def set_timeout(self, timeout: int) -> None: """ Defines how long :meth:`expect()`, :meth:`waitfor()` or :meth:`expect_prompt()` wait for data from the remote host before they @@ -854,7 +927,7 @@ def set_timeout(self, timeout): """ self.timeout = int(timeout) - def get_timeout(self): + def get_timeout(self) -> int: """ Returns the current response timeout in seconds. @@ -863,16 +936,17 @@ def get_timeout(self): """ return self.timeout - def get_host(self): + def get_host(self) -> str | None: """ Returns the name or address of the currently connected host. - :rtype: string - :return: A name or an address (whatever was used initially). + :rtype: str|None + :return: A name or an address (whatever was used initially), or None + before the first call of connect(). """ return self.host - def get_protocol(self): + def get_protocol(self) -> str: """ Returns the name of this protocol. @@ -885,7 +959,13 @@ def get_protocol(self): """ return self._name - def get_response(self, *, strip_command=True, strip_chars='\x00\x07\x08', strip_ansi=True): + def get_response( + self, + *, + strip_command: bool = True, + strip_chars: str = '\x00\x07\x08', + strip_ansi: bool = True, + ) -> str: """ Returns a clean copy of the latest plaintext response. @@ -903,8 +983,12 @@ def get_response(self, *, strip_command=True, strip_chars='\x00\x07\x08', strip_ @staticmethod def cleanup_response( - response, *, strip_command=True, strip_chars='\x00\x07\x08', strip_ansi=True - ): + response: str | None, + *, + strip_command: bool = True, + strip_chars: str = '\x00\x07\x08', + strip_ansi: bool = True, + ) -> str: """ Returns a clean copy of the given plaintext response. @@ -938,7 +1022,7 @@ def cleanup_response( response = _ansi_escape_re.sub('', response) return response - def guess_os(self): + def guess_os(self) -> str: """ Returns an identifier that specifies the operating system that is running on the remote host. This OS is obtained by watching the @@ -952,9 +1036,10 @@ def guess_os(self): :rtype: string :return: A string to help identify the remote operating system. """ - return self._os_guesser.get('os') + # The guesser always has an 'os' with a confidence of at least 0. + return cast(str, self._os_guesser.get('os')) - def _enforce_connect_rate_limit(self, rate_limit=0): + def _enforce_connect_rate_limit(self, rate_limit: int = 0) -> None: """ Waits until the connect keeps the rate limit: connects to the same host, also concurrent ones, are spaced by 60 / rate_limit + 0.1 @@ -964,7 +1049,7 @@ def _enforce_connect_rate_limit(self, rate_limit=0): :param rate_limit: The connection rate-limit (connects per minute); 0 uses connect_rate_limit. """ - key = self.host.lower() + key = cast(str, self.host).lower() # connect() checks that the host is set interval = 0.0 precision = 3 @@ -988,24 +1073,27 @@ def _enforce_connect_rate_limit(self, rate_limit=0): self._dbg(2, "Remaining connection delay: %s seconds" % delay) time.sleep(delay) - def _connect(self): + def _connect(self) -> bool: """ Should be overwritten. That's the place where the connection should be established. + + :return: True if the connection is open. """ raise NotImplementedError() - def connect(self, hostname=None, port=None): + def connect(self, hostname: str | None = None, port: int | None = None) -> bool: """ Opens the connection to the remote host or IP address. Raises a ProtocolException if the adapter is already connected or hostname/port is missing or invalid (e.g. first connect). The hostname and port may be omitted for reconnects to the same device. - :type hostname: string + :type hostname: str|None :param hostname: The remote host or IP address. - :type port: int + :type port: int|None :param port: The remote TCP port number. + :return: True if the connection is open. """ if self.connected: msg = 'already connected to %s:%s' % (self.host, self.port) @@ -1031,9 +1119,20 @@ def connect(self, hostname=None, port=None): self._update_auto_driver() return self._connected - def _get_account(self, account): + def _get_account(self, account: 'Account | AccountProxy | _Context | None') -> _Context: + """ + Returns a context for the given account, or for that of the last call + if None is given, and keeps the account for the next call; the account + factory, if any, provides the account, else it is locked. Leaving the + returned context releases the account; a given context gets a new + context of the same account. + + :raises TypeError: If there is neither an account nor an account + factory. + """ if isinstance(account, Context) or isinstance(account, _Context): - return account.context() + # A _Context passes the call on to the Context that it wraps. + return cast(_Context, account.context()) if account is None: account = self._last_account if self.account_factory: @@ -1045,7 +1144,12 @@ def _get_account(self, account): self._last_account = account return account.context() - def login(self, account=None, app_account=None, flush=True): + def login( + self, + account: 'Account | AccountProxy | _Context | None' = None, + app_account: 'Account | AccountProxy | _Context | None' = None, + flush: bool = True, + ) -> None: """ Log into the connected host using the best method available. If no account is given, the account of the last call of login() or @@ -1079,7 +1183,12 @@ def login(self, account=None, app_account=None, flush=True): self.expect_prompt() self.auto_app_authorize(app_account, flush=flush) - def authenticate(self, account=None, app_account=None, flush=True): + def authenticate( + self, + account: 'Account | AccountProxy | _Context | None' = None, + app_account: 'Account | AccountProxy | _Context | None' = None, + flush: bool = True, + ) -> None: """ Like login(), but skips the authorization procedure. @@ -1101,19 +1210,21 @@ def authenticate(self, account=None, app_account=None, flush=True): self.protocol_authenticate(account) self.app_authenticate(app_account, flush=flush) - def _protocol_authenticate(self, user, password): + def _protocol_authenticate(self, user: str, password: str | None) -> None: """ Should be overwritten if protocol authentication is supported. """ pass - def _protocol_authenticate_by_key(self, user, key): + def _protocol_authenticate_by_key(self, user: str, key: 'PrivateKey') -> None: """ Should be overwritten if protocol authentication is supported. """ pass - def protocol_authenticate(self, account=None): + def protocol_authenticate( + self, account: 'Account | AccountProxy | _Context | None' = None + ) -> None: """ Low-level API to perform protocol-level authentication on protocols that support it. @@ -1138,7 +1249,7 @@ def protocol_authenticate(self, account=None): self._proto_authenticated = True self._dbg(2, "Finished to protocol-authenticate %s." % user) - def is_protocol_authenticated(self): + def is_protocol_authenticated(self) -> bool: """ Returns True if the protocol-level authentication procedure was completed, False otherwise. @@ -1148,13 +1259,27 @@ def is_protocol_authenticated(self): """ return self._proto_authenticated - def _app_authenticate(self, account, password, flush=True, bailout=False): + def _app_authenticate( + self, + account: _Context, + password: str, + flush: bool = True, + bailout: bool = False, + ) -> None: """ Generic implementation for app-level authentications or authorizations. A login error counts only if the host sent it after the last call of send(), e.g. after the password: banners and the message of the day may contain words like "denied" or "failed". + + :param account: The context of the account whose name is sent at a + username prompt. + :param password: The password that is sent at a password prompt, or + from which the response to an S/Key challenge is computed. + :param flush: Whether to consume the CLI prompt from the buffer. + :param bailout: Whether to return after sending the password, without + waiting for a CLI prompt. """ user = account.get_name() @@ -1239,7 +1364,12 @@ def _app_authenticate(self, account, password, flush=True, bailout=False): else: assert False # No such section - def app_authenticate(self, account=None, flush=True, bailout=False): + def app_authenticate( + self, + account: 'Account | AccountProxy | _Context | None' = None, + flush: bool = True, + bailout: bool = False, + ) -> None: """ Attempt to perform application-level authentication. Application level authentication is needed on devices where the username and @@ -1272,7 +1402,8 @@ def app_authenticate(self, account=None, flush=True, bailout=False): :type flush: bool :param flush: Whether to flush the last prompt from the buffer. :type bailout: bool - :param bailout: Whether to wait for a prompt after sending the password. + :param bailout: Whether to return right after sending the password, + without waiting for a prompt. """ with self._get_account(account) as account: user = account.get_name() @@ -1282,7 +1413,7 @@ def app_authenticate(self, account=None, flush=True, bailout=False): self._app_authenticated = True self._dbg(2, "Finished to app-authenticate %s." % user) - def is_app_authenticated(self): + def is_app_authenticated(self) -> bool: """ Returns True if the application-level authentication procedure was completed, False otherwise. @@ -1292,7 +1423,12 @@ def is_app_authenticated(self): """ return self._app_authenticated - def app_authorize(self, account=None, flush=True, bailout=False): + def app_authorize( + self, + account: 'Account | AccountProxy | _Context | None' = None, + flush: bool = True, + bailout: bool = False, + ) -> None: """ Like app_authenticate(), but uses the authorization password of the account. @@ -1311,7 +1447,8 @@ def app_authorize(self, account=None, flush=True, bailout=False): :type flush: bool :param flush: Whether to flush the last prompt from the buffer. :type bailout: bool - :param bailout: Whether to wait for a prompt after sending the password. + :param bailout: Whether to return right after sending the password, + without waiting for a prompt. """ with self._get_account(account) as account: user = account.get_name() @@ -1323,7 +1460,12 @@ def app_authorize(self, account=None, flush=True, bailout=False): self._app_authorized = True self._dbg(2, "Finished to app-authorize %s." % user) - def auto_app_authorize(self, account=None, flush=True, bailout=False): + def auto_app_authorize( + self, + account: 'Account | AccountProxy | _Context | None' = None, + flush: bool = True, + bailout: bool = False, + ) -> None: """ Like app_authorize(), but instead of just waiting for a user or password prompt, it automatically initiates the authorization @@ -1343,14 +1485,15 @@ def auto_app_authorize(self, account=None, flush=True, bailout=False): :type flush: bool :param flush: Whether to flush the last prompt from the buffer. :type bailout: bool - :param bailout: Whether to wait for a prompt after sending the password. + :param bailout: Whether to return right after sending the password, + without waiting for a prompt. """ with self._get_account(account) as account: driver = self.get_driver() self._dbg(4, 'Calling %s.auto_authorize().' % repr(driver)) driver.auto_authorize(self, account, flush, bailout) - def is_app_authorized(self): + def is_app_authorized(self) -> bool: """ Returns True if the application-level authorization procedure was completed, False otherwise. @@ -1360,14 +1503,14 @@ def is_app_authorized(self): """ return self._app_authorized - def _send(self, data): + def _send(self, data: str) -> None: """ Should be overwritten. That's the place where the data is sent to the remote host without waiting for response. """ raise NotImplementedError() - def send(self, data): + def send(self, data: str) -> None: """ Sends the given data to the remote host. Returns without waiting for a response while the response (self.response) attribute is cleared. Please @@ -1397,7 +1540,7 @@ def send(self, data): self._send(data) @contextlib.contextmanager - def _secret(self): + def _secret(self) -> Iterator[None]: """Marks the data sent within this context as secret, so it is not logged.""" self._sending_secret = True try: @@ -1405,7 +1548,7 @@ def _secret(self): finally: self._sending_secret = False - def execute(self, command, consume=True): + def execute(self, command: str, consume: bool = True) -> tuple[int, re.Match[str]]: """ Sends the given command to the remote host (with a CR appended) and waits for a prompt in the response. Netconf expects the RPC body as @@ -1437,7 +1580,7 @@ def execute(self, command, consume=True): self.send(command + '\r') return self.expect_prompt(consume) - def _domatch(self, prompt, flush): + def _domatch(self, prompt: list[re.Pattern[str]], flush: bool) -> tuple[int, re.Match[str]]: """ Waits until one of the given regular expressions matches the received data, and sets self.response to the data before the match. A match of @@ -1478,7 +1621,9 @@ def _domatch(self, prompt, flush): self._dbg(4, 'Response (flush=%s): %s' % (repr(flush), repr(self.response))) return n, match - def _search(self, prompt, searched): + def _search( + self, prompt: list[re.Pattern[str]], searched: int + ) -> tuple[int, re.Match[str] | None, int, int]: """ Searches the buffer for the first of the given regular expressions that matches. A regular expression that ends with $ or \\Z is searched @@ -1524,7 +1669,7 @@ def _search(self, prompt, searched): return n, match, start, tail_offset + _unstrip(cuts, match.end(), after=True) return -1, None, 0, 0 - def _fill_buffer(self, timeout): + def _fill_buffer(self, timeout: float | None) -> bool | None: """ Should be overwritten. Waits for data from the remote host, and passes the received text to _receive_cb() and to the buffer. @@ -1538,14 +1683,22 @@ def _fill_buffer(self, timeout): """ raise NotImplementedError() - def _waitfor(self, prompt): + def _waitfor( + self, prompt: str | re.Pattern[str] | Iterable[str | re.Pattern[str]] + ) -> tuple[int, re.Match[str]]: + """ + Like waitfor(), but does not retry if the driver is replaced while + waiting. + """ prompt = to_regexs(prompt) self._dbg(2, "Waiting for a prompt (non-consuming).") self._dbg(3, "Matching prompt pattern: " + repr([p.pattern for p in prompt])) result = self._domatch(prompt, flush=False) return result - def waitfor(self, prompt): + def waitfor( + self, prompt: str | re.Pattern[str] | Iterable[str | re.Pattern[str]] + ) -> tuple[int, re.Match[str]]: """ Monitors the data received from the remote host and waits until the response matches the given prompt. Once a match has been found, @@ -1587,14 +1740,22 @@ def waitfor(self, prompt): continue # retry return result - def _expect(self, prompt): + def _expect( + self, prompt: str | re.Pattern[str] | Iterable[str | re.Pattern[str]] + ) -> tuple[int, re.Match[str]]: + """ + Like expect(), but neither retries if the driver is replaced while + waiting nor checks the response for errors. + """ prompt = to_regexs(prompt) self._dbg(2, "Expecting a prompt (consuming).") self._dbg(3, "Matching prompt pattern: " + repr([p.pattern for p in prompt])) result = self._domatch(prompt, flush=True) return result - def expect(self, prompt): + def expect( + self, prompt: str | re.Pattern[str] | Iterable[str | re.Pattern[str]] + ) -> tuple[int, re.Match[str]]: """ Like waitfor(), but also removes the response and the matched string from the buffer containing the incoming data. In other words, the @@ -1626,7 +1787,7 @@ def expect(self, prompt): self._check_response_for_errors() return result - def expect_prompt(self, consume=True): + def expect_prompt(self, consume: bool = True) -> tuple[int, re.Match[str]]: """ Monitors the data received from the remote host and waits for a prompt in the response. @@ -1657,7 +1818,11 @@ def expect_prompt(self, consume=True): result = self.waitfor(self.get_prompt()) return result - def _check_response_for_errors(self): + def _check_response_for_errors(self) -> None: + """ + Raises an error if a line of the response matches one of the error + patterns; see _raise_on_response_with_errors(). + """ error_prompts = self.get_error_prompt() if not error_prompts: return @@ -1670,7 +1835,12 @@ def _check_response_for_errors(self): self._dbg(3, "Error prompt %s matches line %s: %s" % args) self._raise_on_response_with_errors(line) - def add_monitor(self, pattern, callback, limit=80): + def add_monitor( + self, + pattern: str | re.Pattern[str] | Iterable[str | re.Pattern[str]], + callback: Callable[[Self, int, re.Match[str]], object], + limit: int = 80, + ) -> None: """ Calls the given function whenever the given pattern matches the incoming data. @@ -1694,7 +1864,7 @@ def add_monitor(self, pattern, callback, limit=80): """ self._buffer.add_monitor(pattern, partial(callback, self), limit) - def cancel_expect(self): + def cancel_expect(self) -> None: """ Cancel the current call to :meth:`expect()` or :meth:`waitfor()` as soon as control returns to the protocol adapter. This method may be @@ -1704,21 +1874,29 @@ def cancel_expect(self): self._dbg(4, "Cancelling any expect() or waitfor() calls.") self._expect_canceled = True - def _raise_on_canceled_expect(self): + def _raise_on_canceled_expect(self) -> NoReturn: + """ + Clears the request to cancel, and raises DriverReplacedException if + the driver was replaced, or else ExpectCancelledException. + """ self._expect_canceled = False if self._driver_replaced: self._driver_replaced = False raise DriverReplacedException() raise ExpectCancelledException() - def _raise_on_response_with_errors(self, line): + def _raise_on_response_with_errors(self, line: str) -> None: + """ + Raises InvalidCommandException with the given line of the response, + which matched an error pattern, cut to 200 characters. + """ # Only the error line: the full response (self.response) may contain secrets. line = line.strip() if len(line) > 200: line = line[:200] + '...' raise InvalidCommandException('Error in the response from the device: ' + line) - def _raise_on_login_error(self): + def _raise_on_login_error(self) -> None: """ Raises LoginFailure if one of the login error patterns matches the data that the host sent after the last call of send(). @@ -1731,21 +1909,36 @@ def _raise_on_login_error(self): if regex.search(text): raise LoginFailure('Login failed') - def _raise_on_response_timeout(self): + def _raise_on_response_timeout(self) -> NoReturn: + """ + Raises TimeoutException with the last 200 characters of the buffer. + """ buffer = str(self._buffer)[-200:] if buffer: buffer = '...' + buffer msg = 'Timeout while waiting for a response/prompt. Buffer: %s' raise TimeoutException(msg % repr(buffer)) - def _raise_on_response_eof(self): + def _raise_on_response_eof(self) -> NoReturn: + """ + Raises ConnectionClosedException with the last 200 characters of the + buffer. + """ buffer = str(self._buffer)[-200:] if buffer: buffer = '...' + buffer msg = 'EOF while waiting for a response/prompt. Buffer: %s' raise ConnectionClosedException(msg % repr(buffer)) - def _call_key_handlers(self, key_handlers, data): + def _call_key_handlers( + self, key_handlers: Mapping[str, Callable[[Self], object]] | None, data: str + ) -> bool: + """ + Calls the key handler for the given typed character, if there is one, + with this connection. + + :return: Whether a key handler was called. + """ if key_handlers is not None: for key, func in list(key_handlers.items()): if data == key: @@ -1753,14 +1946,26 @@ def _call_key_handlers(self, key_handlers, data): return True return False - def _set_terminal_size(self, rows, cols): + def _set_terminal_size(self, rows: int, cols: int) -> None: """ Should be overwritten. That's the place where the terminal size is set in the interactive shell. """ raise NotImplementedError() - def _open_posix_shell(self, channel, key_handlers, handle_window_size): + def _open_posix_shell( + self, + channel: 'FileDescriptorLike', + key_handlers: Mapping[str, Callable[[Self], object]] | None, + handle_window_size: bool, + ) -> None: + """ + Runs the interactive session on a POSIX terminal: switches the + terminal to raw mode, passes the typed characters to the key handlers + or sends them, and receives data from the channel, until the host + closes the connection or the input ends. Restores the terminal + settings afterwards. + """ # Typed characters may take several bytes. decoder = codecs.getincrementaldecoder(sys.stdin.encoding or 'utf-8')(errors='replace') # We need to make sure to use an unbuffered stdin, else multi-byte @@ -1771,7 +1976,7 @@ def _open_posix_shell(self, channel, key_handlers, handle_window_size): # Update the terminal size whenever the size changes. if handle_window_size: - def handle_sigwinch(signum, frame): + def handle_sigwinch(signum: int | None, frame: FrameType | None) -> None: rows, cols = get_terminal_size() self._dbg(2, "Trying to update terminal size: %dx%d" % (rows, cols)) self._set_terminal_size(rows, cols) @@ -1815,8 +2020,19 @@ def handle_sigwinch(signum, frame): if handle_window_size: signal.signal(signal.SIGWINCH, old_handler or signal.SIG_DFL) - def _open_windows_shell(self, channel, key_handlers, handle_window_size): - def receive(): + def _open_windows_shell( + self, + channel: 'FileDescriptorLike', + key_handlers: Mapping[str, Callable[[Self], object]] | None, + handle_window_size: bool, + ) -> None: + """ + Runs the interactive session on Windows: receives data in a thread, + and passes the typed characters to the key handlers or sends them, + until the input ends. The channel and handle_window_size are not used. + """ + + def receive() -> None: # NOTE: Windows stdin is always cooked, remote may send double-echos. # With remote PTY/Shell connections we might see our echos from # our cooked stdin and also from the remote channel. Needs to be @@ -1844,20 +2060,38 @@ def receive(): except EOFError: self._dbg(1, 'User hit ^Z or F6') - def _open_shell(self, channel, key_handlers, handle_window_size): + def _open_shell( + self, + channel: 'FileDescriptorLike', + key_handlers: Mapping[str, Callable[[Self], object]] | None, + handle_window_size: bool, + ) -> None: + """ + Runs the interactive session on the local terminal, with + _open_posix_shell() if termios is available, else with + _open_windows_shell(). + """ if _have_termios: return self._open_posix_shell(channel, key_handlers, handle_window_size) else: return self._open_windows_shell(channel, key_handlers, handle_window_size) - def _interact(self, key_handlers=None, handle_window_size=True): + def _interact( + self, + key_handlers: Mapping[str, Callable[[Self], object]] | None = None, + handle_window_size: bool = True, + ) -> None: """ Should be overwritten. That's the place where the interactive shell is opened. """ raise NotImplementedError() - def interact(self, key_handlers=None, handle_window_size=True): + def interact( + self, + key_handlers: Mapping[str, Callable[[Self], object]] | None = None, + handle_window_size: bool = True, + ) -> None: """ Opens a simple interactive shell. Returns when the remote host sends EOF. @@ -1873,7 +2107,8 @@ def interact(self, key_handlers=None, handle_window_size=True): handle_window_size is not supported on Windows platforms. :type key_handlers: dict(str: callable) - :param key_handlers: A dictionary mapping chars to a functions. + :param key_handlers: A dictionary mapping chars to functions, which + are called with this connection instead of sending the char. :type handle_window_size: bool :param handle_window_size: Whether the connected host is notified when the terminal size changes. @@ -1881,7 +2116,7 @@ def interact(self, key_handlers=None, handle_window_size=True): self._dbg(1, "Switching to interactive session.") self._interact(key_handlers=key_handlers, handle_window_size=handle_window_size) - def _capture_device_context(self): + def _capture_device_context(self) -> dict[str, Any]: """Captures all context-specific state into a plain dict snapshot.""" return { # Authentication @@ -1904,7 +2139,7 @@ def _capture_device_context(self): 'manual_login_error_re': self._manual_login_error_re, } - def switch_device_context(self, *, reset_auth=True): + def switch_device_context(self, *, reset_auth: bool = True) -> None: """ Saves the current OS guesser, driver, and prompt state onto an internal stack and resets them for a new device context. Calling the method @@ -1964,7 +2199,7 @@ def switch_device_context(self, *, reset_auth=True): self._manual_password_re = None self._manual_login_error_re = None - def _restore_device_context(self, snapshot): + def _restore_device_context(self, snapshot: dict[str, Any]) -> None: """Restores all context-specific state from a snapshot dict.""" # Authentication self._app_authenticated = snapshot['app_authenticated'] @@ -1986,7 +2221,7 @@ def _restore_device_context(self, snapshot): self._manual_password_re = snapshot['manual_password_re'] self._manual_login_error_re = snapshot['manual_login_error_re'] - def restore_device_context(self): + def restore_device_context(self) -> None: """ Pops and fully restores the most recently saved device context from the internal stack, including the OS guesser state, the driver, all prompt @@ -1999,7 +2234,7 @@ def restore_device_context(self): raise ProtocolException('No saved device context to restore.') self._restore_device_context(self._device_context_stack.pop()) - def _logout(self): + def _logout(self) -> None: """ Logout from the remote host using the driver-specific command. """ @@ -2010,14 +2245,14 @@ def _logout(self): except Exception: pass - def _close(self, force=False): + def _close(self, force: bool = False) -> None: """ Should be overwritten. That's the place where the connection should be closed and protocol-specific references/resources cleared. """ raise NotImplementedError() - def close(self, force=False): + def close(self, force: bool = False) -> None: """ Closes the connection with the remote host. The instance may be reconnected afterwards by calling connect() again. diff --git a/src/Exscript/util/match.py b/src/Exscript/util/match.py index cde1f874..79e82214 100644 --- a/src/Exscript/util/match.py +++ b/src/Exscript/util/match.py @@ -25,11 +25,19 @@ """ import re +from typing import Any from ..protocols import Protocol +# The functions return Any: the shape of a match (None, a string, or a tuple) +# depends on the number of groups in the regular expression. -def _first_match(string, compiled): + +def _first_match(string: str, compiled: re.Pattern[str]) -> Any: + """ + Returns the first match of the compiled regular expression in the + given string, shaped as described in first_match(). + """ match = compiled.search(string) if match is None and compiled.groups <= 1: return None @@ -43,7 +51,7 @@ def _first_match(string, compiled): return match.groups() -def first_match(string, regex, flags=re.M): +def first_match(string: str | Protocol, regex: str, flags: int = re.M) -> Any: r""" Matches the given string against the given regex. @@ -87,7 +95,7 @@ def first_match(string, regex, flags=re.M): return _first_match(string, re.compile(regex, flags)) -def any_match(string, regex, flags=re.M): +def any_match(string: str | Protocol, regex: str, flags: int = re.M) -> list[Any]: r""" Matches the given string against the given regex. @@ -135,7 +143,7 @@ def any_match(string, regex, flags=re.M): if isinstance(string, Protocol): string = string.get_response() compiled = re.compile(regex, flags) - results = [] + results: list[Any] = [] if compiled.groups <= 1: for line in string.split('\n'): match = _first_match(line, compiled) From 6c65f1e1a2d68d1d9c1f7b08aaeb2953833db100 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 4 Nov 2026 14:01:08 +0100 Subject: [PATCH 061/250] Annotate the template parser and interpreter Every function gets annotations for mypy --strict and a docstring. Template values are Any, as their items depend on the template. Apart from Scope setting its parent again to declare its type, only annotations, docstrings and typing.cast() are added. --- src/Exscript/interpreter/__init__.py | 5 + src/Exscript/interpreter/append.py | 42 +++- src/Exscript/interpreter/assign.py | 44 +++- src/Exscript/interpreter/code.py | 30 ++- src/Exscript/interpreter/enter.py | 41 +++- src/Exscript/interpreter/exception.py | 7 + src/Exscript/interpreter/execute.py | 70 ++++++- src/Exscript/interpreter/expression.py | 129 ++++++++++-- src/Exscript/interpreter/extract.py | 65 +++++- src/Exscript/interpreter/fail.py | 43 +++- src/Exscript/interpreter/functioncall.py | 52 ++++- src/Exscript/interpreter/ifcondition.py | 48 ++++- src/Exscript/interpreter/loop.py | 53 ++++- src/Exscript/interpreter/number.py | 30 ++- src/Exscript/interpreter/parser.py | 78 +++++-- src/Exscript/interpreter/program.py | 46 ++++- src/Exscript/interpreter/regex.py | 52 ++++- src/Exscript/interpreter/rpc.py | 33 ++- src/Exscript/interpreter/scope.py | 100 +++++++-- src/Exscript/interpreter/string.py | 104 ++++++++-- src/Exscript/interpreter/template.py | 50 ++++- src/Exscript/interpreter/term.py | 56 ++++- src/Exscript/interpreter/trying.py | 40 +++- src/Exscript/interpreter/variable.py | 41 +++- src/Exscript/parselib/__init__.py | 6 + src/Exscript/parselib/exception.py | 11 +- src/Exscript/parselib/lexer.py | 251 +++++++++++++++++++---- src/Exscript/parselib/token.py | 93 +++++++-- 28 files changed, 1419 insertions(+), 201 deletions(-) diff --git a/src/Exscript/interpreter/__init__.py b/src/Exscript/interpreter/__init__.py index 22d341a0..a978c80f 100644 --- a/src/Exscript/interpreter/__init__.py +++ b/src/Exscript/interpreter/__init__.py @@ -20,6 +20,11 @@ # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" +The interpreter of Exscript templates. Parser compiles a template; the +compiled template runs on a connection. +""" + from .parser import Parser __all__ = [ diff --git a/src/Exscript/interpreter/append.py b/src/Exscript/interpreter/append.py index 36262fc3..1a4d80af 100644 --- a/src/Exscript/interpreter/append.py +++ b/src/Exscript/interpreter/append.py @@ -20,12 +20,35 @@ # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -from ..parselib import Token +""" +The append statement of the template language. +""" + +from typing import TYPE_CHECKING + +from ..parselib import Lexer, Token from .expression import Expression +from .scope import Scope + +if TYPE_CHECKING: + from .parser import Parser class Append(Token): - def __init__(self, lexer, parser, parent): + """ + The statement ``append to ``, which appends the + value of the expression to the list in the variable. + """ + + def __init__(self, lexer: Lexer, parser: 'Parser', parent: Scope) -> None: + """ + Constructor. Parses the statement and defines the variable as an + empty list. + + :param lexer: The lexer that reads the template. + :param parser: The parser of the template. + :param parent: The scope that contains the statement. + """ Token.__init__(self, self.__class__.__name__, lexer, parser, parent) # First expect an expression. @@ -53,12 +76,23 @@ def __init__(self, lexer, parser, parent): # End of statement. self.mark_end(rstrip=True) - def value(self, context): + def value(self, context: Scope) -> int: + """ + Appends the value of the expression to the variable. + + :param context: The scope in which the template runs. + :return: 1. + """ existing = self.parent.get(self.varname) args = {self.varname: existing + self.expr.value(context)} self.parent.define(**args) return 1 - def dump(self, indent=0): + def dump(self, indent: int = 0) -> None: + """ + Prints the statement and its expression to stdout. + + :param indent: The indentation of this token. + """ print((' ' * indent) + self.name, "to", self.varname) self.expr.dump(indent + 1) diff --git a/src/Exscript/interpreter/assign.py b/src/Exscript/interpreter/assign.py index 10990d2e..faac8dc6 100644 --- a/src/Exscript/interpreter/assign.py +++ b/src/Exscript/interpreter/assign.py @@ -20,12 +20,36 @@ # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -from ..parselib import Token +""" +Variable assignments in the template language. +""" + +from typing import TYPE_CHECKING, Any + +from ..parselib import Lexer, Token from .expression import Expression +from .scope import Scope + +if TYPE_CHECKING: + from .parser import Parser class Assign(Token): - def __init__(self, lexer, parser, parent): + """ + The statement `` = ``, which assigns the value of + the expression to the variable. + """ + + def __init__(self, lexer: Lexer, parser: 'Parser', parent: Scope) -> None: + """ + Constructor. Parses the statement and declares the variable. Raises + a CompileError if the name starts and ends with two underscores, as + the names of internal variables do. + + :param lexer: The lexer that reads the template. + :param parser: The parser of the template. + :param parent: The scope that contains the statement. + """ Token.__init__(self, self.__class__.__name__, lexer, parser, parent) # Extract the variable name. @@ -56,12 +80,24 @@ def __init__(self, lexer, parser, parent): # End of statement. self.mark_end(rstrip=True) - def dump(self, indent=0): + def dump(self, indent: int = 0) -> None: + """ + Prints the statement and its expression to stdout. + + :param indent: The indentation of this token. + """ print((' ' * indent) + self.name, self.varname, 'start') self.expression.dump(indent + 1) print((' ' * indent) + self.name, self.varname, 'end') - def value(self, context): + # Any: the value of the expression (see Expression.value()). + def value(self, context: Scope) -> Any: + """ + Assigns the value of the expression to the variable. + + :param context: The scope in which the template runs. + :return: The value of the expression. + """ result = self.expression.value(context) self.parent.define(**{self.varname: result}) return result diff --git a/src/Exscript/interpreter/code.py b/src/Exscript/interpreter/code.py index 3942cb29..4a474739 100644 --- a/src/Exscript/interpreter/code.py +++ b/src/Exscript/interpreter/code.py @@ -20,10 +20,16 @@ # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" +Code blocks of a template: the statements in curly brackets. +""" + import re +from typing import TYPE_CHECKING import Exscript.interpreter.template +from ..parselib import Lexer from .append import Append from .assign import Assign from .enter import Enter @@ -36,10 +42,11 @@ from .string import varname_re from .trying import Try -# IDEA: Add support for 'None' in the template engine? -# Right now a regex group that does not capture anything might inject a none -# value into a variable. We would need to test this (e.g. in addition) and then -# check if there is some kind of None handling/comparison. If not, we should add it. +if TYPE_CHECKING: + from .parser import Parser + +# IDEA: Add support for 'None' in the template engine? A regex group that does +# not participate in a match stores None in a variable. varname = varname_re.pattern keywords = [ @@ -94,7 +101,20 @@ class Code(Scope): - def __init__(self, lexer, parser, parent): + """ + A code block: the statements in curly brackets, or the body of an if, + loop or try statement up to its else or end keyword. In a body, the text + between a closing and the next opening curly bracket is a Template. + """ + + def __init__(self, lexer: Lexer, parser: 'Parser', parent: Scope) -> None: + """ + Constructor. Parses the statements of the block. + + :param lexer: The lexer that reads the template. + :param parser: The parser of the template. + :param parent: The parent scope. + """ Scope.__init__(self, self.__class__.__name__, lexer, parser, parent) lexer.set_grammar(grammar_c) lexer.next_if('open_curly_bracket') diff --git a/src/Exscript/interpreter/enter.py b/src/Exscript/interpreter/enter.py index 3b46ca77..bcb584bd 100644 --- a/src/Exscript/interpreter/enter.py +++ b/src/Exscript/interpreter/enter.py @@ -20,21 +20,54 @@ # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -from ..parselib import Token +""" +The enter statement of the template language. +""" + +from typing import TYPE_CHECKING + +from ..parselib import Lexer, Token from .execute import Execute +from .scope import Scope + +if TYPE_CHECKING: + from .parser import Parser class Enter(Token): - def __init__(self, lexer, parser, parent): + """ + The statement ``enter``, which sends an empty command to the remote + host. + """ + + def __init__(self, lexer: Lexer, parser: 'Parser', parent: Scope) -> None: + """ + Constructor. Parses the statement. + + :param lexer: The lexer that reads the template. + :param parser: The parser of the template. + :param parent: The scope that contains the statement. + """ Token.__init__(self, self.__class__.__name__, lexer, parser, parent) lexer.expect(self, 'keyword', 'enter') self.execute = Execute(lexer, parser, parent, '') # End of statement. self.mark_end(rstrip=True) - def value(self, context): + def value(self, context: Scope) -> int: + """ + Sends the empty command. + + :param context: The scope in which the template runs. + :return: 1. + """ return self.execute.value(context) - def dump(self, indent=0): + def dump(self, indent: int = 0) -> None: + """ + Prints the statement and its command to stdout. + + :param indent: The indentation of this token. + """ print((' ' * indent) + self.name) self.execute.dump(indent + 1) diff --git a/src/Exscript/interpreter/exception.py b/src/Exscript/interpreter/exception.py index ae16a38e..b1ffd6c3 100644 --- a/src/Exscript/interpreter/exception.py +++ b/src/Exscript/interpreter/exception.py @@ -20,9 +20,16 @@ # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" +Exceptions of the template interpreter. +""" class ExscriptException(Exception): + """ + Base class for the exceptions of the template interpreter. + """ + pass diff --git a/src/Exscript/interpreter/execute.py b/src/Exscript/interpreter/execute.py index 960e7f97..45de6f25 100644 --- a/src/Exscript/interpreter/execute.py +++ b/src/Exscript/interpreter/execute.py @@ -20,13 +20,22 @@ # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" +The commands of a template, which are sent to the remote host. +""" + import re import textwrap +from typing import TYPE_CHECKING, cast -from ..parselib import Token +from ..parselib import Lexer, Token from ..util.xml import validate_xml +from .scope import Scope from .string import String, varstring_re +if TYPE_CHECKING: + from .parser import Parser + grammar = { 'leading_blank_lines': r'^(?:[ \t]*[\r\n])+', 'trailing_blank_lines': r'(?:[\r\n][ \t]*)+$', @@ -38,7 +47,31 @@ class Execute(String): - def __init__(self, lexer, parser, parent, command, escape_offset=0): + """ + A command that is sent to the remote host. Variables in the command + ($name) are replaced by their values when the command is sent, and the + response of the host is stored in the __response__ variable. + """ + + def __init__( + self, lexer: Lexer, parser: 'Parser', parent: Scope, command: str, escape_offset: int = 0 + ) -> None: + """ + Constructor. Checks that the variables in the command are defined, + and declares the __response__ variable. In an element, it also + checks the XML syntax of the command: an error raises a + CompileError, or emits a CompileWarning if the command contains + variables. + + :param lexer: The lexer that reads the template, positioned behind + the command. + :param parser: The parser of the template. + :param parent: The scope that contains the command. + :param command: The command as read from the template. + :param escape_offset: The number of characters by which the + evaluation of escape sequences shortened the command, for the + position of the command in error messages. + """ Token.__init__(self, self.__class__.__name__, lexer, parser, parent) self.delimiter = '' # Command strings are not delimited in Execute tokens. self.string = command # Original command will be evaluated at runtime. @@ -73,8 +106,9 @@ def __init__(self, lexer, parser, parent, command, escape_offset=0): # The start mark points to the first non-whitespace character of the # RPC body. We will need the original start position of the command. + # (The pattern also matches an empty string, so it always matches.) original_start = self.start - len( - '' + leading_whitespace_re.search(self.string).group(0) + '' + cast(re.Match[str], leading_whitespace_re.search(self.string)).group(0) ) # Prepare full RPC command to eliminate most false-positives. @@ -97,12 +131,14 @@ def __init__(self, lexer, parser, parent, command, escape_offset=0): except XMLSyntaxError as exc: # XMLSyntaxError position (line, offset) needs to be converted # to an absolute character offset. Then adjust the start/end marks. + # lxml always sets lineno and offset, which SyntaxError types as + # int | None. lines = rpc_command.splitlines(keepends=True) - exc_start = len(''.join(lines[: exc.lineno - 1])) - exc_start += exc.offset + exc_start = len(''.join(lines[: cast(int, exc.lineno) - 1])) + exc_start += cast(int, exc.offset) # We assume the end positions are either both set or not! - exc_end = len(''.join(lines[: (exc.end_lineno or exc.lineno) - 1])) - exc_end += exc.end_offset or exc.offset + 1 + exc_end = len(''.join(lines[: (exc.end_lineno or cast(int, exc.lineno)) - 1])) + exc_end += exc.end_offset or cast(int, exc.offset) + 1 self.mark_start(original_start + exc_start) self.mark_end(distance=(exc_end - exc_start)) if has_vars: @@ -119,7 +155,18 @@ def __init__(self, lexer, parser, parent, command, escape_offset=0): self.parent.define(__response__=['']) - def value(self, context): + def value(self, context: Scope) -> int: + """ + Sends the command, without line comments, surrounding blank lines + and common indentation, and with the variables replaced by their + values. Waits for the prompt unless the parser was created with + no_prompt. Stores the response in the __response__ variable, one + line per item. Raises an ExecuteError if the __connection__ variable + is not defined. + + :param context: The scope in which the template runs. + :return: 1. + """ connvar = "__connection__" if not self.parent.is_defined(connvar): error = 'Undefined variable %s' % repr(connvar) @@ -144,5 +191,10 @@ def value(self, context): self.parent.define(__response__=response) return 1 - def dump(self, indent=0): + def dump(self, indent: int = 0) -> None: + """ + Prints the command to stdout. + + :param indent: The indentation of this token. + """ print((' ' * indent) + self.name, repr(self.string)) diff --git a/src/Exscript/interpreter/expression.py b/src/Exscript/interpreter/expression.py index f087d7af..7072f542 100644 --- a/src/Exscript/interpreter/expression.py +++ b/src/Exscript/interpreter/expression.py @@ -20,21 +20,53 @@ # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" +Expressions of the template language. +""" + +from typing import TYPE_CHECKING, Any, cast + import Exscript.interpreter.term -from ..parselib import Token +from ..parselib import Lexer, Token +from .scope import Scope + +if TYPE_CHECKING: + from .parser import Parser class ExpressionNode(Token): - def __init__(self, lexer, parser, parent, parent_node=None): + """ + A node of an expression: a term, optionally followed by an operator and + the rest of the expression, which is the right-hand node. An expression + that starts with "not" has no term. + """ + + def __init__( + self, + lexer: Lexer, + parser: 'Parser', + parent: Scope, + parent_node: 'ExpressionNode | None' = None, + ) -> None: + """ + Constructor. Parses a term and, if an operator follows, the operator + and the rest of the expression. + + :param lexer: The lexer that reads the template. + :param parser: The parser of the template. + :param parent: The scope that contains the expression. + :param parent_node: The node whose right-hand node this node is, or + None. + """ # Skip whitespace before initializing the token to make sure that self.start # points to the beginning of the expression (which makes for prettier error # messages). lexer.skip(['whitespace', 'newline']) Token.__init__(self, self.__class__.__name__, lexer, parser, parent) - self.lft = None - self.rgt = None + self.lft: Exscript.interpreter.term.Term | ExpressionNode | None = None + self.rgt: Exscript.interpreter.term.Term | ExpressionNode | None = None self.op = None self.op_type = None self.parent_node = parent_node @@ -74,7 +106,12 @@ def __init__(self, lexer, parser, parent, parent_node=None): # End of statement. self.mark_end(rstrip=True) - def priority(self): + def priority(self) -> int: + """ + Returns the priority of the operator of the node, from 1 for "and" + and "or" to 7 for "%", or 8 if the node has no operator. Raises an + Exception for an unknown operator. + """ if self.op is None: return 8 elif self.op_type == 'arithmetic_operator' and self.op == '%': @@ -96,25 +133,39 @@ def priority(self): else: raise Exception('Invalid operator: ' + repr(self.op)) - def value(self, context): + # Any: the value of a term, or a list with the result of the operator. + def value(self, context: Scope) -> Any: + """ + Evaluates the node. A list operand stands for its first item, or '' + if it is empty. Arithmetic operators raise an ExecuteError if an + operand is not a number. + + :param context: The scope in which the template runs. + :return: The value of the term if the node has no operator, or a + list with the result of the operator. + """ + # The operator decides which operands are set: lft is set unless the + # operator is "not", and rgt is set if there is an operator. # Special behavior where we only have one term. if self.op is None: - return self.lft.value(context) + return cast(Token, self.lft).value(context) elif self.op == 'not': - return [not self.rgt.value(context)[0]] + return [not cast(Token, self.rgt).value(context)[0]] # There are only two types of values: Regular expressions and lists. # We also have to make sure that empty lists do not cause an error. - lft_lst = self.lft.value(context) + # The first item of a list can be a string, a number or anything else + # that a function returned, hence Any. + lft_lst = cast(Token, self.lft).value(context) if type(lft_lst) == type([]): if len(lft_lst) > 0: - lft = lft_lst[0] + lft: Any = lft_lst[0] else: lft = '' - rgt_lst = self.rgt.value(context) + rgt_lst = cast(Token, self.rgt).value(context) if type(rgt_lst) == type([]): if len(rgt_lst) > 0: - rgt = rgt_lst[0] + rgt: Any = rgt_lst[0] else: rgt = '' @@ -178,7 +229,12 @@ def value(self, context): elif self.op == '-': return [int(lft) - int(rgt)] - def dump(self, indent=0): + def dump(self, indent: int = 0) -> None: + """ + Prints the node, its operator and its operands to stdout. + + :param indent: The indentation of this token. + """ print((' ' * indent) + self.name, self.op, 'start') if self.lft is not None: self.lft.dump(indent + 1) @@ -189,7 +245,20 @@ def dump(self, indent=0): class Expression(Token): - def __init__(self, lexer, parser, parent): + """ + An expression: a tree of ExpressionNode objects, ordered by the priority + of the operators. + """ + + def __init__(self, lexer: Lexer, parser: 'Parser', parent: Scope) -> None: + """ + Constructor. Parses the expression and reorders its nodes by the + priority of the operators. + + :param lexer: The lexer that reads the template. + :param parser: The parser of the template. + :param parent: The scope that contains the expression. + """ Token.__init__(self, self.__class__.__name__, lexer, parser, parent) # Parse the expression. @@ -201,7 +270,20 @@ def __init__(self, lexer, parser, parent): # End of statement. self.mark_end(rstrip=True) - def prioritize(self, start, prio=1): + # start is an ExpressionNode, a Term or None. It is Any because the code + # walks Terms like nodes (a Term has lft, rgt and op set to None, and the + # priority 6) and relies on Terms never reaching the reparenting part, + # which needs attributes that only nodes have. + def prioritize(self, start: Any, prio: int = 1) -> None: + """ + Reorders the tree from the given node on, so that operators with a + higher priority are evaluated first: the first node with the given + priority becomes the parent of the preceding nodes with a higher + priority. Continues recursively with the priorities up to 5. + + :param start: The node at which the reordering starts. + :param prio: The priority of the operators to move up. + """ # print("Prioritizing from", start.op, "with prio", prio, (start.lft,) # start.rgt) if prio == 6: @@ -252,10 +334,23 @@ def prioritize(self, start, prio=1): self.prioritize(current.lft, prio + 1) self.prioritize(current.rgt, prio) - def value(self, context): + # Any: see ExpressionNode.value(). + def value(self, context: Scope) -> Any: + """ + Evaluates the expression. + + :param context: The scope in which the template runs. + :return: The value of a single term, or a list with the result of + the operators. + """ return self.root.value(context) - def dump(self, indent=0): + def dump(self, indent: int = 0) -> None: + """ + Prints the expression tree to stdout. + + :param indent: The indentation of this token. + """ print((' ' * indent) + self.name, 'start') self.root.dump(indent + 1) print((' ' * indent) + self.name, 'end') diff --git a/src/Exscript/interpreter/extract.py b/src/Exscript/interpreter/extract.py index 3f4e34a2..85266634 100644 --- a/src/Exscript/interpreter/extract.py +++ b/src/Exscript/interpreter/extract.py @@ -20,16 +20,44 @@ # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -from ..parselib import Token +""" +The extract statement of the template language. +""" + +from typing import TYPE_CHECKING + +from ..parselib import Lexer, Token from .regex import Regex +from .scope import Scope from .term import Term +if TYPE_CHECKING: + from .parser import Parser + class Extract(Token): - def __init__(self, lexer, parser, parent): + """ + The statement ``extract // as|into [from ]``. + It matches each line of the response of the last command (or of the + given list) against the regular expression, and stores the capture + groups of the matching lines in the variables: "as" replaces their + values, "into" appends to them. + """ + + def __init__(self, lexer: Lexer, parser: 'Parser', parent: Scope) -> None: + """ + Constructor. Parses the statement and defines the variables as empty + lists. Raises a CompileError on syntax errors, if the parser has + no_prompt set (then there is no response), or if the number of + variables differs from the number of capture groups. + + :param lexer: The lexer that reads the template. + :param parser: The parser of the template. + :param parent: The scope that contains the statement. + """ Token.__init__(self, self.__class__.__name__, lexer, parser, parent) self.varnames = [] - self.variables = {} + self.variables: dict[str, list[str | None]] = {} self.append = False self.source = None @@ -102,7 +130,15 @@ def __init__(self, lexer, parser, parent): # End of statement. self.mark_end(rstrip=True) - def extract(self, context): + def extract(self, context: Scope) -> None: + """ + Matches each line of the source against the regular expression, and + collects the capture groups of the matching lines in the variables + attribute, one list per variable. A group that did not participate + in a match adds None. + + :param context: The scope in which the template runs. + """ # Re-initialize the variable content, because this method # might be called multiple times. for varname in self.varnames: @@ -128,15 +164,21 @@ def extract(self, context): try: value = match.group(i) except IndexError: - # This happens if the user provided a regex with less - # groups in it than the number of variables. This should have - # been caught already during parsing (comparing group count). + # A variable in the regular expression can change its number + # of groups at runtime. msg = 'Extraction to variable #%s has no corresponding' % i msg += ' regex capture group (index out of range).' self.lexer.runtime_error(msg, self) self.variables[varname].append(value) - def value(self, context): + def value(self, context: Scope) -> int: + """ + Extracts the capture groups and stores them in the variables: "as" + replaces their values, "into" appends to them. + + :param context: The scope in which the template runs. + :return: 1. + """ self.extract(context) if not self.append: self.parent.define(**self.variables) @@ -146,7 +188,12 @@ def value(self, context): self.parent.define(**{key: existing + self.variables[key]}) return 1 - def dump(self, indent=0): + def dump(self, indent: int = 0) -> None: + """ + Prints the statement to stdout. + + :param indent: The indentation of this token. + """ mode = self.append and 'into' or 'as' source = self.source is not None and self.source or '__response__' print( diff --git a/src/Exscript/interpreter/fail.py b/src/Exscript/interpreter/fail.py index 10061809..a0a919d0 100644 --- a/src/Exscript/interpreter/fail.py +++ b/src/Exscript/interpreter/fail.py @@ -20,13 +20,36 @@ # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -from ..parselib import Token +""" +The fail statement of the template language. +""" + +from typing import TYPE_CHECKING + +from ..parselib import Lexer, Token from .exception import FailException from .expression import Expression +from .scope import Scope + +if TYPE_CHECKING: + from .parser import Parser class Fail(Token): - def __init__(self, lexer, parser, parent): + """ + The statement ``fail [if ]``, which raises a + FailException with the value of the first expression as the message, + if the condition is true or not given. + """ + + def __init__(self, lexer: Lexer, parser: 'Parser', parent: Scope) -> None: + """ + Constructor. Parses the statement. + + :param lexer: The lexer that reads the template. + :param parser: The parser of the template. + :param parent: The scope that contains the statement. + """ Token.__init__(self, self.__class__.__name__, lexer, parser, parent) self.expression = None @@ -51,12 +74,24 @@ def __init__(self, lexer, parser, parent): # End of statement. self.mark_end(rstrip=True) - def value(self, context): + def value(self, context: Scope) -> int: + """ + Raises a FailException with the first item of the message, if the + condition is true or not given. + + :param context: The scope in which the template runs. + :return: 1 if the condition is false. + """ if self.expression is None or self.expression.value(context)[0]: raise FailException(self.msg.value(context)[0]) return 1 - def dump(self, indent=0): + def dump(self, indent: int = 0) -> None: + """ + Prints the statement and its expressions to stdout. + + :param indent: The indentation of this token. + """ print((' ' * indent) + self.name, 'start') self.msg.dump(indent + 1) if self.expression: diff --git a/src/Exscript/interpreter/functioncall.py b/src/Exscript/interpreter/functioncall.py index 017713b4..46abb1ef 100644 --- a/src/Exscript/interpreter/functioncall.py +++ b/src/Exscript/interpreter/functioncall.py @@ -20,14 +20,39 @@ # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" +Function calls in the template language. +""" + +from typing import TYPE_CHECKING, Any, cast + import Exscript.interpreter.expression -from ..parselib import Token +from ..parselib import Lexer, Token from .exception import PermissionError +from .scope import Scope + +if TYPE_CHECKING: + from .parser import Parser class FunctionCall(Token): - def __init__(self, lexer, parser, parent): + """ + A call of a function, such as ``string.replace(x, "a", "b")``. The + function is a variable of the scope; it is called with the scope and + the values of the arguments. + """ + + def __init__(self, lexer: Lexer, parser: 'Parser', parent: Scope) -> None: + """ + Constructor. Parses the call. Raises a CompileError if the function + is not defined, and a PermissionError if the parser is in secure + mode and the function is not marked as secure. + + :param lexer: The lexer that reads the template. + :param parser: The parser of the template. + :param parent: The scope that contains the call. + """ Token.__init__(self, self.__class__.__name__, lexer, parser, parent) self.funcname = None self.arguments = [] @@ -62,22 +87,37 @@ def __init__(self, lexer, parser, parent): # End of statement. self.mark_end(rstrip=True) - def dump(self, indent=0): + def dump(self, indent: int = 0) -> None: + """ + Prints the call and its arguments to stdout. + + :param indent: The indentation of this token. + """ print((' ' * indent) + self.name, self.funcname, 'start') for argument in self.arguments: argument.dump(indent + 1) print((' ' * indent) + self.name, self.funcname, 'end') - def value(self, context): + # Any: the result of the function. + def value(self, context: Scope) -> Any: + """ + Calls the function with the scope and the values of the arguments. + Raises an ExecuteError if the function is not defined, or if it + raises an exception. + + :param context: The scope in which the template runs. + :return: The result of the function. + """ argument_values = [arg.value(context) for arg in self.arguments] function = self.parent.get(self.funcname) + # The constructor sets funcname. if function is None: - self.mark_end(distance=len(self.funcname)) + self.mark_end(distance=len(cast(str, self.funcname))) self.lexer.runtime_error('Undefined function %s' % repr(self.funcname), self) try: result = function(self.parent, *argument_values) except Exception as e: - self.mark_end(distance=len(self.funcname)) + self.mark_end(distance=len(cast(str, self.funcname))) self.lexer.runtime_error( '%s while executing function %s: %s' % (type(e).__name__, repr(self.funcname), e), self, diff --git a/src/Exscript/interpreter/ifcondition.py b/src/Exscript/interpreter/ifcondition.py index 3aa5873e..2e13ae19 100644 --- a/src/Exscript/interpreter/ifcondition.py +++ b/src/Exscript/interpreter/ifcondition.py @@ -20,14 +20,37 @@ # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" +The if statement of the template language. +""" + +from typing import TYPE_CHECKING, Any + import Exscript.interpreter.code -from ..parselib import Token +from ..parselib import Lexer, Token from .expression import Expression +from .scope import Scope + +if TYPE_CHECKING: + from .parser import Parser class IfCondition(Token): - def __init__(self, lexer, parser, parent): + """ + The if statement, with an optional else block. An ``else if`` is + stored as a nested IfCondition in the else block. + """ + + def __init__(self, lexer: Lexer, parser: 'Parser', parent: Scope) -> None: + """ + Constructor. Parses the statement up to and including its end + keyword. + + :param lexer: The lexer that reads the template. + :param parser: The parser of the template. + :param parent: The scope that contains the statement. + """ Token.__init__(self, self.__class__.__name__, lexer, parser, parent) # Expect an expression. @@ -37,8 +60,9 @@ def __init__(self, lexer, parser, parent): # Body of the if block. self.if_block = Exscript.interpreter.code.Code(lexer, parser, parent) - self.elif_blocks = [] - self.else_block = None + # Any: never filled; an "else if" is stored in else_block. + self.elif_blocks: list[Any] = [] + self.else_block: IfCondition | Exscript.interpreter.code.Code | None = None # If there is no "else" statement, just return. lexer.skip(['whitespace', 'newline']) @@ -62,14 +86,26 @@ def __init__(self, lexer, parser, parent): # End of statement. self.mark_end(rstrip=True) - def value(self, context): + def value(self, context: Scope) -> int: + """ + Evaluates the if block if the first item of the condition is true, + and the else block otherwise. + + :param context: The scope in which the template runs. + :return: 1. + """ if self.expression.value(context)[0]: self.if_block.value(context) elif self.else_block is not None: self.else_block.value(context) return 1 - def dump(self, indent=0): + def dump(self, indent: int = 0) -> None: + """ + Prints the statement, its condition and its blocks to stdout. + + :param indent: The indentation of this token. + """ print((' ' * indent) + self.name, 'start') self.expression.dump(indent + 1) self.if_block.dump(indent + 1) diff --git a/src/Exscript/interpreter/loop.py b/src/Exscript/interpreter/loop.py index fbd23313..a24ad266 100644 --- a/src/Exscript/interpreter/loop.py +++ b/src/Exscript/interpreter/loop.py @@ -20,15 +20,43 @@ # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" +The loop statement of the template language. +""" + +from typing import TYPE_CHECKING, cast + import Exscript.interpreter.code -from ..parselib import Token +from ..parselib import Lexer, Token from .expression import Expression +from .scope import Scope from .term import Term +if TYPE_CHECKING: + from .parser import Parser + class Loop(Token): - def __init__(self, lexer, parser, parent): + """ + The loop statement in three forms: ``loop as `` + iterates over lists in parallel, ``loop from to [as + ]`` over a range of numbers (in the variable "counter" by + default), and ``loop while|until `` while a condition + holds. The first two forms may also have a while or until condition. + """ + + def __init__(self, lexer: Lexer, parser: 'Parser', parent: Scope) -> None: + """ + Constructor. Parses the statement up to and including its end + keyword, and defines the loop variables as empty lists. Raises a + CompileError if the number of lists differs from the number of + variables. + + :param lexer: The lexer that reads the template. + :param parser: The parser of the template. + :param parent: The scope that contains the statement. + """ Token.__init__(self, self.__class__.__name__, lexer, parser, parent) self.during = None self.until = None @@ -114,7 +142,16 @@ def __init__(self, lexer, parser, parent): # End of statement. self.mark_end(rstrip=True) - def value(self, context): + def value(self, context: Scope) -> int: + """ + Runs the loop. A range loop counts from the first number up to, but + not including, the second. A while or until condition is checked + before each iteration. Raises an ExecuteError if the lists have + different lengths. + + :param context: The scope in which the template runs. + :return: 1. + """ if len(self.list_variables) == 0 and not self.thefrom: # If this is a "while" loop, iterate as long as the condition is # True. @@ -132,7 +169,8 @@ def value(self, context): # Retrieve the lists from the list terms. if self.thefrom: start = self.thefrom.value(context)[0] - stop = self.theto.value(context)[0] + # The constructor sets theto together with thefrom. + stop = cast(Expression, self.theto).value(context)[0] lists = [list(range(start, stop))] else: lists = [var.value(context) for var in self.list_variables] @@ -157,7 +195,12 @@ def value(self, context): self.block.value(context) return 1 - def dump(self, indent=0): + def dump(self, indent: int = 0) -> None: + """ + Prints the statement, its conditions and its block to stdout. + + :param indent: The indentation of this token. + """ print((' ' * indent) + self.name, end=' ') print(self.list_variables, 'as', ', '.join(self.iter_varnames), 'start') if self.during is not None: diff --git a/src/Exscript/interpreter/number.py b/src/Exscript/interpreter/number.py index 98651abe..41f41085 100644 --- a/src/Exscript/interpreter/number.py +++ b/src/Exscript/interpreter/number.py @@ -20,14 +20,38 @@ # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" +Number literals of the template language. +""" + +from .scope import Scope class Number: - def __init__(self, number): + """ + A number in a template, and the keywords true (1) and false (0). + """ + + def __init__(self, number: str | int) -> None: + """ + Constructor. + + :param number: The number, or its decimal digits. + """ self.number = int(number) - def value(self, context): + def value(self, context: Scope) -> list[int]: + """ + Returns the number as a list with one item. + + :param context: The scope in which the template runs. + """ return [self.number] - def dump(self, indent=0): + def dump(self, indent: int = 0) -> None: + """ + Prints the number to stdout. + + :param indent: The indentation of this token. + """ print((' ' * indent) + 'Number', self.number) diff --git a/src/Exscript/interpreter/parser.py b/src/Exscript/interpreter/parser.py index e486943b..732f7f76 100644 --- a/src/Exscript/interpreter/parser.py +++ b/src/Exscript/interpreter/parser.py @@ -20,47 +20,93 @@ # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" +The parser of the template language. +""" + import copy from collections.abc import Callable, Iterable +from typing import Any, cast from ..parselib import Lexer from .program import Program class Parser: - def __init__(self, **kwargs): - self.no_escaping = kwargs.get('no_escaping', False) - self.no_prompt = kwargs.get('no_prompt', False) - self.strip_command = kwargs.get('strip_command', True) - self.secure_only = kwargs.get('secure', False) - self.debug = kwargs.get('debug', 0) - self.variables = {} - - def define(self, **kwargs): + """ + Compiles templates. The variables that are defined on the parser are + available in every template that it compiles. + """ + + def __init__(self, **kwargs: Any) -> None: + """ + Constructor. + + :param kwargs: The options ``no_escaping`` (evaluate no escape + sequences, except for CR, LF and TAB in strings), ``no_prompt`` + (do not wait for a prompt after each command), ``strip_command`` + (strip the echo of the command from the response, default True), + ``secure`` (forbid the functions that are not marked as secure) + and ``debug`` (with 5 or more, dump the compiled template). + """ + self.no_escaping: bool = kwargs.get('no_escaping', False) + self.no_prompt: bool = kwargs.get('no_prompt', False) + self.strip_command: bool = kwargs.get('strip_command', True) + self.secure_only: bool = kwargs.get('secure', False) + self.debug: int = kwargs.get('debug', 0) + self.variables: dict[str, Any] = {} + + def define(self, **kwargs: Any) -> None: + """ + Defines the given variables. A value that is a string or not + iterable is wrapped in a list, unless it is callable. + """ for key, value in list(kwargs.items()): if ( not isinstance(value, str) and isinstance(value, Iterable) - or isinstance(value, Callable) + # mypy takes Callable for a special form, not for the ABC it is. + or isinstance(value, cast(type, Callable)) ): self.variables[key] = value else: self.variables[key] = [value] - def define_object(self, **kwargs): + def define_object(self, **kwargs: Any) -> None: + """ + Defines the given variables, with the values as given. + """ self.variables.update(kwargs) - def _create_lexer(self): + def _create_lexer(self) -> Lexer: + """ + Returns a new lexer that creates a Program with a deep copy of the + variables. + """ variables = copy.deepcopy(self.variables) return Lexer(Program, self, variables, debug=self.debug) - def parse(self, string, filename=None): + def parse(self, string: str, filename: str | None = None) -> Program: + """ + Compiles the given template. + + :param string: The template. + :param filename: The file name that error messages show. + :return: The compiled template. + """ lexer = self._create_lexer() - return lexer.parse(string, filename) + # The lexer returns what the parser class (Program) creates. + return cast(Program, lexer.parse(string, filename)) + + def parse_file(self, filename: str) -> Program: + """ + Compiles the template in the given UTF-8 file. - def parse_file(self, filename): + :param filename: The name of the file. + :return: The compiled template. + """ lexer = self._create_lexer() - return lexer.parse_file(filename) + return cast(Program, lexer.parse_file(filename)) if __name__ == "__main__": diff --git a/src/Exscript/interpreter/program.py b/src/Exscript/interpreter/program.py index 17de1b80..8a9fb2c5 100644 --- a/src/Exscript/interpreter/program.py +++ b/src/Exscript/interpreter/program.py @@ -20,20 +20,50 @@ # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" +The root of the parse tree of a template. +""" + import copy +from typing import TYPE_CHECKING, Any, cast +from ..parselib import Lexer from .scope import Scope from .template import Template +if TYPE_CHECKING: + from .parser import Parser + class Program(Scope): - def __init__(self, lexer, parser, variables, **kwargs): + """ + The root scope of a compiled template. It holds the variables of the + template and runs it with execute(). + """ + + def __init__( + self, lexer: Lexer, parser: 'Parser', variables: dict[str, Any], **kwargs: Any + ) -> None: + """ + Constructor. Parses the template that the lexer reads. + + :param lexer: The lexer that reads the template. + :param parser: The parser of the template. + :param variables: The variables that are defined when the template + starts; execute() starts with a copy of them. + :param kwargs: Passed to Scope. + """ Scope.__init__(self, 'Program', lexer, parser, None, **kwargs) self.variables = variables self.init_variables = variables self.add(Template(lexer, parser, self)) - def init(self, *args, **kwargs): + def init(self, *args: Any, **kwargs: Any) -> None: + """ + Sets the initial values of the given variables. A value that is not + a list is wrapped in a list. Names that contain a dot or start with + an underscore are ignored. + """ for key in kwargs: if key.find('.') >= 0 or key.startswith('_'): continue @@ -42,9 +72,17 @@ def init(self, *args, **kwargs): else: self.init_variables[key] = [kwargs[key]] - def execute(self, *args, **kwargs): + def execute(self, *args: Any, **kwargs: Any) -> dict[str, Any]: + """ + Runs the template with a copy of the initial variables. + + :param kwargs: ``variables``: a dict of variables that are defined + in addition, or replace the initial ones. + :return: The variables after the template has run. + """ self.variables = copy.copy(self.init_variables) if 'variables' in kwargs: - self.variables.update(kwargs.get('variables')) + # The key exists, so get() returns the dict of the caller. + self.variables.update(cast(dict[str, Any], kwargs.get('variables'))) self.value(self) return self.variables diff --git a/src/Exscript/interpreter/regex.py b/src/Exscript/interpreter/regex.py index bfd98db5..346d27f9 100644 --- a/src/Exscript/interpreter/regex.py +++ b/src/Exscript/interpreter/regex.py @@ -20,10 +20,20 @@ # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" +Regular expressions in the template language. +""" + import re +from typing import TYPE_CHECKING +from ..parselib import Lexer +from .scope import Scope from .string import String +if TYPE_CHECKING: + from .parser import Parser + modifier_grammar = ( ('modifier', r'[a-zA-Z]'), ('invalid_char', r'.'), # used to implicitly exit the loop @@ -35,7 +45,22 @@ class Regex(String): - def __init__(self, lexer, parser, parent): + """ + A regular expression between slashes, such as ``/^interface (.*)$/i``. + The only modifier is i, which ignores case. Variables in the expression + are replaced by their values when it is evaluated. + """ + + def __init__(self, lexer: Lexer, parser: 'Parser', parent: Scope) -> None: + """ + Constructor. Parses the expression and its modifiers, and compiles it + to count its capture groups. Raises a CompileError for an unknown + modifier or an invalid expression. + + :param lexer: The lexer that reads the template. + :param parser: The parser of the template. + :param parent: The scope that contains the expression. + """ # String parser collects the regex. String.__init__(self, lexer, parser, parent, purpose='regex') self.groups = 0 @@ -66,7 +91,15 @@ def __init__(self, lexer, parser, parent): else: self.groups = regex.groups - def eval_escape(self, ttype, token): + def eval_escape(self, ttype: str, token: str) -> str: + """ + Returns the given escape sequence unchanged, for the compiler of + regular expressions, except for an escaped delimiter, which is + returned without the backslash. + + :param ttype: The token type of the escape sequence. + :param token: The escape sequence. + """ # Simple escape sequence with one character. if len(token) == 2: char = token[1] @@ -75,9 +108,20 @@ def eval_escape(self, ttype, token): # Anything else will be handled by Python's regex compiler. return token - def value(self, context): + def value(self, context: Scope) -> re.Pattern[str]: + """ + Returns the compiled expression, with the variables replaced by their + values. + + :param context: The scope in which the template runs. + """ pattern = String.value(self, context)[0] return re.compile(pattern, self.flags) - def dump(self, indent=0): + def dump(self, indent: int = 0) -> None: + """ + Prints the expression to stdout. + + :param indent: The indentation of this token. + """ print((' ' * indent) + self.name, repr(self.string)) diff --git a/src/Exscript/interpreter/rpc.py b/src/Exscript/interpreter/rpc.py index 0ba9f546..b5f82687 100644 --- a/src/Exscript/interpreter/rpc.py +++ b/src/Exscript/interpreter/rpc.py @@ -20,11 +20,20 @@ # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" +NETCONF RPCs in a template: the elements. +""" + import re +from typing import TYPE_CHECKING, Any +from ..parselib import Lexer from .execute import Execute from .scope import Scope +if TYPE_CHECKING: + from .parser import Parser + grammar = ( ('line_comment', r'(?:^|(?<=[\r\n]))[ \t]*#[^\r\n]*(?:[\r\n]|$)'), ('open_rpc_tag', r''), @@ -40,7 +49,21 @@ class RPCBody(Scope): - def __init__(self, lexer, parser, parent): + """ + An element of a template. Its body, without the rpc tags, becomes + a command (an Execute token) that is sent to the remote host. The XML + syntax of the body is checked when the template is compiled. + """ + + def __init__(self, lexer: Lexer, parser: 'Parser', parent: Scope) -> None: + """ + Constructor. Parses the element from the opening to the closing rpc + tag. + + :param lexer: The lexer that reads the template. + :param parser: The parser of the template. + :param parent: The parent scope. + """ Scope.__init__(self, self.__class__.__name__, lexer, parser, parent) lexer.set_grammar(grammar_c) lexer.expect(self, 'open_rpc_tag') @@ -68,5 +91,11 @@ def __init__(self, lexer, parser, parent): # End of statement. self.mark_end(rstrip=True) - def execute(self): + # Any: the value of the last child (see Scope.value()). + def execute(self) -> Any: + """ + Evaluates the element, with the element itself as the context. + + :return: The value of the last child, or 1 if there are no children. + """ return self.value(self) diff --git a/src/Exscript/interpreter/scope.py b/src/Exscript/interpreter/scope.py index 400753a8..6d078d05 100644 --- a/src/Exscript/interpreter/scope.py +++ b/src/Exscript/interpreter/scope.py @@ -20,15 +20,50 @@ # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" +The base class of the tokens that hold variables. +""" + import copy +from typing import TYPE_CHECKING, Any + +from ..parselib import Lexer, Token -from ..parselib import Token +if TYPE_CHECKING: + from .parser import Parser class Scope(Token): - def __init__(self, name, lexer, parser, parent=None, *args, **kwargs): + """ + A token that holds variables and evaluates its children in order. + define() stores variables in the root scope (the Program); get() looks + them up in the scope and then in its parents. + """ + + def __init__( + self, + name: str, + lexer: Lexer, + parser: 'Parser', + parent: 'Scope | None' = None, + *args: Any, + **kwargs: Any, + ) -> None: + """ + Constructor. Raises TypeError if the value of a given variable is not + a list, unless its name contains a dot or starts with an underscore. + + :param name: The name of the token. + :param lexer: The lexer that reads the template. + :param parser: The parser of the template. + :param parent: The parent scope, or None for the root scope. + :param kwargs: ``variables``: a dict that maps the names of the + variables in the scope to their values. + """ Token.__init__(self, name, lexer, parser, parent) - self.variables = kwargs.get('variables', {}) + # Token declares the parent as Any; the parent of a scope is a scope. + self.parent: Scope | None = parent + self.variables: dict[str, Any] = kwargs.get('variables', {}) self.children = [] self.exit_requested = 0 for key in self.variables: @@ -38,10 +73,19 @@ def __init__(self, name, lexer, parser, parent=None, *args, **kwargs): msg = '%s: variable %s must be a list, but got %s' raise TypeError(msg % (self.name, key, type(self.variables[key]))) - def exit_request(self): + def exit_request(self) -> None: + """ + Sets the exit_requested flag. A Template stops parsing when its flag + is set. + """ self.exit_requested = 1 - def define(self, **kwargs): + def define(self, **kwargs: Any) -> None: + """ + Defines the given variables in the root scope. A value that is not + a list is wrapped in a list, unless the name contains a dot (as the + names of functions do) or starts with an underscore. + """ if self.parent is not None: return self.parent.define(**kwargs) for key in kwargs: @@ -50,17 +94,24 @@ def define(self, **kwargs): else: self.variables[key] = [kwargs[key]] - def define_object(self, **kwargs): + def define_object(self, **kwargs: Any) -> None: + """ + Defines the given variables in this scope, with the values as given. + """ self.variables.update(kwargs) - def is_defined(self, name): + def is_defined(self, name: str) -> int: + """ + Returns 1 if a variable with the given name is defined in this scope + or in one of its parents, 0 otherwise. + """ if name in self.variables: return 1 if self.parent is not None: return self.parent.is_defined(name) return 0 - def get_vars(self): + def get_vars(self) -> dict[str, Any]: """ Returns a complete dict of all variables that are defined in this scope, including the variables of the parent. @@ -73,7 +124,7 @@ def get_vars(self): vars.update(self.variables) return vars - def copy_public_vars(self): + def copy_public_vars(self) -> dict[str, Any]: """ Like get_vars(), but does not include any private variables and deep copies each variable. @@ -82,26 +133,49 @@ def copy_public_vars(self): vars = dict([k for k in list(vars.items()) if not k[0].startswith('_')]) return copy.deepcopy(vars) - def get(self, name, default=None): + # Any: the values of variables are lists (mostly of strings), functions, + # and objects such as the connection. + def get(self, name: str, default: Any = None) -> Any: + """ + Returns the value of the variable with the given name from this + scope or from the nearest parent that defines it, or the given + default if no scope defines it. + """ if name in self.variables: return self.variables[name] if self.parent is None: return default return self.parent.get(name, default) - def value(self, context): + # Any: the value of the last child, which depends on its class. + def value(self, context: 'Scope') -> Any: + """ + Evaluates the children in order. + + :param context: The scope in which the template runs. + :return: The value of the last child, or 1 if there are no children. + """ result = 1 for child in self.children: result = child.value(context) return result - def dump(self, indent=0): + def dump(self, indent: int = 0) -> None: + """ + Prints the name of the scope to stdout, followed by a dump of each + child with one more level of indentation, and an end line. + + :param indent: The indentation of this token. + """ print((' ' * indent) + self.name, 'start') for child in self.children: child.dump(indent + 1) print((' ' * indent) + self.name, 'end') - def dump_vars(self): + def dump_vars(self) -> None: + """ + Prints the variables of the root scope to stdout. + """ if self.parent is not None: self.parent.dump_vars() return diff --git a/src/Exscript/interpreter/string.py b/src/Exscript/interpreter/string.py index e7b883a0..6a68fa5d 100644 --- a/src/Exscript/interpreter/string.py +++ b/src/Exscript/interpreter/string.py @@ -20,10 +20,19 @@ # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" +Strings of the template language, and the substitution of variables. +""" + import ast import re +from typing import TYPE_CHECKING, Any, cast + +from ..parselib import Lexer, Token +from .scope import Scope -from ..parselib import Token +if TYPE_CHECKING: + from .parser import Parser varname_re = re.compile(r'(__[a-zA-Z][\w_]*__|[a-zA-Z][\w_]*)') varstring_re = re.compile(r'(\\?)\$([\w_]*)') @@ -49,7 +58,27 @@ class String(Token): - def __init__(self, lexer, parser, parent, purpose='string'): + """ + A string in double quotes. Escape sequences are evaluated when the + template is compiled; variables in the string ($name) are replaced by + their values when it is evaluated. Regex and Execute derive from it. + """ + + def __init__( + self, lexer: Lexer, parser: 'Parser', parent: Scope, purpose: str = 'string' + ) -> None: + """ + Constructor. Parses the string from the current token, which is the + opening delimiter, to the closing delimiter. Raises a CompileError at + a line break, and an ExecuteError if the string contains a variable + that is not defined. + + :param lexer: The lexer that reads the template. + :param parser: The parser of the template. + :param parent: The scope that contains the string. + :param purpose: The kind of string ('string' or 'regex'), for the + token types and the error messages. + """ Token.__init__(self, self.__class__.__name__, lexer, parser, parent) # Create a grammar depending on the delimiting character. @@ -97,7 +126,18 @@ def __init__(self, lexer, parser, parent, purpose='string'): # End of statement. self.mark_end(rstrip=True) - def eval_escape(self, ttype, token): + def eval_escape(self, ttype: str, token: str) -> str: + """ + Returns the text that the given escape sequence stands for. Unicode, + hexadecimal and octal sequences and control characters are evaluated + unless the parser disables escaping; CR, LF and TAB are always + evaluated. An escaped dollar sign stays escaped for the substitution + of variables; any other escaped character is returned without the + backslash. + + :param ttype: The token type of the escape sequence. + :param token: The escape sequence. + """ # Supporting various types of escape sequences. The easiest way is to # evaluate them as strings with the builtin ast module. if not self.parser.no_escaping and ttype in ( @@ -109,9 +149,9 @@ def eval_escape(self, ttype, token): ): if token == '\\e': # Invalid sequence in Python. return '\x1b' - return ast.literal_eval('"' + token + '"') + return cast(str, ast.literal_eval('"' + token + '"')) if ttype == 'escaped_common_controls': - return ast.literal_eval('"' + token + '"') + return cast(str, ast.literal_eval('"' + token + '"')) # Simple escape sequence with one character. if len(token) == 2: char = token[1] @@ -120,7 +160,15 @@ def eval_escape(self, ttype, token): # Simply unescape anything else. return token[1:] - def _variable_error(self, field, field_start, msg): + def _variable_error(self, field: str, field_start: int, msg: str) -> None: + """ + Moves the marks of the token to the given variable reference, and + raises an ExecuteError with the given message. + + :param field: The variable reference, such as $name. + :param field_start: The position of the reference in the string. + :param msg: The error message. + """ # Narrow down the marks on the variable that caused the error. self.mark_start(self.start + len(self.delimiter) + field_start) self.mark_end(distance=len(field)) @@ -128,7 +176,14 @@ def _variable_error(self, field, field_start, msg): # Tokens that include variables in a string may use this callback to # substitute the variable against its value. - def variable_sub_cb(self, match): + def variable_sub_cb(self, match: re.Match[str]) -> str: + """ + Returns the value of the variable that the given match of + varstring_re refers to, for re.sub(); a list is joined with newlines. + An escaped reference yields the reference without the backslash, and + a lone dollar sign stays as is. Raises an ExecuteError if the name is + not a valid variable name, or if the variable is not defined. + """ field_start = match.start(0) field = match.group(0) escape = match.group(1) @@ -160,19 +215,32 @@ def variable_sub_cb(self, match): # Tokens that include variables in a string may use this callback to # make sure that the variable is already declared. - def variable_test_cb(self, match): + def variable_test_cb(self, match: re.Match[str]) -> str: + """ + Checks the variable that the given match refers to, as + variable_sub_cb() does, and returns the match unchanged, for + re.sub(). + """ self.variable_sub_cb(match) return match.group(0) # Tokens that include variables in a string may use this callback to mask # the sigil ($, \$) with underscores while keeping the variable name intact. - def variable_sigil_sub_cb(self, match): + def variable_sigil_sub_cb(self, match: re.Match[str]) -> str: + """ + Returns the given match of varstring_re with the sigil replaced by + underscores of the same length, for re.sub(). + """ # Replace only the sigil ($, \$) with underscores; keep the variable name. return '_' * (len(match.group(0)) - len(match.group(2))) + match.group(2) # Tokens that need to make any string invisible to the lexer, like comments, # while keeping its length, may use this callback. - def invisible_sub_cb(self, match, char=' '): + def invisible_sub_cb(self, match: re.Match[str], char: str = ' ') -> str: + """ + Returns the given match with all characters but the line breaks + replaced by the given character, for re.sub(). + """ lines = match.group(0).splitlines(keepends=True) for i, line in enumerate(lines): length = len(line) @@ -185,8 +253,20 @@ def invisible_sub_cb(self, match, char=' '): lines[i] = char * length return ''.join(lines) - def value(self, context): + # Any: the subclasses Regex and Execute return a compiled pattern and 1. + def value(self, context: Scope) -> Any: + """ + Returns the string, with the variables replaced by their values, as a + list with one item. + + :param context: The scope in which the template runs. + """ return [varstring_re.sub(self.variable_sub_cb, self.string)] - def dump(self, indent=0): + def dump(self, indent: int = 0) -> None: + """ + Prints the string to stdout. + + :param indent: The indentation of this token. + """ print((' ' * indent) + self.name, repr(self.string)) diff --git a/src/Exscript/interpreter/template.py b/src/Exscript/interpreter/template.py index 81be3c13..4cd391dc 100644 --- a/src/Exscript/interpreter/template.py +++ b/src/Exscript/interpreter/template.py @@ -20,14 +20,23 @@ # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" +The text of a template, outside of code blocks. +""" + import ast import re +from typing import TYPE_CHECKING, Any, cast +from ..parselib import Lexer from .code import Code from .execute import Execute from .rpc import RPCBody from .scope import Scope +if TYPE_CHECKING: + from .parser import Parser + escaped_grammar = [ # Unicode range: 0x00000000 to 0x0010FFFF ('escaped_unicode_32bit', r'\\U[0-9a-fA-F]{8}'), @@ -56,7 +65,24 @@ class Template(Scope): - def __init__(self, lexer, parser, parent, *args, **kwargs): + """ + The text of a template outside of code blocks. Each line of text is a + command (an Execute token) that is sent to the remote host. Code blocks + in curly brackets and elements become child tokens as well. + """ + + def __init__( + self, lexer: Lexer, parser: 'Parser', parent: Scope, *args: Any, **kwargs: Any + ) -> None: + """ + Constructor. Parses the text up to the end of the input or, inside a + code block, up to and including the next opening curly bracket. + + :param lexer: The lexer that reads the template. + :param parser: The parser of the template. + :param parent: The parent scope. + :param kwargs: Passed to Scope. + """ Scope.__init__(self, self.__class__.__name__, lexer, parser, parent, **kwargs) lexer.set_grammar(grammar_c) buffer = '' @@ -115,7 +141,17 @@ def __init__(self, lexer, parser, parent, *args, **kwargs): # End of statement. self.mark_end(rstrip=True) - def eval_escape(self, ttype, token): + def eval_escape(self, ttype: str, token: str) -> str: + """ + Returns the text that the given escape sequence stands for. Unicode, + hexadecimal and octal sequences and control characters are evaluated + unless the parser disables escaping. An escaped dollar sign stays + escaped for the Execute token; any other escaped character is + returned without the backslash. + + :param ttype: The token type of the escape sequence. + :param token: The escape sequence. + """ # Supporting various types of escape sequences. The easiest way is to # evaluate them as strings with the builtin ast module. if not self.parser.no_escaping and ttype in ( @@ -128,7 +164,7 @@ def eval_escape(self, ttype, token): ): if token == '\\e': # Invalid sequence in Python. return '\x1b' - return ast.literal_eval('"' + token + '"') + return cast(str, ast.literal_eval('"' + token + '"')) # Simple escape sequence with one character. if len(token) == 2: char = token[1] @@ -137,5 +173,11 @@ def eval_escape(self, ttype, token): # Simply unescape anything else. return token[1:] - def execute(self): + # Any: the value of the last child (see Scope.value()). + def execute(self) -> Any: + """ + Evaluates the template, with the template itself as the context. + + :return: The value of the last child, or 1 if there are no children. + """ return self.value(self) diff --git a/src/Exscript/interpreter/term.py b/src/Exscript/interpreter/term.py index 2e33ffef..52d582c9 100644 --- a/src/Exscript/interpreter/term.py +++ b/src/Exscript/interpreter/term.py @@ -20,18 +20,42 @@ # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -from ..parselib import Token +""" +The terms of expressions in the template language. +""" + +from typing import TYPE_CHECKING, Any, cast + +from ..parselib import Lexer, Token from .functioncall import FunctionCall from .number import Number from .regex import Regex +from .scope import Scope from .string import String from .variable import Variable +if TYPE_CHECKING: + from .parser import Parser + class Term(Token): - def __init__(self, lexer, parser, parent): + """ + An operand of an expression: a variable, a function call, a string, a + regular expression, or a number (also as hexadecimal or octal number, + or as the keywords true and false). + """ + + def __init__(self, lexer: Lexer, parser: 'Parser', parent: Scope) -> None: + """ + Constructor. Parses the term. Raises a NameError if a variable is + not defined. + + :param lexer: The lexer that reads the template. + :param parser: The parser of the template. + :param parent: The scope that contains the expression. + """ Token.__init__(self, self.__class__.__name__, lexer, parser, parent) - self.term = None + self.term: Variable | FunctionCall | String | Number | None = None self.lft = None self.rgt = None self.op = None @@ -68,12 +92,28 @@ def __init__(self, lexer, parser, parent): # End of statement. self.mark_end(rstrip=True) - def priority(self): + def priority(self) -> int: + """ + Returns 6, the priority that Expression.prioritize() assumes for a + term. + """ return 6 - def value(self, context): - return self.term.value(context) + # Any: the value of a variable or the result of a function. + def value(self, context: Scope) -> Any: + """ + Returns the value of the term. + + :param context: The scope in which the template runs. + """ + # The constructor sets the term or raises an error. + return cast('Variable | FunctionCall | String | Number', self.term).value(context) + + def dump(self, indent: int = 0) -> None: + """ + Prints the term to stdout. - def dump(self, indent=0): + :param indent: The indentation of this token. + """ print((' ' * indent) + self.name) - self.term.dump(indent + 1) + cast('Variable | FunctionCall | String | Number', self.term).dump(indent + 1) diff --git a/src/Exscript/interpreter/trying.py b/src/Exscript/interpreter/trying.py index 3af15965..00354ccd 100644 --- a/src/Exscript/interpreter/trying.py +++ b/src/Exscript/interpreter/trying.py @@ -20,28 +20,62 @@ # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" +The try statement of the template language. +""" + +from typing import TYPE_CHECKING + import Exscript.interpreter.code +from ..parselib import Lexer from ..protocols.exception import ProtocolException from .scope import Scope +if TYPE_CHECKING: + from .parser import Parser + class Try(Scope): - def __init__(self, lexer, parser, parent): + """ + The try statement, which ignores protocol errors (ProtocolException) + in its block. + """ + + def __init__(self, lexer: Lexer, parser: 'Parser', parent: Scope) -> None: + """ + Constructor. Parses the statement up to and including its end + keyword. + + :param lexer: The lexer that reads the template. + :param parser: The parser of the template. + :param parent: The scope that contains the statement. + """ Scope.__init__(self, self.__class__.__name__, lexer, parser, parent) lexer.expect(self, 'keyword', 'try') self.block = Exscript.interpreter.code.Code(lexer, parser, parent) # End of statement. self.mark_end(rstrip=True) - def value(self, context): + def value(self, context: Scope) -> int: + """ + Evaluates the block, and stops it at the first ProtocolException. + + :param context: The scope in which the template runs. + :return: 1. + """ try: self.block.value(context) except ProtocolException: return 1 return 1 - def dump(self, indent=0): + def dump(self, indent: int = 0) -> None: + """ + Prints the statement and its block to stdout. + + :param indent: The indentation of this token. + """ print((' ' * indent) + self.name, 'start') self.block.dump(indent + 1) print((' ' * indent) + self.name, 'end') diff --git a/src/Exscript/interpreter/variable.py b/src/Exscript/interpreter/variable.py index d6444ab5..b5f5978c 100644 --- a/src/Exscript/interpreter/variable.py +++ b/src/Exscript/interpreter/variable.py @@ -20,23 +20,56 @@ # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -from ..parselib import Token +""" +Variables in expressions of the template language. +""" + +from typing import TYPE_CHECKING, Any + +from ..parselib import Lexer, Token +from .scope import Scope + +if TYPE_CHECKING: + from .parser import Parser class Variable(Token): - def __init__(self, lexer, parser, parent): + """ + A reference to a variable in an expression. + """ + + def __init__(self, lexer: Lexer, parser: 'Parser', parent: Scope) -> None: + """ + Constructor. Parses the name of the variable. + + :param lexer: The lexer that reads the template. + :param parser: The parser of the template. + :param parent: The scope that contains the expression. + """ Token.__init__(self, self.__class__.__name__, lexer, parser, parent) self.varname = lexer.token()[1] lexer.expect(self, 'varname') # End of statement. self.mark_end(rstrip=True) - def value(self, context): + # Any: see Scope.get(). + def value(self, context: Scope) -> Any: + """ + Returns the value of the variable. Raises an ExecuteError if the + variable is not defined. + + :param context: The scope in which the template runs. + """ val = self.parent.get(self.varname) if val is None: msg = 'Undefined variable %s' % repr(self.varname) self.lexer.runtime_error(msg, self) return val - def dump(self, indent=0): + def dump(self, indent: int = 0) -> None: + """ + Prints the name of the variable to stdout. + + :param indent: The indentation of this token. + """ print((' ' * indent) + self.name, self.varname) diff --git a/src/Exscript/parselib/__init__.py b/src/Exscript/parselib/__init__.py index 59fed4b5..4a8a16f4 100644 --- a/src/Exscript/parselib/__init__.py +++ b/src/Exscript/parselib/__init__.py @@ -20,6 +20,12 @@ # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" +A small library for recursive descent parsers: a lexer that splits the +input into tokens with regular expressions, and a base class for the +tokens of the parse tree. The template interpreter is built on it. +""" + from .lexer import Lexer from .token import Token diff --git a/src/Exscript/parselib/exception.py b/src/Exscript/parselib/exception.py index 4357a79a..7a68c228 100644 --- a/src/Exscript/parselib/exception.py +++ b/src/Exscript/parselib/exception.py @@ -20,11 +20,15 @@ # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" +Exceptions and warnings of the lexer. +""" class LexerException(Exception): """ - Fallback exception that is called when the error type is not known. + Fallback exception that is raised when the error type is not known. It + is the base class of CompileError and ExecuteError. """ pass @@ -49,8 +53,9 @@ class CompileError(LexerException): class CompileWarning(LexerWarning): """ - Raised during the compilation procedure if the template contained - a syntax error, that does not prevent compilation but should be noted. + Emitted as a warning during the compilation procedure if the template + contained a syntax error, that does not prevent compilation but should + be noted. """ pass diff --git a/src/Exscript/parselib/lexer.py b/src/Exscript/parselib/lexer.py index 04616dbd..e95be81a 100644 --- a/src/Exscript/parselib/lexer.py +++ b/src/Exscript/parselib/lexer.py @@ -20,38 +20,78 @@ # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" +A lexer that splits its input into tokens with regular expressions. +""" + import re import warnings +from collections.abc import Callable, Sequence from types import SimpleNamespace +from typing import TYPE_CHECKING, Any, cast from .exception import CompileError, CompileWarning, ExecuteError, LexerException +if TYPE_CHECKING: + from .token import Token + class Lexer: - def __init__(self, parser_cls, *args, **kwargs): + """ + Splits its input into tokens with a stack of grammars, and reports + errors with their position in the input. A grammar is a sequence of + (token type, compiled regular expression) pairs: the first pair whose + expression matches at the current position defines the current token. + The parser class that is passed to the constructor reads the tokens and + builds the parse tree. + """ + + def __init__(self, parser_cls: Callable[..., Any], *args: Any, **kwargs: Any) -> None: """ - The given args are passed to the parser_cls constructor. + Constructor. The given args are passed to the parser_cls constructor. + + :param parser_cls: The class that parse() calls with this lexer and + the given args; it returns the parse result. + :param kwargs: ``debug``: with a value of 5 or more, parse() dumps + the parse result to stdout. """ self.parser_cls = parser_cls self.parser_cls_args = args - self.filename = None + self.filename: str | None = None self.input = '' self.input_length = 0 self.current_char = 0 self.last_char = 0 - self.token_buffer = None - self.grammar = [] - self.debug = kwargs.get('debug', False) + self.token_buffer: tuple[str, str] | None = None + self.grammar: list[Sequence[tuple[str, re.Pattern[str]]]] = [] + self.debug: int = kwargs.get('debug', False) + + def set_grammar(self, grammar: Sequence[tuple[str, re.Pattern[str]]]) -> None: + """ + Pushes the given grammar onto the stack of grammars, so that it + becomes the current grammar. - def set_grammar(self, grammar): + :param grammar: (token type, compiled regular expression) pairs, in + the order in which they are tried. + """ self.grammar.append(grammar) self.token_buffer = None - def restore_grammar(self): + def restore_grammar(self) -> None: + """ + Removes the current grammar from the stack of grammars, so that the + previous grammar becomes current again. + """ self.grammar.pop() self.token_buffer = None - def match(self): + def match(self) -> None: + """ + Finds the token at the current position with the current grammar and + stores its type and text in the token buffer. At the end of the + input, the token is ('EOF', ''). Raises a CompileError if no + expression of the grammar matches. + """ if self.current_char >= self.input_length: self.token_buffer = ('EOF', '') return @@ -67,31 +107,62 @@ def match(self): # Match 1..50 chars (eager) up to the last space, newline or EOF to # prevent cutoffs in the middle of a word/line. regex = re.compile(r'.{1,50}(?=[ \t\r\n]|$)', flags=re.S) - context = regex.match(self.input, sender.start).group(0) + # The cast assumes a match. The pattern fails without a blank or the end + # of the input within 50 characters; the grammars of the interpreter fail + # only at a backslash before a line break or the end of the input. + context = cast(re.Match[str], regex.match(self.input, sender.start)).group(0) error = 'Invalid syntax: %s' % repr(context) self.syntax_error(error, sender) - def _get_line_number_from_char(self, char): + def _get_line_number_from_char(self, char: int) -> int: + """ + Returns the number (counted from 1) of the line that contains the + given position. + """ return self.input[:char].count('\n') + 1 - def get_current_line_number(self): + def get_current_line_number(self) -> int: + """ + Returns the number (counted from 1) of the line at the current + position. + """ return self._get_line_number_from_char(self.current_char) - def _get_line(self, number): + def _get_line(self, number: int) -> str: + """ + Returns the line with the given number (counted from 1), without + the newline. + """ return self.input.split('\n')[number - 1] - def _get_lines(self, first, last): + def _get_lines(self, first: int, last: int) -> list[str]: + """ + Returns the lines from the first to the last given number (counted + from 1, both included), without the newlines. + """ return self.input.split('\n')[first - 1 : last] - def get_current_line(self): + def get_current_line(self) -> str: + """ + Returns the line at the current position, without the newline. + """ line = self.get_current_line_number() return self._get_line(line) - def _get_line_from_char(self, char): + def _get_line_from_char(self, char: int) -> str: + """ + Returns the line that contains the given position, without the + newline. + """ line = self._get_line_number_from_char(char) return self._get_line(line) - def _get_line_position_from_char(self, char): + def _get_line_position_from_char(self, char: int) -> tuple[int, int]: + """ + Returns the position of the first character of the line that + contains the given position, and the position of the newline that + ends the line (or the length of the input). + """ line_start = char while line_start != 0: if self.input[line_start - 1] == '\n': @@ -102,7 +173,16 @@ def _get_line_position_from_char(self, char): line_end = self.input_length return line_start, line_end - def _error(self, exc_cls, error, sender=None): + def _error( + self, exc_cls: type[Exception], error: str, sender: 'Token | SimpleNamespace | None' = None + ) -> None: + """ + Raises an exception of the given class with the given message, or + emits it as a warning if the class is a Warning. If a sender is given, + the message also shows the lines of the input that the sender marks, + with the marked range underlined, and the file name and line + number(s). + """ if not sender: if issubclass(exc_cls, Warning): warnings.warn(exc_cls('\n' + error), stacklevel=3) @@ -133,7 +213,8 @@ def _error(self, exc_cls, error, sender=None): for ln in lines: if ln.strip() == '': continue - offset = min(offset, len(whitespace_re.match(ln).group(0))) + # The pattern also matches an empty string, so it always matches. + offset = min(offset, len(cast(re.Match[str], whitespace_re.match(ln)).group(0))) token_length = max(token_length, len(ln.rstrip())) token_length -= offset @@ -149,46 +230,111 @@ def _error(self, exc_cls, error, sender=None): return raise exc_cls('\n' + output) - def error(self, error, sender=None, exc_cls=LexerException): + def error( + self, + error: str, + sender: 'Token | SimpleNamespace | None' = None, + exc_cls: type[Exception] = LexerException, + ) -> None: + """ + Raises an exception of the given class with the given message, or + emits it as a warning if the class is a Warning. The message shows + the position of the sender, as described in syntax_error(). + + :param error: The error message. + :param sender: The token that caused the error. + :param exc_cls: The exception class. + """ self._error(exc_cls, error, sender) - def syntax_error(self, error, sender=None): + def syntax_error(self, error: str, sender: 'Token | SimpleNamespace | None' = None) -> None: + """ + Raises a CompileError with the given message. If a sender is given, + the message also shows the lines of the input that the sender marks, + with the marked range underlined, and the file name and line + number(s). + + :param error: The error message. + :param sender: The token that caused the error. + """ self._error(CompileError, error, sender) - def syntax_warning(self, warning, sender=None): + def syntax_warning(self, warning: str, sender: 'Token | SimpleNamespace | None' = None) -> None: + """ + Emits a CompileWarning with the given message, which shows the + position of the sender as described in syntax_error(). + + :param warning: The warning message. + :param sender: The token that caused the warning. + """ self._error(CompileWarning, warning, sender) - def runtime_error(self, error, sender=None): + def runtime_error(self, error: str, sender: 'Token | SimpleNamespace | None' = None) -> None: + """ + Raises an ExecuteError with the given message, which shows the + position of the sender as described in syntax_error(). + + :param error: The error message. + :param sender: The token that caused the error. + """ self._error(ExecuteError, error, sender) - def forward(self, chars=1): + def forward(self, chars: int = 1) -> None: + """ + Moves the current position forward by the given number of + characters, and remembers the previous position in last_char. + """ self.last_char = self.current_char self.current_char += chars self.token_buffer = None - def __next__(self): + def __next__(self) -> None: if self.token_buffer: if self.token_buffer[0] == 'EOF': msg = "Unexpected EOF while iterating tokens." raise StopIteration(msg) self.forward(len(self.token_buffer[1])) - def next_if(self, types, token=None): + def next_if(self, types: str | list[str], token: str | None = None) -> int: + """ + Moves past the current token if it has one of the given types and, + if a token text is given, that text. + + :param types: A token type, or a list of token types. + :param token: The text that the token must have. + :return: 1 if the lexer moved past the token, 0 otherwise. + """ if type(types) is not type([]): - types = [types] + types = [cast(str, types)] if self.current_is(types, token): next(self) return 1 return 0 - def skip(self, types, token=None): + def skip(self, types: str | list[str], token: str | None = None) -> None: + """ + Moves past consecutive tokens that have one of the given types and, + if a token text is given, that text. + + :param types: A token type, or a list of token types. + :param token: The text that the tokens must have. + """ while self.next_if(types, token): pass - def expect(self, sender, types, token=None): + def expect(self, sender: 'Token', types: str | list[str], token: str | None = None) -> None: + """ + Moves past the current token if it has one of the given types and, + if a token text is given, that text. Otherwise, moves the marks of + the sender to the current token and raises a CompileError. + + :param sender: The token that expects the token. + :param types: A token type, or a list of token types. + :param token: The text that the token must have. + """ cur_type, cur_token = self.token() if type(types) is not type([]): - types = [types] + types = [cast(str, types)] if self.next_if(types, token): return if token: @@ -206,23 +352,46 @@ def expect(self, sender, types, token=None): sender.end = self.current_char + len(cur_token) self.syntax_error(error, sender) - def current_is(self, types, token=None): + def current_is(self, types: str | list[str], token: str | None = None) -> int: + """ + Returns whether the current token has one of the given types and, if + a token text is given, that text. + + :param types: A token type, or a list of token types. + :param token: The text that the token must have. + :return: 1 if the token matches, 0 otherwise. + """ if self.token_buffer is None: self.match() if type(types) is not type([]): - types = [types] - if self.token_buffer[0] not in types: + types = [cast(str, types)] + # match() fills the token buffer or raises an error. + if cast(tuple[str, str], self.token_buffer)[0] not in types: return 0 - if token is None or self.token_buffer[1] == token: + if token is None or cast(tuple[str, str], self.token_buffer)[1] == token: return 1 return 0 - def token(self): + def token(self) -> tuple[str, str]: + """ + Returns the type and the text of the current token. + """ if self.token_buffer is None: self.match() - return self.token_buffer + # match() fills the token buffer or raises an error. + return cast(tuple[str, str], self.token_buffer) + + # The parse result is Any: the parser class defines it. + def parse(self, string: str, filename: str | None = None) -> Any: + """ + Resets the lexer to the given input and parses it: calls the parser + class with this lexer and the args that were passed to the + constructor. - def parse(self, string, filename=None): + :param string: The input. + :param filename: The file name that error messages show. + :return: The object that the parser class returns. + """ # Re-initialize, so that the same lexer instance may be used multiple # times. self.filename = filename @@ -239,6 +408,12 @@ def parse(self, string, filename=None): compiled.dump() return compiled - def parse_file(self, filename): + def parse_file(self, filename: str) -> Any: + """ + Like parse(), but reads the input from the given UTF-8 file. + + :param filename: The name of the file. + :return: The object that the parser class returns. + """ with open(filename, encoding='utf-8') as fp: return self.parse(fp.read(), filename) diff --git a/src/Exscript/parselib/token.py b/src/Exscript/parselib/token.py index ac12d4fd..0be37ded 100644 --- a/src/Exscript/parselib/token.py +++ b/src/Exscript/parselib/token.py @@ -20,6 +20,14 @@ # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" +The base class for the tokens of a parse tree. +""" + +from collections.abc import Callable +from typing import Any + +from .lexer import Lexer class Token: @@ -32,16 +40,22 @@ class Iterator: A tree iterator that walks through all tokens. """ - def __init__(self, current): + def __init__(self, current: 'Token') -> None: """ Constructor. + + :param current: The token at which the iteration starts. """ self.path = [current] - def __iter__(self): + def __iter__(self) -> 'Token.Iterator': return self - def _next(self): + def _next(self) -> 'Token': + """ + Returns the current token and moves on to the next token in + depth-first order. Raises StopIteration at the end of the tree. + """ # Make sure that the end is not yet reached. if len(self.path) == 0: raise StopIteration() @@ -71,27 +85,53 @@ def _next(self): break return current - def __next__(self): + def __next__(self) -> 'Token': # By using this loop we avoid an (expensive) recursive call. while True: next = self._next() if next is not None: return next - def __init__(self, name, lexer, parser, parent=None): + # The parser and the parent are Any: the application defines them, and + # parselib only stores them. The interpreter passes its Parser and a Scope. + def __init__(self, name: str, lexer: Lexer, parser: Any, parent: Any = None) -> None: + """ + Constructor. The marks of the new token span the character at the + current position of the lexer. + + :param name: The name of the token, shown by dump(). + :param lexer: The lexer that reads the input. + :param parser: The parser that builds the parse tree. + :param parent: The parent of the token, or None. + """ self.lexer = lexer self.parser = parser self.parent = parent self.name = name - self.children = [] + self.children: list[Token] = [] self.start = lexer.current_char self.end = lexer.current_char + 1 - def value(self, context): + # The context and the value are Any: the application defines them. The + # value of a token depends on its class. + def value(self, context: Any) -> Any: + """ + Evaluates the children of the token in the given context. Subclasses + return the value of the token. + + :param context: The context in which the token is evaluated. + """ for child in self.get_children(): child.value(context) - def mark_start(self, char=None): + def mark_start(self, char: int | None = None) -> None: + """ + Moves the start mark of the token to the given position, or to the + current position of the lexer. If the end mark is not behind the new + start mark, the end mark is moved one character behind it. + + :param char: The position of the first character of the token. + """ if char is None: char = self.lexer.current_char # EOF will point behind the last character index (length of input)! @@ -99,7 +139,20 @@ def mark_start(self, char=None): if self.start >= self.end: self.mark_end(distance=1) - def mark_end(self, char=None, distance=None, rstrip=False): + def mark_end( + self, char: int | None = None, distance: int | None = None, rstrip: bool | str = False + ) -> None: + """ + Moves the end mark of the token to the given position, to the given + distance from the start mark, or to the current position of the + lexer. Raises ValueError if both char and distance are given. + + :param char: The position behind the last character of the token. + :param distance: The length of the token. + :param rstrip: Whether to move the end mark back over trailing + whitespace. A string moves it back over the characters in it + instead. + """ if char is not None and distance is not None: msg = 'must not specify both arguments "char" and "distance"' raise ValueError(msg) @@ -112,7 +165,7 @@ def mark_end(self, char=None, distance=None, rstrip=False): # at the end of a Token init, that finally skipped trailing whitespaces. if rstrip: if isinstance(rstrip, str): - isspace = lambda c: c in rstrip # noqa: E731 + isspace: Callable[[str], bool] = lambda c: c in rstrip # noqa: E731 else: isspace = lambda c: c.isspace() # noqa: E731 while char > 0 and isspace(self.lexer.input[char - 1 : char]): @@ -120,19 +173,31 @@ def mark_end(self, char=None, distance=None, rstrip=False): # EOF will point behind the last character index (length of input)! self.end = max(self.start, min(char, self.lexer.input_length)) - def __iter__(self): + def __iter__(self) -> 'Token.Iterator': """ Returns an iterator that points to the first token. """ return Token.Iterator(self) - def add(self, child): + def add(self, child: 'Token') -> None: + """ + Appends the given token to the children of this token. + """ self.children.append(child) - def get_children(self): + def get_children(self) -> list['Token']: + """ + Returns the children of this token. + """ return self.children - def dump(self, indent=0): + def dump(self, indent: int = 0) -> None: + """ + Prints the name of the token to stdout, followed by a dump of each + child with one more level of indentation. + + :param indent: The indentation of this token. + """ print((' ' * indent) + self.name) for child in self.get_children(): child.dump(indent + 1) From 915c94ddff579c7cfdd41fe430c5613a6aefbcc2 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 4 Nov 2026 14:01:08 +0100 Subject: [PATCH 062/250] Check the annotated modules strictly The 49 annotated modules leave the relaxed mypy override, and pre-commit fails below 72 percent docstring coverage instead of 49. --- pyproject.toml | 51 +------------------------------------------------- 1 file changed, 1 insertion(+), 50 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 6d29c148..25b791a3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -131,40 +131,11 @@ ignore_missing_imports = true # ncclient and others ship no type hints # Modules that are not annotated yet (P8). Remove a module from this list when # it passes the strict checks; the list only shrinks. module = [ - "Exscript.account", "Exscript.contrib.otel", "Exscript.emulators.command", "Exscript.emulators.iosemu", "Exscript.emulators.vdevice", - "Exscript.host", - "Exscript.interpreter.append", - "Exscript.interpreter.assign", - "Exscript.interpreter.code", - "Exscript.interpreter.enter", - "Exscript.interpreter.execute", - "Exscript.interpreter.expression", - "Exscript.interpreter.extract", - "Exscript.interpreter.fail", - "Exscript.interpreter.functioncall", - "Exscript.interpreter.ifcondition", - "Exscript.interpreter.loop", - "Exscript.interpreter.number", - "Exscript.interpreter.parser", - "Exscript.interpreter.program", - "Exscript.interpreter.regex", - "Exscript.interpreter.rpc", - "Exscript.interpreter.scope", - "Exscript.interpreter.string", - "Exscript.interpreter.template", - "Exscript.interpreter.term", - "Exscript.interpreter.trying", - "Exscript.interpreter.variable", - "Exscript.key", - "Exscript.logger", - "Exscript.parselib.lexer", - "Exscript.parselib.token", "Exscript.protocols", - "Exscript.protocols.drivers", "Exscript.protocols.drivers.ace", "Exscript.protocols.drivers.adtran", "Exscript.protocols.drivers.aironet", @@ -174,7 +145,6 @@ module = [ "Exscript.protocols.drivers.bigip", "Exscript.protocols.drivers.brocade", "Exscript.protocols.drivers.cienasaos", - "Exscript.protocols.drivers.driver", "Exscript.protocols.drivers.enterasys", "Exscript.protocols.drivers.enterasys_wc", "Exscript.protocols.drivers.eos", @@ -201,10 +171,7 @@ module = [ "Exscript.protocols.drivers.zte", "Exscript.protocols.drivers.zyxel", "Exscript.protocols.dummy", - "Exscript.protocols.exception", "Exscript.protocols.netconf", - "Exscript.protocols.osguesser", - "Exscript.protocols.protocol", "Exscript.protocols.ssh2", "Exscript.protocols.telnet", "Exscript.protocols.telnetlib", @@ -221,30 +188,14 @@ module = [ "Exscript.stdlib.mysys", "Exscript.stdlib.string", "Exscript.stdlib.util", - "Exscript.util.buffer", - "Exscript.util.cast", - "Exscript.util.collections", - "Exscript.util.crypt", - "Exscript.util.decorator", - "Exscript.util.event", "Exscript.util.file", - "Exscript.util.impl", "Exscript.util.interact", - "Exscript.util.ip", - "Exscript.util.ipv4", - "Exscript.util.ipv6", "Exscript.util.log", "Exscript.util.mail", - "Exscript.util.match", "Exscript.util.report", "Exscript.util.sigint", "Exscript.util.start", - "Exscript.util.syslog", "Exscript.util.template", - "Exscript.util.tty", - "Exscript.util.url", - "Exscript.util.weakmethod", - "Exscript.util.xml", "Exscript.workqueue.job", "Exscript.workqueue.mainloop", "Exscript.workqueue.pipeline", @@ -267,7 +218,7 @@ extra_checks = false # Docstring coverage (P8). Private functions count; other dunder methods than # __init__, nested functions and property setters do not. Raise fail-under as # docstrings are added; the goal is 100. -fail-under = 49 +fail-under = 72 ignore-magic = true ignore-nested-functions = true ignore-setters = true From f9f6d7e9d1cb09b34c881a896c1ab790b0c746ca Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 4 Nov 2026 14:24:19 +0100 Subject: [PATCH 063/250] Test the exscript command line tool The idea follows pull request #246 by pattems. --- tests/Exscript/ScriptsTest.py | 64 +++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 tests/Exscript/ScriptsTest.py diff --git a/tests/Exscript/ScriptsTest.py b/tests/Exscript/ScriptsTest.py new file mode 100644 index 00000000..44a8ec7b --- /dev/null +++ b/tests/Exscript/ScriptsTest.py @@ -0,0 +1,64 @@ +import os +import subprocess +import sys +import tempfile +import unittest + +BASEDIR = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..')) +SCRIPT = os.path.join(BASEDIR, 'scripts', 'exscript') + + +class ScriptsTest(unittest.TestCase): + """Runs the exscript command line tool like a user.""" + + def setUp(self): + self.tmpdir = tempfile.TemporaryDirectory() + self.addCleanup(self.tmpdir.cleanup) + + def run_exscript(self, *args): + env = dict(os.environ, HOME=self.tmpdir.name, USERPROFILE=self.tmpdir.name) + proc = subprocess.run( # noqa: S603 - runs this repository's script with fixed arguments + [sys.executable, SCRIPT, *args], + capture_output=True, + encoding='utf-8', + env=env, + timeout=60, + ) + return proc.returncode, proc.stdout + proc.stderr + + def write_template(self, text): + path = os.path.join(self.tmpdir.name, 'test.exscript') + with open(path, 'w', encoding='utf-8') as fp: + fp.write(text) + return path + + def testHelp(self): + code, output = self.run_exscript('--help') + self.assertEqual(code, 0, output) + for option in '--load-driver', '--insecure', '--ssh-host-key-policy': + self.assertIn(option, output) + + def testRunTemplate(self): + template = self.write_template('{sys.message("hello")}\n') + args = '--non-interactive', '--no-authentication', template, 'dummy://r1' + code, output = self.run_exscript(*args) + self.assertEqual(code, 0, output) + self.assertIn('hello', output) + + def testSecureModeByDefault(self): + template = self.write_template('{sys.env("PATH")}\n') + args = '--non-interactive', '--no-authentication', template, 'dummy://r1' + code, output = self.run_exscript(*args) + self.assertEqual(code, 2, output) + self.assertIn("insecure function 'sys.env' is not permitted", output) + + code, output = self.run_exscript('--insecure', *args) + self.assertEqual(code, 0, output) + + +def suite(): + return unittest.TestLoader().loadTestsFromTestCase(ScriptsTest) + + +if __name__ == '__main__': + unittest.TextTestRunner(verbosity=2).run(suite()) From 9eda3ecc8a9986f4761b35f33c92b03a87480944 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 10 Nov 2026 15:31:17 +0100 Subject: [PATCH 064/250] Fix the login error pattern of the ericsson_ban driver It was a string, but the protocol calls search() on each pattern, so a login over this driver failed with AttributeError instead of reporting the login error. --- CHANGES.md | 4 ++++ .../protocols/drivers/ericsson_ban.py | 2 +- tests/Exscript/protocols/OsGuesserTest.py | 21 +++++++++++++++++++ 3 files changed, 26 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index c95e49ec..d5f568df 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -277,6 +277,10 @@ marked with **BREAKING**; a migration guide will accompany the release. local shell) before the connection is closed gracefully, otherwise the logout command may fail. +- **Login Errors of `ericsson_ban`:** A login over the `ericsson_ban` driver no longer + fails with `AttributeError: 'str' object has no attribute 'search'`; its login + error pattern was a string instead of a compiled pattern. A test checks the + patterns of all drivers. - **`disable_driver()`:** `Exscript.protocols.drivers.disable_driver()` works; it raised a `TypeError` after it had removed the driver only from `driver_map`, so the OS guesser still chose it. diff --git a/src/Exscript/protocols/drivers/ericsson_ban.py b/src/Exscript/protocols/drivers/ericsson_ban.py index 7cc2d016..4222c42f 100644 --- a/src/Exscript/protocols/drivers/ericsson_ban.py +++ b/src/Exscript/protocols/drivers/ericsson_ban.py @@ -37,7 +37,7 @@ _ban_re = re.compile(r"BLM\d+ - Broadband Loop Multiplexer", re.I) -_login_fail_re = [r'Login failed'] +_login_fail_re = [re.compile(r'Login failed')] class EricssonBanDriver(Driver): diff --git a/tests/Exscript/protocols/OsGuesserTest.py b/tests/Exscript/protocols/OsGuesserTest.py index 25f4339e..a8fa42e5 100644 --- a/tests/Exscript/protocols/OsGuesserTest.py +++ b/tests/Exscript/protocols/OsGuesserTest.py @@ -6,6 +6,7 @@ from Exscript.emulators import VirtualDevice from Exscript.protocols import Dummy, drivers from Exscript.protocols.drivers import isam +from Exscript.protocols.exception import LoginFailure from Exscript.protocols.osguesser import OsGuesser @@ -210,6 +211,26 @@ def testDisableDriver(self): self.assertEqual(self.login().guess_os(), 'unknown') self.assertRaises(TypeError, Dummy().set_driver, 'acme') + def testPatternsAreCompiled(self): + # The protocol calls search() on each pattern; a string would fail + # with an AttributeError during the login. + attributes = 'user_re', 'password_re', 'prompt_re', 'error_re', 'login_error_re' + for name, driver in drivers.driver_map.items(): + for attribute in attributes: + for pattern in getattr(driver, attribute): + self.assertIsInstance(pattern, re.Pattern, f'{name}.{attribute}') + + def testLoginFailureOverEricssonBan(self): + device = VirtualDevice('ban1', login_type=VirtualDevice.LOGIN_TYPE_NONE, strict=False) + device.set_prompt('Login failed\n') + conn = Dummy(device=device, driver='ericsson_ban', timeout=1) + conn.connect('ban1') + self.addCleanup(conn.close, True) + conn.send('admin\r') + self.assertRaises( + LoginFailure, conn.app_authenticate, Account('admin', 'x', needs_lock=False) + ) + def suite(): loader = unittest.TestLoader() From 63a4bac340171df7962bfe7bde1f94cbb1693c92 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 10 Nov 2026 15:31:17 +0100 Subject: [PATCH 065/250] Fix get_filename() checking that the file exists It passed its check function and the history to prompt() one position too early, so it never checked the file, and a history raised a TypeError. It passes them by keyword now. --- CHANGES.md | 4 ++++ src/Exscript/util/interact.py | 2 +- tests/Exscript/util/interactTest.py | 20 ++++++++++++++++++-- 3 files changed, 23 insertions(+), 3 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index d5f568df..daf521f5 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -350,6 +350,10 @@ The names below refer to functions callable from templates, not protocol methods - **`Exscript.util.syslog.netlog()`:** Works again; on Python 3 it always raised a `TypeError`. It also works over IPv6, and the module can be imported on Windows, which has no `syslog` module. +- **`Exscript.util.interact.get_filename()`:** Rejects names of files that do not + exist and asks again, as documented. It passed its check and history one + position too early to `prompt()`, so it never checked the name, and failed with + a `TypeError` when a history was given. - **Pseudo Protocol on Windows:** `pseudo://` URLs accept absolute Windows paths such as `pseudo://C:\devices\router.py`; the drive letter was taken for a port number. - **`Exscript.util.tty.get_terminal_size()`:** Uses `shutil.get_terminal_size()`: the diff --git a/src/Exscript/util/interact.py b/src/Exscript/util/interact.py index fa7d9eba..c8d3a87a 100644 --- a/src/Exscript/util/interact.py +++ b/src/Exscript/util/interact.py @@ -213,7 +213,7 @@ def _validate(string): if not os.path.isfile(string): return 'File not found. Please enter a filename.' - return prompt(key, message, default, True, _validate, history) + return prompt(key, message, default, check=_validate, history=history) def get_user(prompt=None): diff --git a/tests/Exscript/util/interactTest.py b/tests/Exscript/util/interactTest.py index 242ae9de..8b91d67e 100644 --- a/tests/Exscript/util/interactTest.py +++ b/tests/Exscript/util/interactTest.py @@ -1,6 +1,7 @@ import os import tempfile import unittest +import unittest.mock import Exscript.util.interact from Exscript.util.interact import InputHistory @@ -54,8 +55,23 @@ def testPrompt(self): # Can't really be tested, as it is interactive. def testGetFilename(self): - pass - # Can't really be tested, as it is interactive. + # A name of a file that does not exist is rejected, and the prompt repeats. + tmpdir = tempfile.TemporaryDirectory() + self.addCleanup(tmpdir.cleanup) + existing = os.path.join(tmpdir.name, 'hosts.txt') + with open(existing, 'w', encoding='utf-8'): + pass + history = InputHistory(os.path.join(tmpdir.name, 'history')) + missing = os.path.join(tmpdir.name, 'missing.txt') + inputs = iter([missing, ' ' + existing + ' ']) + with ( + unittest.mock.patch('builtins.input', lambda prompt: next(inputs)), + unittest.mock.patch('builtins.print') as output, + ): + filename = Exscript.util.interact.get_filename('hosts', 'Host file', history=history) + self.assertEqual(filename, existing) + output.assert_called_once_with('File not found. Please enter a filename.') + self.assertEqual(history.get('hosts'), existing) def testGetUser(self): pass From e5711915833cec7c223540e8f065057742302df0 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 10 Nov 2026 15:34:01 +0100 Subject: [PATCH 066/250] Add the SSH algorithm and authentication options to the changelog disabled_algorithms, preferred_authentications and their command line options were missing. They answer upstream issues #215 and #172. --- CHANGES.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index daf521f5..ff09481e 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -131,6 +131,14 @@ marked with **BREAKING**; a migration guide will accompany the release. keyboard-interactive took 5 SSH connections, now 3. - **All Addresses for SSH:** SSH and NETCONF connections try all addresses of the host, e.g. IPv4 if IPv6 fails, as Telnet does; before, they tried only the first. +- **SSH Algorithms and Authentication Methods:** The new options + `disabled_algorithms` and `preferred_authentications` (also `Host` options; CLI: + `--ssh-disabled-algs` and `--ssh-preferred-auth`) turn off SSH algorithms that a + device offers but does not handle correctly (passed to paramiko's `Transport`), and + choose the authentication methods and their order, like OpenSSH's + `PreferredAuthentications`. The default order is `publickey`, + `keyboard-interactive`, `password`; methods that the server does not offer are + skipped. - **BREAKING - Terminal Size:** SSH and Telnet sessions get a terminal of 511 columns and 1000 lines; the new options `term_width` and `term_height` (also `Host` options) choose another size. Before, Exscript used the size of the terminal that From 72e1070f8434c3885c9be03e52872586ed56d536 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 10 Nov 2026 15:37:24 +0100 Subject: [PATCH 067/250] Annotate the emulators Only annotations, docstrings and typing.cast() are added. --- src/Exscript/emulators/command.py | 35 ++++++++++++----- src/Exscript/emulators/iosemu.py | 33 +++++++++++----- src/Exscript/emulators/vdevice.py | 62 ++++++++++++++++++++++++------- 3 files changed, 98 insertions(+), 32 deletions(-) diff --git a/src/Exscript/emulators/command.py b/src/Exscript/emulators/command.py index 99262b26..ac6e7837 100644 --- a/src/Exscript/emulators/command.py +++ b/src/Exscript/emulators/command.py @@ -25,29 +25,40 @@ """ import re -from collections.abc import Iterable +from collections.abc import Callable, Iterable +from typing import Any class CommandSet: """ - A set of commands to be used by the Dummy adapter. + A set of commands to be used by a VirtualDevice. """ - def __init__(self, strict=True): + def __init__(self, strict: bool = True) -> None: """ Constructor. + + :param strict: Whether eval() raises an error for a command that + matches no registered command. """ self.strict = strict - self.response_list = [] + self.response_list: list[tuple[re.Pattern[str], str | Callable[[str], str]]] = [] - def __deepcopy__(self, memo): + def __deepcopy__(self, memo: dict[int, Any]) -> 'CommandSet': cmdset = CommandSet(self.strict) for regex, response in self.response_list: regex = re.compile(regex.pattern, regex.flags) cmdset.response_list.append((regex, response)) return cmdset - def add(self, command, handler, handler_decorator=None): + def add( + self, + command: str | re.Pattern[str], + handler: str | Callable[[str], str], + handler_decorator: ( + Callable[[str | Callable[[str], str]], str | Callable[[str], str]] | None + ) = None, + ) -> None: """ Register a command and response handler pair. @@ -73,7 +84,13 @@ def add(self, command, handler, handler_decorator=None): handler = handler_decorator(handler) self.response_list.append((command, handler)) - def add_from_file(self, filename, handler_decorator=None): + def add_from_file( + self, + filename: str, + handler_decorator: ( + Callable[[str | Callable[[str], str]], str | Callable[[str], str]] | None + ) = None, + ) -> None: """ Wrapper around add() that reads the response handlers from the file with the given name. The file is a Python script containing @@ -88,7 +105,7 @@ def add_from_file(self, filename, handler_decorator=None): :raises ValueError: Raised when the file does not define a variable named "commands" or if "commands" is not iterable. """ - args = {} + args: dict[str, Any] = {} with open(filename, encoding="utf-8") as fin: source = fin.read() code = compile(source, filename, mode="exec") @@ -102,7 +119,7 @@ def add_from_file(self, filename, handler_decorator=None): for command, response in commands: self.add(command, response, handler_decorator) - def eval(self, command): + def eval(self, command: str) -> str | None: """ Evaluate the given string against all registered commands and return the defined response. diff --git a/src/Exscript/emulators/iosemu.py b/src/Exscript/emulators/iosemu.py index 39748e6f..aaca1f8e 100644 --- a/src/Exscript/emulators/iosemu.py +++ b/src/Exscript/emulators/iosemu.py @@ -21,10 +21,11 @@ # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ -Cisco IOS emulator used for internal testing. +A Cisco IOS emulator for testing. """ import re +from typing import cast from .vdevice import VirtualDevice @@ -43,8 +44,13 @@ ''' -def show_diag(data): - slot = re.search(r'(\d+)', data).groups()[0] +def show_diag(data: str) -> str: + """ + Returns the response to "show diag": the diagnostics of a line card in + the slot whose number is the first number in the given command. + """ + # The command matched r'show diag \d+', so it contains a number. + slot = cast(re.Match[str], re.search(r'(\d+)', data)).groups()[0] return ( """ SLOT %s (RP/LC 0 ): 16 Port ISE Packet Over SONET OC-3c/STM-1 Single Mode/IR LC connector @@ -173,14 +179,23 @@ def show_diag(data): class IOSEmulator(VirtualDevice): + """ + A virtual device that emulates a Cisco IOS router: its banner, its login + prompts, its prompt, and the responses to a few show commands. + """ + def __init__( self, - hostname, - echo=True, - login_type=VirtualDevice.LOGIN_TYPE_BOTH, - strict=True, - banner=None, - ): + hostname: str, + echo: bool = True, + login_type: int = VirtualDevice.LOGIN_TYPE_BOTH, + strict: bool = True, + banner: str | None = None, + ) -> None: + """ + Constructor. Takes the same arguments as VirtualDevice; the banner + defaults to an IOS login banner that shows the hostname. + """ thebanner = iosbanner % (hostname, hostname, hostname) VirtualDevice.__init__( self, diff --git a/src/Exscript/emulators/vdevice.py b/src/Exscript/emulators/vdevice.py index 3372c826..7ccf5594 100644 --- a/src/Exscript/emulators/vdevice.py +++ b/src/Exscript/emulators/vdevice.py @@ -24,6 +24,9 @@ Defines the behavior of a virtual device, as needed by :class:`Exscript.servers`. """ +import re +from collections.abc import Callable + from . import CommandSet @@ -38,19 +41,29 @@ class VirtualDevice: PROMPT_STAGE_USERNAME, PROMPT_STAGE_PASSWORD, PROMPT_STAGE_CUSTOM = list(range(1, 4)) - def __init__(self, hostname, echo=True, login_type=LOGIN_TYPE_BOTH, strict=True, banner=None): + def __init__( + self, + hostname: str, + echo: bool = True, + login_type: int = LOGIN_TYPE_BOTH, + strict: bool = True, + banner: str | None = None, + ) -> None: """ + Constructor. + :type hostname: str :param hostname: The hostname, used for the prompt. - :param echo: bool + :type echo: bool :keyword echo: whether to echo the command in a response. - :param login_type: int + :type login_type: int :keyword login_type: integer constant, one of LOGIN_TYPE_PASSWORDONLY, LOGIN_TYPE_USERONLY, LOGIN_TYPE_BOTH, LOGIN_TYPE_NONE. - :param strict: bool + :type strict: bool :keyword strict: Whether to raise when a given command has no handler. - :param banner: str + :type banner: str|None :keyword banner: A string to show as soon as the connection is opened. + Defaults to a welcome message with the hostname. """ self.hostname = str(hostname) self.banner = str(banner or 'Welcome to %s!\n' % hostname) @@ -63,10 +76,19 @@ def __init__(self, hostname, echo=True, login_type=LOGIN_TYPE_BOTH, strict=True, self.password_prompt = 'Password: ' self.init() - def _get_prompt(self): + def _get_prompt(self) -> str: + """ + Returns the prompt of the current login stage (user, password or + command prompt), and moves on to the next stage. The input after the + password prompt is not echoed, and the command prompt marks the + device as logged in. + + :raises RuntimeError: Raised when the prompt stage is invalid. + """ if self.prompt_stage == self.PROMPT_STAGE_USERNAME: if self.login_type == self.LOGIN_TYPE_USERONLY: - self.prompt_stage = self.PROMPT_STAGE_CUSTOM + # The first assignment in the source; init() sets the first stage. + self.prompt_stage: int = self.PROMPT_STAGE_CUSTOM else: self.prompt_stage = self.PROMPT_STAGE_PASSWORD return self.user_prompt @@ -80,13 +102,20 @@ def _get_prompt(self): else: raise RuntimeError('invalid prompt stage') - def _create_autoprompt_handler(self, handler): + def _create_autoprompt_handler( + self, handler: str | Callable[[str], str] + ) -> Callable[[str], str]: + """ + Returns a response handler that appends a newline and the prompt + (see _get_prompt()) to the given response, or to the response of + the given response handler. + """ if callable(handler): return lambda x: handler(x) + '\n' + self._get_prompt() else: return lambda x: handler + '\n' + self._get_prompt() - def get_prompt(self): + def get_prompt(self) -> str: """ Returns the prompt of the device. @@ -95,7 +124,7 @@ def get_prompt(self): """ return self.prompt - def set_prompt(self, prompt): + def set_prompt(self, prompt: str) -> None: """ Change the prompt of the device. @@ -104,7 +133,12 @@ def set_prompt(self, prompt): """ self.prompt = prompt - def add_command(self, command, handler, prompt=True): + def add_command( + self, + command: str | re.Pattern[str], + handler: str | Callable[[str], str], + prompt: bool = True, + ) -> None: """ Registers a command. @@ -127,7 +161,7 @@ def add_command(self, command, handler, prompt=True): deco = self._create_autoprompt_handler if prompt else None self.commands.add(command, handler, deco) - def add_commands_from_file(self, filename, prompt=True): + def add_commands_from_file(self, filename: str, prompt: bool = True) -> None: """ Wrapper around add_command that reads the response handlers from the file with the given name. The file is a Python script containing @@ -142,7 +176,7 @@ def add_commands_from_file(self, filename, prompt=True): deco = self._create_autoprompt_handler if prompt else None self.commands.add_from_file(filename, deco) - def init(self): + def init(self) -> str: """ Init or reset the virtual device. @@ -161,7 +195,7 @@ def init(self): return self.banner + self._get_prompt() - def do(self, command): + def do(self, command: str) -> str: """ "Executes" the given command on the virtual device, and returns the response. From 2204ede1d753f460449526f2bfe3b96dfaae60c4 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 10 Nov 2026 15:37:27 +0100 Subject: [PATCH 068/250] Annotate the test servers SSHd takes the paramiko constants from paramiko.common, where the stubs declare them. Otherwise only annotations, docstrings and typing.cast() are added. --- src/Exscript/servers/server.py | 58 +++++++++++++++----- src/Exscript/servers/sshd.py | 97 +++++++++++++++++++++++++++------ src/Exscript/servers/telnetd.py | 25 ++++++++- 3 files changed, 145 insertions(+), 35 deletions(-) diff --git a/src/Exscript/servers/server.py b/src/Exscript/servers/server.py index 6898e189..d4c7cb51 100644 --- a/src/Exscript/servers/server.py +++ b/src/Exscript/servers/server.py @@ -29,6 +29,9 @@ import socket import threading from multiprocessing import Pipe +from typing import cast + +from ..emulators import VirtualDevice _logger = logging.getLogger(__name__) _logger.addHandler(logging.NullHandler()) @@ -51,7 +54,7 @@ class Server(threading.Thread): daemon.join() # Wait until it terminates. """ - def __init__(self, host, port, device, encoding='utf8'): + def __init__(self, host: str, port: int, device: VirtualDevice, encoding: str = 'utf8') -> None: """ Constructor. @@ -73,18 +76,28 @@ def __init__(self, host, port, device, encoding='utf8'): self.timeout = 0.5 self.dbg = 0 self.running = False - self.socket = None + self.socket: socket.socket | None = None # set by start() self.device = device # Shared device does not support concurrent connections! self.encoding = encoding self.to_child, self.to_parent = Pipe() - self.processes = [] + self.processes: list[threading.Thread] = [] - def _dbg(self, level, msg, exc_info=None): + def _dbg(self, level: int, msg: str, exc_info: bool | BaseException | None = None) -> None: + """ + Logs the given message if the debug level (the dbg attribute) is at + least the given level. + """ if level > self.dbg: return self._logger.debug(msg, exc_info=exc_info) - def _poll_child_process(self): + def _poll_child_process(self) -> bool: + """ + Reads a message from the pipe, if one is waiting. The message + "shutdown" from exit(), or an error while reading, stops the server. + + :return: Whether another message was read while the server runs. + """ if not self.to_parent.poll(): return False try: @@ -100,13 +113,27 @@ def _poll_child_process(self): return False return True - def _handle_connection(self, conn): + def _handle_connection(self, conn: socket.socket) -> None: + """ + Talks to the client on the given accepted connection until it + disconnects or the server stops. Subclasses implement it. + + :raises NotImplementedError: Always, in this base class. + """ raise NotImplementedError() - def _get_server_type(self): + def _get_server_type(self) -> str: + """ + Returns the class name in lower case, which names the threads and + appears in the debug messages. + """ return type(self).__name__.lower() - def _run(self): + def _run(self) -> None: + """ + Accepts connections until the server stops, and talks to each client + in a new thread. Then waits for these threads and closes the socket. + """ self._dbg(1, 'Starting the %s server.' % self._get_server_type()) self.running = True @@ -116,11 +143,13 @@ def _run(self): # Purge finished handlers to void zombie accumulation. self.processes = [p for p in self.processes if p.is_alive()] - r, w, x = select.select([self.socket], [], [], self.timeout) + # start() sets the socket before the thread runs; the casts + # are for the type checker. + r, w, x = select.select([cast(socket.socket, self.socket)], [], [], self.timeout) if not r: continue - conn, addr = self.socket.accept() + conn, addr = cast(socket.socket, self.socket).accept() self._dbg(1, 'Accepted connection from: %s:%s' % (addr[0], addr[1])) proc = threading.Thread( target=self._handle_connection, @@ -134,9 +163,9 @@ def _run(self): for proc in self.processes: proc.join() self.processes = [] - self.socket.close() + cast(socket.socket, self.socket).close() - def start(self): + def start(self) -> None: """ Starts listening, and then the thread that accepts the connections. Clients can connect as soon as this returns. With port 0, the @@ -156,19 +185,20 @@ def start(self): self._dbg(1, 'Server bound to: %s:%s' % (self.host, self.port)) threading.Thread.start(self) - def exit(self): + def exit(self) -> None: """ Stop the daemon without waiting for the thread to terminate. """ self._dbg(1, 'Shutdown the %s server.' % self._get_server_type()) self.to_child.send('shutdown') - def exit_command(self, cmd): + def exit_command(self, cmd: str) -> str: """ Like exit(), but may be used as a handler in add_command. :type cmd: str :param cmd: The command that causes the server to exit. + :return: An empty string, the response to the command. """ self.exit() return '' diff --git a/src/Exscript/servers/sshd.py b/src/Exscript/servers/sshd.py index cc5f43ca..adc2a43e 100644 --- a/src/Exscript/servers/sshd.py +++ b/src/Exscript/servers/sshd.py @@ -33,6 +33,8 @@ import paramiko from paramiko import ServerInterface +from ..emulators import VirtualDevice +from ..key import PrivateKey from ..version import __version__ from .server import Server @@ -43,6 +45,11 @@ class _ParamikoServer(ServerInterface): + """ + The paramiko server interface of SSHd. It accepts session channels, any + password, any public key, and any shell and pseudo-terminal request. + """ + # Must match the public key of the test suite! data = ( b'AAAAB3NzaC1yc2EAAAADAQABAAABAQChbnHaPrmxa5NpvtKXSMfMJEklUlinrnaNc' @@ -54,31 +61,61 @@ class _ParamikoServer(ServerInterface): ) good_pub_key = paramiko.RSAKey(data=base64.decodebytes(data)) - def __init__(self): + def __init__(self) -> None: + """ + Constructor. + """ self.event = threading.Event() self._logger = _logger - def check_channel_request(self, kind, chanid): + # The constants come from paramiko.common: the paramiko package has them + # too, but its type stubs do not. + def check_channel_request(self, kind: str, chanid: int) -> int: + """ + Accepts session channels and rejects channels of other kinds. + """ if kind == 'session': - return paramiko.OPEN_SUCCEEDED - return paramiko.OPEN_FAILED_ADMINISTRATIVELY_PROHIBITED - - def check_auth_password(self, username, password): - return paramiko.AUTH_SUCCESSFUL # TODO: paramiko.AUTH_FAILED - - def check_auth_publickey(self, username, key): - return paramiko.AUTH_SUCCESSFUL - - def get_allowed_auths(self, username): + return paramiko.common.OPEN_SUCCEEDED + return paramiko.common.OPEN_FAILED_ADMINISTRATIVELY_PROHIBITED + + def check_auth_password(self, username: str, password: str) -> int: + """ + Accepts any user name and password. + """ + return paramiko.common.AUTH_SUCCESSFUL # TODO: paramiko.AUTH_FAILED + + def check_auth_publickey(self, username: str, key: paramiko.PKey) -> int: + """ + Accepts any user name and public key. + """ + return paramiko.common.AUTH_SUCCESSFUL + + def get_allowed_auths(self, username: str) -> str: + """ + Returns the allowed authentication methods: password and public key. + """ return 'password,publickey' - def check_channel_shell_request(self, channel): + def check_channel_shell_request(self, channel: paramiko.Channel) -> bool: + """ + Accepts the shell request, and sets the event attribute. + """ self.event.set() return True def check_channel_pty_request( - self, channel, term, width, height, pixelwidth, pixelheight, modes - ): + self, + channel: paramiko.Channel, + term: bytes, + width: int, + height: int, + pixelwidth: int, + pixelheight: int, + modes: bytes, + ) -> bool: + """ + Accepts any pseudo-terminal request. + """ return True @@ -97,7 +134,16 @@ class SSHd(Server): :keyword key: An Exscript.PrivateKey object. """ - def __init__(self, host, port, device, key=None): + def __init__( + self, host: str, port: int, device: VirtualDevice, key: PrivateKey | None = None + ) -> None: + """ + Constructor. Takes the arguments of :class:`Server` except the + encoding, which is UTF-8. + + :param key: The host key; the server reads the RSA key from its file. + Defaults to the key in ~/.ssh/id_rsa. + """ Server.__init__(self, host, port, device) self._logger = _logger if key: @@ -106,7 +152,16 @@ def __init__(self, host, port, device, key=None): keyfile = os.path.expanduser('~/.ssh/id_rsa') self.host_key = paramiko.RSAKey(filename=keyfile) - def _recvline(self, channel, buf): + def _recvline(self, channel: paramiko.Channel, buf: bytes) -> tuple[str | None, bytes]: + """ + Reads from the channel until the given buffer, extended by the data + read, contains a complete line. Line ends are normalized to newlines. + + :param buf: The data that was read but not yet returned as a line. + :return: The first line, decoded and with a newline at the end, and + the rest of the data. The line is None if the client disconnected + or the server stops before a line is complete. + """ while b'\n' not in buf: if not self.running: return None, buf @@ -120,7 +175,13 @@ def _recvline(self, channel, buf): lines = buf.split(b'\n') return lines[0].decode(self.encoding) + '\n', b'\n'.join(lines[1:]) - def _handle_connection(self, conn): + def _handle_connection(self, conn: socket.socket) -> None: + """ + Negotiates SSH, accepts any login and waits for a shell request, then + sends the banner and the responses of the reset device to the client + until it disconnects or the server stops. Logs errors, and closes the + connection. + """ t = paramiko.Transport(conn) t.local_version = local_version t.name = '%s-server-%s:%s' % (self._get_server_type(), self.host, self.port) diff --git a/src/Exscript/servers/telnetd.py b/src/Exscript/servers/telnetd.py index cb908b6d..2a8d18aa 100644 --- a/src/Exscript/servers/telnetd.py +++ b/src/Exscript/servers/telnetd.py @@ -28,6 +28,7 @@ import select import socket +from ..emulators import VirtualDevice from .server import Server _logger = logging.getLogger(__name__) @@ -47,11 +48,24 @@ class Telnetd(Server): daemon.join() # Wait until it terminates. """ - def __init__(self, host, port, device): + def __init__(self, host: str, port: int, device: VirtualDevice) -> None: + """ + Constructor. Takes the arguments of :class:`Server` except the + encoding, which is UTF-8. + """ Server.__init__(self, host, port, device) self._logger = _logger - def _recvline(self, conn, buf): + def _recvline(self, conn: socket.socket, buf: bytes) -> tuple[str | None, bytes]: + """ + Reads from the client until the given buffer, extended by the data + read, contains a complete line. Line ends are normalized to newlines. + + :param buf: The data that was read but not yet returned as a line. + :return: The first line, decoded and with a newline at the end, and + the rest of the data. The line is None if the client disconnected + or the server stops before a line is complete. + """ while b'\n' not in buf: r, w, x = select.select([conn], [], [], self.timeout) if not self.running: @@ -65,7 +79,12 @@ def _recvline(self, conn, buf): lines = buf.split(b'\n') return lines[0].decode(self.encoding) + '\n', b'\n'.join(lines[1:]) - def _handle_connection(self, conn): + def _handle_connection(self, conn: socket.socket) -> None: + """ + Sends the banner and the responses of the reset device to the client + until it disconnects or the server stops. Logs errors, and closes the + connection. + """ try: # send the banner conn.send(self.device.init().encode('utf8')) From 6e01d4a774b050760c5c800ab92ad065a8d9c3a7 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 10 Nov 2026 15:37:27 +0100 Subject: [PATCH 069/250] Annotate the Telnet and Dummy adapters telnetlib and the functions of Exscript.protocols are annotated, too. Only annotations, docstrings and typing.cast() are added. --- src/Exscript/protocols/__init__.py | 25 +++++--- src/Exscript/protocols/dummy.py | 80 +++++++++++++++++++----- src/Exscript/protocols/telnet.py | 68 ++++++++++++++++----- src/Exscript/protocols/telnetlib.py | 94 +++++++++++++++++------------ 4 files changed, 194 insertions(+), 73 deletions(-) diff --git a/src/Exscript/protocols/__init__.py b/src/Exscript/protocols/__init__.py index 71326b3e..3592f6df 100644 --- a/src/Exscript/protocols/__init__.py +++ b/src/Exscript/protocols/__init__.py @@ -20,6 +20,13 @@ # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" +The protocol adapters, and functions that create them by the name of the +protocol or from a URL. +""" + +from typing import TYPE_CHECKING, Any, cast + from .. import Account from ..util.cast import to_host from ..util.url import Url @@ -29,7 +36,10 @@ from .ssh2 import SSH2 from .telnet import Telnet -protocol_map = { +if TYPE_CHECKING: + from ..host import Host + +protocol_map: dict[str, type[Protocol]] = { 'dummy': Dummy, 'pseudo': Dummy, 'telnet': Telnet, @@ -39,13 +49,13 @@ } -def get_protocol_from_name(name): +def get_protocol_from_name(name: str) -> type[Protocol]: """ Returns the protocol class for the protocol with the given name. :type name: str :param name: The name of the protocol. - :rtype: Protocol + :rtype: type[Protocol] :return: The protocol class. """ cls = protocol_map.get(name) @@ -54,7 +64,7 @@ def get_protocol_from_name(name): return cls -def create_protocol(name, **kwargs): +def create_protocol(name: str, **kwargs: Any) -> Protocol: """ Returns an instance of the protocol with the given name. @@ -69,7 +79,7 @@ def create_protocol(name, **kwargs): return cls(**kwargs) -def prepare(host, default_protocol='ssh', **kwargs): +def prepare(host: 'str | Host', default_protocol: str = 'ssh', **kwargs: Any) -> Protocol: """ Creates an instance of the protocol by either parsing the given URL-formatted hostname using :class:`Exscript.util.url`, or according to @@ -89,11 +99,12 @@ def prepare(host, default_protocol='ssh', **kwargs): conn = create_protocol(protocol, **kwargs) if protocol == 'pseudo': filename = host.get_address() - conn.device.add_commands_from_file(filename) + # protocol_map maps 'pseudo' to Dummy. + cast(Dummy, conn).device.add_commands_from_file(filename) return conn -def connect(host, default_protocol='ssh', **kwargs): +def connect(host: 'str | Host', default_protocol: str = 'ssh', **kwargs: Any) -> Protocol: """ Like :class:`prepare()`, but also connects to the host by calling :class:`Protocol.connect()`. If the URL or host contain any login info, this diff --git a/src/Exscript/protocols/dummy.py b/src/Exscript/protocols/dummy.py index cd32f9e6..8a97ab95 100644 --- a/src/Exscript/protocols/dummy.py +++ b/src/Exscript/protocols/dummy.py @@ -25,10 +25,15 @@ """ import logging +import re +from typing import TYPE_CHECKING, Any, cast from ..emulators import VirtualDevice from .protocol import Protocol +if TYPE_CHECKING: + from ..key import PrivateKey + _logger = logging.getLogger(__name__) _logger.addHandler(logging.NullHandler()) @@ -39,7 +44,7 @@ class Dummy(Protocol): a :class:`Exscript.emulators.VirtualDevice` internally. """ - def __init__(self, device=None, **kwargs): + def __init__(self, device: VirtualDevice | None = None, **kwargs: Any) -> None: """ .. HINT:: Also supports all keyword arguments that :class:`Protocol` supports. @@ -55,49 +60,94 @@ def __init__(self, device=None, **kwargs): self._init_done = False # Public self.connect_rate_limit = 0 - self.device = device + # A default device replaces None below; the cast is for the type checker. + self.device = cast(VirtualDevice, device) if not self.device: self.device = VirtualDevice('dummy', strict=False) - def is_dummy(self): + def is_dummy(self) -> bool: + """ + Returns True, because this adapter talks to a virtual device instead + of a network host. + """ return True - def _say(self, string): + def _say(self, string: str) -> None: + """ + Handles the given text as if the device had sent it: passes it to + _receive_cb() and appends it to the buffer. + """ self._receive_cb(string) self._buffer.append(string) - def _connect(self): - # To more correctly mimic the behavior of a network device, we - # do not send the banner here, but in authenticate() instead. + def _connect(self) -> bool: + """ + Opens the virtual connection by clearing the buffer, and returns True. + Unlike a network host, the device sends no banner here; _doinit() + handles it later. + """ + # To mimic a network device more closely, the banner is not sent here, + # but by _doinit() when the client authenticates or first waits for data. self._buffer.clear() return True - def _doinit(self): + def _doinit(self) -> None: + """ + Initializes the device and handles its initial response, the banner + and the first prompt, as received data, once per connection. + """ if not self._init_done: self._init_done = True self._say(self.device.init()) - def _protocol_authenticate(self, user, password): + def _protocol_authenticate(self, user: str, password: str | None) -> None: + """ + Calls _doinit(); the virtual device checks no credentials at the + protocol level. + """ self._doinit() - def _protocol_authenticate_by_key(self, user, key): + def _protocol_authenticate_by_key(self, user: str, key: 'PrivateKey') -> None: + """ + Calls _doinit(); the virtual device checks no credentials at the + protocol level. + """ self._doinit() - def _send(self, data): + def _send(self, data: str) -> None: + """ + Passes the given data to the device, and handles the response of the + device as received data. + """ response = self.device.do(data) self._send_cb(data) self._say(response) - def _domatch(self, prompt, flush): + def _domatch(self, prompt: list[re.Pattern[str]], flush: bool) -> tuple[int, re.Match[str]]: + """ + Like Protocol._domatch(), but calls _doinit() first, so the banner + and the first prompt are received also without authentication. + """ self._doinit() return super()._domatch(prompt, flush) - def _fill_buffer(self, timeout): + def _fill_buffer(self, timeout: float | None) -> bool | None: + """ + Returns None, as if the time ran out: the device answers only in + _send(), so no more data arrives. + """ return None # the device answers only in _send(), so nothing more arrives - def _logout(self): + def _logout(self) -> None: + """ + Does nothing: the virtual device needs no logout. + """ pass - def _close(self, force=False): + def _close(self, force: bool = False) -> None: + """ + Handles a newline as received data, and resets the session, so the + next connection receives the banner again. + """ self._say('\n') self._init_done = False diff --git a/src/Exscript/protocols/telnet.py b/src/Exscript/protocols/telnet.py index ab69222d..a27dea70 100644 --- a/src/Exscript/protocols/telnet.py +++ b/src/Exscript/protocols/telnet.py @@ -25,6 +25,9 @@ """ import logging +import socket +from collections.abc import Callable, Mapping +from typing import Any, Self, cast from . import telnetlib from .protocol import Protocol @@ -38,22 +41,33 @@ class Telnet(Protocol): The Telnet protocol adapter. """ - def __init__(self, **kwargs): + def __init__(self, **kwargs: Any) -> None: + """ + Constructor. Takes the keyword arguments of :class:`Protocol`. + """ Protocol.__init__(self, **kwargs) # Module-Level self._logger = _logger # Internals - self._tn = None + self._tn: telnetlib.Telnet | None = None # cast where a connection is required - def _telnetlib_received(self, data): + def _telnetlib_received(self, data: str) -> None: + """ + Handles the text that the Telnet client received: passes it to + _receive_cb() and appends it to the buffer. + """ self._receive_cb(data) self._buffer.append(data) - def _connect(self): + def _connect(self) -> bool: + """ + Opens the Telnet connection, which negotiates the terminal type and + the window size, and returns True. + """ assert self._tn is None self._tn = telnetlib.Telnet( self.host, - self.port, + cast(int, self.port), # connect() checks that the port is set encoding=self.encoding, connect_timeout=self.connect_timeout, termsize=(self.term_height, self.term_width), @@ -64,24 +78,50 @@ def _connect(self): self._tn.set_debuglevel(1) return True - def _send(self, data): + def _send(self, data: str) -> None: + """ + Sends the given text to the host. + """ try: - self._tn.write(data) # telnetlib handles encoding + cast(telnetlib.Telnet, self._tn).write(data) # telnetlib handles encoding self._send_cb(data) except Exception: self._dbg(1, 'Error while writing to connection') raise - def _fill_buffer(self, timeout): - return self._tn.receive(timeout) # the callback appends the text to the buffer + def _fill_buffer(self, timeout: float | None) -> bool | None: + """ + Waits for data from the host and reads it once. + """ + # The callback appends the text to the buffer. + return cast(telnetlib.Telnet, self._tn).receive(timeout) - def _set_terminal_size(self, rows, cols): - self._tn.set_window_size(rows, cols) + def _set_terminal_size(self, rows: int, cols: int) -> None: + """ + Sets the window size, and sends it if the host asked for it (NAWS). + """ + cast(telnetlib.Telnet, self._tn).set_window_size(rows, cols) - def _interact(self, key_handlers=None, handle_window_size=True): - return self._open_shell(self._tn.sock, key_handlers, handle_window_size) + def _interact( + self, + key_handlers: Mapping[str, Callable[[Self], object]] | None = None, + handle_window_size: bool = True, + ) -> None: + """ + Opens the interactive shell on the socket of the Telnet connection. + """ + return self._open_shell( + cast(socket.socket, cast(telnetlib.Telnet, self._tn).sock), + key_handlers, + handle_window_size, + ) - def _close(self, force=False): + def _close(self, force: bool = False) -> None: + """ + Closes the Telnet connection. Unless forced, first receives the rest + of the data, until the host closes the connection or sends nothing + for 2 seconds. + """ if self._tn: if not force: try: diff --git a/src/Exscript/protocols/telnetlib.py b/src/Exscript/protocols/telnetlib.py index e6f1c61e..e7b8841d 100644 --- a/src/Exscript/protocols/telnetlib.py +++ b/src/Exscript/protocols/telnetlib.py @@ -28,6 +28,8 @@ import logging import socket import struct +from collections.abc import Callable +from typing import Any, cast __all__ = ['Telnet'] @@ -56,27 +58,30 @@ _READ_SIZE = 65536 -def _command(*codes): +def _command(*codes: int) -> bytes: + """ + Returns a Telnet command: IAC followed by the given codes. + """ return bytes([IAC, *codes]) class Telnet: """ - A Telnet connection that answers option negotiation and delivers the - received text to a callback and to expect(). + A Telnet connection that answers the option negotiation and passes the + received text to a callback. """ def __init__( self, - host=None, - port=0, - encoding='latin-1', + host: str | None = None, + port: int = 0, + encoding: str = 'latin-1', *, - connect_timeout=None, - termsize=None, - termtype='dumb', - receive_callback=None, - ): + connect_timeout: float | None = None, + termsize: tuple[int, int] | None = None, + termtype: str = 'dumb', + receive_callback: Callable[[str], object] | None = None, + ) -> None: """ Constructor. Connects to the host if one is given. @@ -98,7 +103,9 @@ def __init__( self.debuglevel = 0 self.host = host self.port = port - self.sock = None + # Set by open(). The methods that send or receive need a connection; + # they cast the socket for the type checker. + self.sock: socket.socket | None = None self.encoding = encoding self.connect_timeout = connect_timeout self.window_size = termsize @@ -108,11 +115,11 @@ def __init__( self.eof = False self.rawq = b'' # received bytes that end with an incomplete command self._decoder = codecs.getincrementaldecoder(encoding)() - self._sb = None # the data of the subnegotiation in progress + self._sb: bytearray | None = None # the data of the subnegotiation in progress if host: self.open(host, port) - def open(self, host, port=0): + def open(self, host: str, port: int = 0) -> None: """ Connects to the host, trying all of its addresses (IPv4 and IPv6). @@ -129,12 +136,12 @@ def open(self, host, port=0): except OSError as exc: raise OSError(f'{exc} => telnet://{host}:{self.port}') from exc - def msg(self, msg, *args): + def msg(self, msg: str, *args: Any) -> None: """Logs a debug message if the debug level is above 0.""" if self.debuglevel > 0: _logger.debug(msg, *args) - def set_debuglevel(self, debuglevel): + def set_debuglevel(self, debuglevel: int) -> None: """ Sets the debug level; above 0, the received data and the option negotiation are logged. @@ -144,14 +151,14 @@ def set_debuglevel(self, debuglevel): """ self.debuglevel = debuglevel - def close(self): + def close(self) -> None: """Closes the connection.""" sock, self.sock = self.sock, None self.eof = True if sock: sock.close() - def write(self, buffer): + def write(self, buffer: str | bytes) -> None: """ Sends the given text or bytes, doubling the IAC bytes. The data is not logged, because it may contain a password. @@ -161,9 +168,9 @@ def write(self, buffer): """ if isinstance(buffer, str): buffer = buffer.encode(self.encoding) - self.sock.sendall(buffer.replace(_IAC, _IAC + _IAC)) + cast(socket.socket, self.sock).sendall(buffer.replace(_IAC, _IAC + _IAC)) - def set_window_size(self, rows, cols): + def set_window_size(self, rows: int, cols: int) -> None: """ Sets the window size, and sends it if the server asked for it (NAWS). @@ -176,37 +183,50 @@ def set_window_size(self, rows, cols): if not self.can_naws: return size = struct.pack('!HH', min(cols, 0xFFFF), min(rows, 0xFFFF)) - self.sock.sendall(_command(SB, NAWS) + size.replace(_IAC, _IAC + _IAC) + _command(SE)) + cast(socket.socket, self.sock).sendall( + _command(SB, NAWS) + size.replace(_IAC, _IAC + _IAC) + _command(SE) + ) - def _negotiate(self, command, option): + def _negotiate(self, command: int, option: int) -> None: + """ + Answers an option negotiation command (DO, DONT, WILL or WONT) of the + server: agrees to send the terminal type and the window size, asks + for the echo, and refuses all other options. + """ self.msg('IAC %s %d', _NEGOTIATION[command], option) if command == DO and option in (TTYPE, NAWS): - self.sock.sendall(_command(WILL, option)) + cast(socket.socket, self.sock).sendall(_command(WILL, option)) if option == NAWS: self.can_naws = True if self.window_size: self.set_window_size(*self.window_size) elif command in (DO, DONT): - self.sock.sendall(_command(WONT, option)) + cast(socket.socket, self.sock).sendall(_command(WONT, option)) elif option == ECHO: # Exscript always asks for the echo, also when the server refuses it. - self.sock.sendall(_command(DO, option)) + cast(socket.socket, self.sock).sendall(_command(DO, option)) else: - self.sock.sendall(_command(DONT, option)) + cast(socket.socket, self.sock).sendall(_command(DONT, option)) - def _subnegotiate(self, data): + def _subnegotiate(self, data: bytes) -> None: + """ + Answers the given subnegotiation of the server: sends the terminal + type if the server asks for it. + """ self.msg('IAC SB %r', data) if data == bytes([TTYPE, TTYPE_SEND]): self.msg('Indicating support for terminal type %s.', self.termtype) ttype = self.termtype.encode('latin-1') - self.sock.sendall(_command(SB, TTYPE, TTYPE_IS) + ttype + _command(SE)) + cast(socket.socket, self.sock).sendall( + _command(SB, TTYPE, TTYPE_IS) + ttype + _command(SE) + ) - def process_rawq(self): + def process_rawq(self) -> None: """ - Moves the received data from the raw queue to the cooked queue, - removes the Telnet commands and answers the option negotiation. - Keeps an incomplete command in the raw queue until more data arrives, - so it never blocks. + Processes the data in the raw queue: removes the Telnet commands, + answers the option negotiation, and passes the decoded text to the + callback. Keeps an incomplete command in the raw queue until more data + arrives, so it never blocks. """ data, self.rawq = self.rawq, b'' cooked = bytearray() @@ -258,17 +278,17 @@ def process_rawq(self): if text and self.data_callback is not None: self.data_callback(text) - def fill_rawq(self): + def fill_rawq(self) -> None: """ Reads once from the socket into the raw queue. Blocks if no data is available, and sets eof when the connection was closed. """ - data = self.sock.recv(_READ_SIZE) + data = cast(socket.socket, self.sock).recv(_READ_SIZE) self.msg('Recv: %r', data) self.eof = not data self.rawq += data - def receive(self, timeout): + def receive(self, timeout: float | None) -> bool | None: """ Waits for data, reads it once and passes the received text to the callback. @@ -282,7 +302,7 @@ def receive(self, timeout): """ if self.eof: return False - self.sock.settimeout(timeout) + cast(socket.socket, self.sock).settimeout(timeout) try: self.fill_rawq() except (TimeoutError, BlockingIOError): # BlockingIOError: timeout 0 From d14249c0bb2b7ed6d412d477220117ed722f390e Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 10 Nov 2026 15:37:27 +0100 Subject: [PATCH 070/250] Annotate the device drivers Each docstring says which banner, prompt or reply the driver recognizes. Only annotations and docstrings are added; the patterns are unchanged. --- src/Exscript/protocols/drivers/ace.py | 40 ++++++++++++-- src/Exscript/protocols/drivers/adtran.py | 35 +++++++++++-- src/Exscript/protocols/drivers/aironet.py | 26 ++++++++-- src/Exscript/protocols/drivers/aix.py | 16 +++++- .../protocols/drivers/arbor_peakflow.py | 18 ++++++- src/Exscript/protocols/drivers/aruba.py | 39 ++++++++++++-- src/Exscript/protocols/drivers/bigip.py | 39 ++++++++++++-- src/Exscript/protocols/drivers/brocade.py | 43 +++++++++++++-- src/Exscript/protocols/drivers/cienasaos.py | 27 ++++++++-- src/Exscript/protocols/drivers/enterasys.py | 18 ++++++- .../protocols/drivers/enterasys_wc.py | 27 ++++++++-- src/Exscript/protocols/drivers/eos.py | 38 ++++++++++++-- .../protocols/drivers/ericsson_ban.py | 22 ++++++-- src/Exscript/protocols/drivers/fortios.py | 30 +++++++++-- src/Exscript/protocols/drivers/generic.py | 11 +++- .../protocols/drivers/hp_pro_curve.py | 42 +++++++++++++-- src/Exscript/protocols/drivers/icotera.py | 18 +++++-- src/Exscript/protocols/drivers/ios.py | 52 +++++++++++++++++-- src/Exscript/protocols/drivers/ios_xr.py | 28 ++++++++-- src/Exscript/protocols/drivers/isam.py | 25 +++++++-- src/Exscript/protocols/drivers/junos.py | 32 ++++++++++-- src/Exscript/protocols/drivers/junos_erx.py | 42 +++++++++++++-- src/Exscript/protocols/drivers/mrv.py | 27 ++++++++-- src/Exscript/protocols/drivers/nxos.py | 41 +++++++++++++-- src/Exscript/protocols/drivers/one_os.py | 41 +++++++++++++-- src/Exscript/protocols/drivers/rios.py | 36 +++++++++++-- src/Exscript/protocols/drivers/shell.py | 24 +++++++-- .../protocols/drivers/smart_edge_os.py | 41 +++++++++++++-- src/Exscript/protocols/drivers/sp_vxoa.py | 40 ++++++++++++-- src/Exscript/protocols/drivers/sros.py | 30 +++++++++-- src/Exscript/protocols/drivers/vrp.py | 32 ++++++++++-- 31 files changed, 877 insertions(+), 103 deletions(-) diff --git a/src/Exscript/protocols/drivers/ace.py b/src/Exscript/protocols/drivers/ace.py index 638fd75c..6848ed3d 100644 --- a/src/Exscript/protocols/drivers/ace.py +++ b/src/Exscript/protocols/drivers/ace.py @@ -25,9 +25,15 @@ """ import re +from typing import TYPE_CHECKING from .driver import Driver +if TYPE_CHECKING: + from ...account import Account, AccountProxy + from ...util.impl import _Context + from ..protocol import Protocol + _user_re = [re.compile(r'user ?name: ?$', re.I)] _password_re = [re.compile(r'(?:(?:\r\n|[\r\n])Password: ?|last resort password:)$')] _prompt_re = [re.compile(r'(?:\r\n|[\r\n])[\-\w+\.:/]+(?:\([^\)]+\))?[>#] ?$')] @@ -41,21 +47,47 @@ class ACEDriver(Driver): - def __init__(self): + """ + The ``ace`` driver for Cisco Application Control Engine (ACE). Its + prompt ends with ">" or "#", e.g. "hostname/context#". + """ + + def __init__(self) -> None: + """ + Constructor. Sets the patterns of the user name prompt "Username:", + the password prompts, the command prompt and the error messages, such + as "%Error" and "Invalid input". + """ Driver.__init__(self, 'ace') self.user_re = _user_re self.password_re = _password_re self.prompt_re = _prompt_re self.error_re = _error_re - def check_head_for_os(self, string): + def check_head_for_os(self, string: str) -> int: + """ + Returns 90 if the data contains "Cisco Application Control Software", + else 0. + """ if 'Cisco Application Control Software' in string: return 90 return 0 - def init_terminal(self, conn): + def init_terminal(self, conn: 'Protocol') -> None: + """ + Turns off the pager with "term len 0". + """ conn.execute('term len 0') - def auto_authorize(self, conn, account, flush, bailout): + def auto_authorize( + self, + conn: 'Protocol', + account: 'Account | AccountProxy | _Context | None', + flush: bool, + bailout: bool, + ) -> None: + """ + Sends "enable", then authorizes with conn.app_authorize(). + """ conn.send('enable\r') conn.app_authorize(account, flush, bailout) diff --git a/src/Exscript/protocols/drivers/adtran.py b/src/Exscript/protocols/drivers/adtran.py index 62b93020..292050ae 100644 --- a/src/Exscript/protocols/drivers/adtran.py +++ b/src/Exscript/protocols/drivers/adtran.py @@ -27,18 +27,47 @@ no SSH banner so no possibility for check_*_for_os(). """ +from typing import TYPE_CHECKING + from .driver import Driver +if TYPE_CHECKING: + from ...account import Account, AccountProxy + from ...util.impl import _Context + from ..protocol import Protocol + class AdtranDriver(Driver): - def __init__(self): + """ + The ``adtran`` driver for the Adtran DSLAMs. It uses the generic + patterns, and the OS guesser does not recognize it. + """ + + def __init__(self) -> None: + """ + Constructor. Keeps the generic patterns. + """ Driver.__init__(self, 'adtran') - def init_terminal(self, conn): + def init_terminal(self, conn: 'Protocol') -> None: + """ + Turns off the pager with "terminal length 0" and sets the terminal + width with "terminal column 0". + """ conn.execute('terminal length 0') conn.execute('terminal column 0') - def auto_authorize(self, conn, account, flush, bailout): + def auto_authorize( + self, + conn: 'Protocol', + account: 'Account | AccountProxy | _Context | None', + flush: bool, + bailout: bool, + ) -> None: + """ + Calls init_terminal(), sends "enable", then authorizes with + conn.app_authorize(). + """ self.init_terminal(conn) conn.send('enable\r\n') conn.app_authorize(account, flush, bailout) diff --git a/src/Exscript/protocols/drivers/aironet.py b/src/Exscript/protocols/drivers/aironet.py index 8ef8dbf6..951b180f 100644 --- a/src/Exscript/protocols/drivers/aironet.py +++ b/src/Exscript/protocols/drivers/aironet.py @@ -25,20 +25,37 @@ """ import re +from typing import TYPE_CHECKING from .driver import Driver +if TYPE_CHECKING: + from ..protocol import Protocol + _user_re = [re.compile(r'User:\s$', re.I)] _prompt_re = [re.compile(r'(?:\r\n|[\r\n])(?:\([^\)]+\))\s>$')] class AironetDriver(Driver): - def __init__(self): + """ + The ``aironet`` driver for Cisco Aironet Wireless Controllers. Its prompt + is a name in parentheses followed by " >", e.g. "(Cisco Controller) >". + """ + + def __init__(self) -> None: + """ + Constructor. Sets the patterns of the user name prompt "User:" and of + the command prompt. + """ Driver.__init__(self, 'aironet') self.user_re = _user_re self.prompt_re = _prompt_re - def check_head_for_os(self, string): + def check_head_for_os(self, string: str) -> int: + """ + Returns 90 if the data contains "(Cisco Controller)" or "(WiSM-slot", + else 87 if it contains ") >", else 0. + """ if '(Cisco Controller)' in string: return 90 elif '(WiSM-slot' in string: @@ -47,5 +64,8 @@ def check_head_for_os(self, string): return 87 return 0 - def init_terminal(self, conn): + def init_terminal(self, conn: 'Protocol') -> None: + """ + Turns off the pager with "config paging disable". + """ conn.execute('config paging disable') diff --git a/src/Exscript/protocols/drivers/aix.py b/src/Exscript/protocols/drivers/aix.py index 07865d53..ffc38ca3 100644 --- a/src/Exscript/protocols/drivers/aix.py +++ b/src/Exscript/protocols/drivers/aix.py @@ -34,12 +34,24 @@ class AIXDriver(Driver): - def __init__(self): + """ + The ``aix`` driver for AIX. Its user name prompt is "Authorized Login:", + and its password prompt "'s Password:". + """ + + def __init__(self) -> None: + """ + Constructor. Sets the patterns of the user name and password prompts. + """ Driver.__init__(self, 'aix') self.user_re = _user_re self.password_re = _password_re - def check_head_for_os(self, string): + def check_head_for_os(self, string: str) -> int: + """ + Returns 70 if the data ends with the user name prompt + "Authorized Login: ", else 75 if it contains the word "AIX", else 0. + """ if _user_re[0].search(string): return 70 if _aix_re.search(string): diff --git a/src/Exscript/protocols/drivers/arbor_peakflow.py b/src/Exscript/protocols/drivers/arbor_peakflow.py index b5329fd8..83df287a 100644 --- a/src/Exscript/protocols/drivers/arbor_peakflow.py +++ b/src/Exscript/protocols/drivers/arbor_peakflow.py @@ -40,13 +40,27 @@ class ArborPeakflowDriver(Driver): - def __init__(self): + """ + The ``arbor_peakflow`` driver for Peakflow SP by Arbor Networks. It + recognizes prompts like "admin@host:/#", the "DIAG>" prompt of the shell + up to version 8.0, and the question "Exit anyway?" on exit. + """ + + def __init__(self) -> None: + """ + Constructor. Sets the patterns of the user name, password and command + prompts. + """ Driver.__init__(self, 'arbor_peakflow') self.user_re = _user_re self.password_re = _password_re self.prompt_re = _prompt_re - def check_head_for_os(self, string): + def check_head_for_os(self, string: str) -> int: + """ + Returns 97 if the data contains the word "Arbor", "ArbOS" or + "Peakflow" in any case, else 0. + """ if _os_re.search(string): return 97 return 0 diff --git a/src/Exscript/protocols/drivers/aruba.py b/src/Exscript/protocols/drivers/aruba.py index 182c5cda..8d1e7e2f 100644 --- a/src/Exscript/protocols/drivers/aruba.py +++ b/src/Exscript/protocols/drivers/aruba.py @@ -25,9 +25,15 @@ """ import re +from typing import TYPE_CHECKING from .driver import Driver +if TYPE_CHECKING: + from ...account import Account, AccountProxy + from ...util.impl import _Context + from ..protocol import Protocol + _user_re = [re.compile(r'user ?name: ?$', re.I)] _password_re = [re.compile(r'(?:(?:\r\n|[\r\n])Password: ?|last resort password:)$')] _prompt_re = [re.compile(r'(?:\r\n|[\r\n])\(\w+\) [>#] ?$')] @@ -41,21 +47,46 @@ class ArubaDriver(Driver): - def __init__(self): + """ + The ``aruba`` driver for Aruba controllers. Its prompt is a name in + parentheses followed by " >" or " #", e.g. "(Aruba650) >". + """ + + def __init__(self) -> None: + """ + Constructor. Sets the patterns of the user name prompt "Username:", + the password prompts, the command prompt and the error messages, such + as "%Error" and "Invalid input". + """ Driver.__init__(self, 'aruba') self.user_re = _user_re self.password_re = _password_re self.prompt_re = _prompt_re self.error_re = _error_re - def check_head_for_os(self, string): + def check_head_for_os(self, string: str) -> int: + """ + Returns 88 if the data contains "aruba" in any case, else 0. + """ if 'aruba' in string.lower(): return 88 return 0 - def init_terminal(self, conn): + def init_terminal(self, conn: 'Protocol') -> None: + """ + Turns off the pager with "no paging". + """ conn.execute('no paging') - def auto_authorize(self, conn, account, flush, bailout): + def auto_authorize( + self, + conn: 'Protocol', + account: 'Account | AccountProxy | _Context | None', + flush: bool, + bailout: bool, + ) -> None: + """ + Sends "enable", then authorizes with conn.app_authorize(). + """ conn.send('enable\r') conn.app_authorize(account, flush, bailout) diff --git a/src/Exscript/protocols/drivers/bigip.py b/src/Exscript/protocols/drivers/bigip.py index 7322e07c..051518e4 100644 --- a/src/Exscript/protocols/drivers/bigip.py +++ b/src/Exscript/protocols/drivers/bigip.py @@ -25,9 +25,15 @@ """ import re +from typing import TYPE_CHECKING from .driver import Driver +if TYPE_CHECKING: + from ...account import Account, AccountProxy + from ...util.impl import _Context + from ..protocol import Protocol + _user_re = [re.compile(r'user ?name: ?$', re.I)] _password_re = [re.compile(r'(?:(?:\r\n|[\r\n])Password: ?|last resort password:)$')] _prompt_re = [re.compile(r'\S+@(\(.*?\)){1,4}\(tmos.*?\)#\s?')] @@ -38,20 +44,45 @@ class BigIPDriver(Driver): - def __init__(self): + """ + The ``bigip`` driver for the TMSH shell of F5 Big-IP systems. It + recognizes prompts like "user@(hostname)(Active)(/Common)(tmos)#". + """ + + def __init__(self) -> None: + """ + Constructor. Sets the patterns of the user name prompt "Username:", + the password prompts, the command prompt and the error messages + "Syntax Error" and "connection timed out". + """ Driver.__init__(self, 'bigip') self.user_re = _user_re self.password_re = _password_re self.prompt_re = _prompt_re self.error_re = _error_re - def check_head_for_os(self, string): + def check_head_for_os(self, string: str) -> int: + """ + Returns 90 if the data contains "(tmos)", else 0. + """ if "(tmos)" in string: return 90 return 0 - def init_terminal(self, conn): + def init_terminal(self, conn: 'Protocol') -> None: + """ + Turns off the pager with "modify cli preference pager disabled". + """ conn.execute('modify cli preference pager disabled') - def auto_authorize(self, conn, account, flush, bailout): + def auto_authorize( + self, + conn: 'Protocol', + account: 'Account | AccountProxy | _Context | None', + flush: bool, + bailout: bool, + ) -> None: + """ + Does nothing, i.e. skips the authorization. + """ pass diff --git a/src/Exscript/protocols/drivers/brocade.py b/src/Exscript/protocols/drivers/brocade.py index c86f45c0..609a1066 100644 --- a/src/Exscript/protocols/drivers/brocade.py +++ b/src/Exscript/protocols/drivers/brocade.py @@ -25,9 +25,15 @@ """ import re +from typing import TYPE_CHECKING from .driver import Driver +if TYPE_CHECKING: + from ...account import Account, AccountProxy + from ...util.impl import _Context + from ..protocol import Protocol + _user_re = [re.compile(r'(?:\r\n|[\r\n])(Please Enter Login Name: |User Name:)$')] _password_re = [re.compile(r'(?:\r\n|[\r\n])(Please Enter Password: |Password:)$')] _warning = r'(?:Warning: \d+ user\(s\) already in config mode\.)' @@ -43,23 +49,52 @@ class BrocadeDriver(Driver): - def __init__(self): + """ + The ``brocade`` driver for Brocade XMR/MLX devices. Its prompt starts with + "telnet@" or "SSH@", e.g. "SSH@router1#" or "telnet@router1(config)#". + """ + + def __init__(self) -> None: + """ + Constructor. Sets the patterns of the user name prompts + "Please Enter Login Name:" and "User Name:", the password prompts, the + command prompt, also right after the warning + "Warning: 1 user(s) already in config mode.", and the error messages, + such as "%Error". + """ Driver.__init__(self, 'brocade') self.user_re = _user_re self.password_re = _password_re self.prompt_re = _prompt_re self.error_re = _error_re - def check_head_for_os(self, string): + def check_head_for_os(self, string: str) -> int: + """ + Returns 95 if the data contains "User Access Verification" and, after + an empty line, "Please Enter Login Name", else 90 if it ends with a + Brocade prompt, else 0. + """ if 'User Access Verification\r\n\r\nPlease Enter Login Name' in string: return 95 if _prompt_re[0].search(string): return 90 return 0 - def init_terminal(self, conn): + def init_terminal(self, conn: 'Protocol') -> None: + """ + Turns off the pager with "terminal length 0". + """ conn.execute('terminal length 0') - def auto_authorize(self, conn, account, flush, bailout): + def auto_authorize( + self, + conn: 'Protocol', + account: 'Account | AccountProxy | _Context | None', + flush: bool, + bailout: bool, + ) -> None: + """ + Sends "enable", then authorizes with conn.app_authorize(). + """ conn.send('enable\r') conn.app_authorize(account, flush, bailout) diff --git a/src/Exscript/protocols/drivers/cienasaos.py b/src/Exscript/protocols/drivers/cienasaos.py index 944c7e8c..27ccc5dc 100644 --- a/src/Exscript/protocols/drivers/cienasaos.py +++ b/src/Exscript/protocols/drivers/cienasaos.py @@ -25,9 +25,13 @@ """ import re +from typing import TYPE_CHECKING from .driver import Driver +if TYPE_CHECKING: + from ..protocol import Protocol + _user_re = [re.compile(r'[^:]* login: ?$', re.I)] _password_re = [re.compile(r'Password: ?$')] _prompt_re = [re.compile(r'(?:\r\n|[\r\n])[\-\w+\.:/]+[>#] ?$')] @@ -41,17 +45,34 @@ class CienaSAOSDriver(Driver): - def __init__(self): + """ + The ``cienasaos`` driver for Ciena SAOS carrier Ethernet devices. Its + prompt is a name followed by ">" or "#", e.g. "hostname>". + """ + + def __init__(self) -> None: + """ + Constructor. Sets the patterns of the user name prompt, e.g. + "hostname login:", the password prompt, the command prompt and the + error messages, such as "SHELL PARSER FAILURE". + """ Driver.__init__(self, 'cienasaos') self.user_re = _user_re self.password_re = _password_re self.prompt_re = _prompt_re self.error_re = _error_re - def check_head_for_os(self, string): + def check_head_for_os(self, string: str) -> int: + """ + Returns 90 if the data contains + "SAOS is True Carrier Ethernet TM software", else 0. + """ if 'SAOS is True Carrier Ethernet TM software' in string: return 90 return 0 - def init_terminal(self, conn): + def init_terminal(self, conn: 'Protocol') -> None: + """ + Turns off the pager with "system shell session set more off". + """ conn.execute('system shell session set more off') diff --git a/src/Exscript/protocols/drivers/enterasys.py b/src/Exscript/protocols/drivers/enterasys.py index af5ff4b5..1b808251 100644 --- a/src/Exscript/protocols/drivers/enterasys.py +++ b/src/Exscript/protocols/drivers/enterasys.py @@ -35,13 +35,27 @@ class EnterasysDriver(Driver): - def __init__(self): + """ + The ``enterasys`` driver for Enterasys devices. Its prompt is a name with + an optional mode in parentheses, followed by ">", "->" or "#", e.g. + "switch(su)->". + """ + + def __init__(self) -> None: + """ + Constructor. Sets the patterns of the user name prompt "Username:", + the password prompt and the command prompt. + """ Driver.__init__(self, 'enterasys') self.user_re = _user_re self.password_re = _password_re self.prompt_re = _prompt_re - def check_head_for_os(self, string): + def check_head_for_os(self, string: str) -> int: + """ + Returns 80 if the data contains the word "Enterasys" in any case, + else 0. + """ if _enterasys_re.search(string): return 80 return 0 diff --git a/src/Exscript/protocols/drivers/enterasys_wc.py b/src/Exscript/protocols/drivers/enterasys_wc.py index b954eea8..512679b5 100644 --- a/src/Exscript/protocols/drivers/enterasys_wc.py +++ b/src/Exscript/protocols/drivers/enterasys_wc.py @@ -27,9 +27,13 @@ """ import re +from typing import TYPE_CHECKING from .driver import Driver +if TYPE_CHECKING: + from ..protocol import Protocol + _user_re = [re.compile(r'(?:\r\n|[\r\n])Username: $')] _password_re = [re.compile(r'(?:\r\n|[\r\n])Password: $')] _prompt_re = [ @@ -40,16 +44,33 @@ class EnterasysWCDriver(Driver): - def __init__(self): + """ + The ``enterasys_wc`` driver for Enterasys/Extreme (HiPath) Wireless + Controller devices. Its prompt ends with "# ", e.g. "HWC-Name# ". + """ + + def __init__(self) -> None: + """ + Constructor. Sets the patterns of the user name prompt "Username:", + the password prompt and the command prompt, which may contain parts + that start with ":". + """ Driver.__init__(self, 'enterasys_wc') self.user_re = _user_re self.password_re = _password_re self.prompt_re = _prompt_re - def check_head_for_os(self, string): + def check_head_for_os(self, string: str) -> int: + """ + Returns 85 if the data contains "Enterasys Wireless Convergence OS" in + any case, else 0. + """ if _hwc_re.search(string): return 85 return 0 - def init_terminal(self, conn): + def init_terminal(self, conn: 'Protocol') -> None: + """ + Does nothing, like the base class. + """ pass diff --git a/src/Exscript/protocols/drivers/eos.py b/src/Exscript/protocols/drivers/eos.py index 2bd1bbf8..b349d557 100644 --- a/src/Exscript/protocols/drivers/eos.py +++ b/src/Exscript/protocols/drivers/eos.py @@ -25,9 +25,15 @@ """ import re +from typing import TYPE_CHECKING from .driver import Driver +if TYPE_CHECKING: + from ...account import Account, AccountProxy + from ...util.impl import _Context + from ..protocol import Protocol + # Match following prompt pattern variations: # Important: Upon first login, EOS does not return \r\n before prompt. # hostname(s1)> @@ -71,7 +77,20 @@ class EOSDriver(Driver): - def __init__(self): + """ + The ``eos`` driver for Arista EOS. Its prompt is a name with up to three + parts in parentheses, followed by ">" or "#", e.g. "hostname(s1)#" or + "hostname(vrf:blah)(config)#". The OS guesser does not recognize EOS. + """ + + def __init__(self) -> None: + """ + Constructor. Sets the patterns of the user name prompts "Username:" + and "login:", the password prompt, the command prompt, also without + the line break that EOS omits before the first prompt, the error + messages, such as "% Invalid input", and the login errors, such as + "Bad secret". + """ Driver.__init__(self, 'eos') self.user_re = _user_re self.password_re = _password_re @@ -79,10 +98,23 @@ def __init__(self): self.error_re = _error_re self.login_error_re = _login_fail_re - def init_terminal(self, conn): + def init_terminal(self, conn: 'Protocol') -> None: + """ + Executes "terminal dont-ask" and turns off the pager with + "terminal length 0". + """ conn.execute('terminal dont-ask') conn.execute('terminal length 0') - def auto_authorize(self, conn, account, flush, bailout): + def auto_authorize( + self, + conn: 'Protocol', + account: 'Account | AccountProxy | _Context | None', + flush: bool, + bailout: bool, + ) -> None: + """ + Sends "enable", then authorizes with conn.app_authorize(). + """ conn.send('enable\r') conn.app_authorize(account, flush, bailout) diff --git a/src/Exscript/protocols/drivers/ericsson_ban.py b/src/Exscript/protocols/drivers/ericsson_ban.py index 4222c42f..9e8359df 100644 --- a/src/Exscript/protocols/drivers/ericsson_ban.py +++ b/src/Exscript/protocols/drivers/ericsson_ban.py @@ -21,7 +21,7 @@ # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ -A driver for devices running Ericsson's Broadband Access Nodes (BAN) OS +A driver for devices running Ericsson's Broadband Access Nodes (BAN) OS. """ import re @@ -41,7 +41,19 @@ class EricssonBanDriver(Driver): - def __init__(self): + """ + The ``ericsson_ban`` driver for devices running Ericsson's Broadband + Access Nodes (BAN) OS. Its prompt is a name with an optional part in + parentheses, followed by "%" or "#"; the question "(y/n)[n]" counts as a + prompt, too. + """ + + def __init__(self) -> None: + """ + Constructor. Sets the patterns of the user name prompt "user:", the + password prompt "pass:", the command prompt, the error message + "(error)" and the login error "Login failed". + """ Driver.__init__(self, 'ericsson_ban') self.user_re = _user_re @@ -54,7 +66,11 @@ def __init__(self): # conn.send('enable\r\n') # conn.app_authorize(account, flush, bailout) - def check_head_for_os(self, string): + def check_head_for_os(self, string: str) -> int: + """ + Returns 90 if the data contains + "BLM - Broadband Loop Multiplexer" in any case, else 0. + """ if _ban_re.search(string): return 90 return 0 diff --git a/src/Exscript/protocols/drivers/fortios.py b/src/Exscript/protocols/drivers/fortios.py index 21b00a54..c4cce366 100644 --- a/src/Exscript/protocols/drivers/fortios.py +++ b/src/Exscript/protocols/drivers/fortios.py @@ -27,9 +27,13 @@ """ import re +from typing import TYPE_CHECKING from .driver import Driver +if TYPE_CHECKING: + from ..protocol import Protocol + _user_re = [re.compile(r'(?:\r\n|[\r\n])Username: $')] _password_re = [re.compile(r'(?:\r\n|[\r\n])Password: $')] _prompt_re = [ @@ -55,14 +59,30 @@ class FortiOSDriver(Driver): - def __init__(self): + """ + The ``fortios`` driver for FortiOS devices. Its prompt is a name, an + optional part in parentheses and "#" or "$", separated by spaces, e.g. + "Forti-Name # ". + """ + + def __init__(self) -> None: + """ + Constructor. Sets the patterns of the user name prompt "Username:", + the password prompt, the command prompt and the error messages, such + as "Command fail". + """ Driver.__init__(self, 'fortios') self.user_re = _user_re self.password_re = _password_re self.prompt_re = _prompt_re self.error_re = _error_re - def check_head_for_os(self, string): + def check_head_for_os(self, string: str) -> int: + """ + Returns 50 if the data is one line with the FortiOS prompt or a + password prompt like "user@host's password:", else 0. By default, a + FortiGate shows only the prompt. + """ # By default Fortigate shows only prompt if len(string.splitlines()) == 1: for head_re in _fortios_re: @@ -70,7 +90,11 @@ def check_head_for_os(self, string): return 50 return 0 - def init_terminal(self, conn): + def init_terminal(self, conn: 'Protocol') -> None: + """ + Turns off the pager with "set output standard" in "config global" and + "config system console", which it leaves with "end". + """ conn.execute('config global') conn.execute('config system console') conn.execute('set output standard') # no paging diff --git a/src/Exscript/protocols/drivers/generic.py b/src/Exscript/protocols/drivers/generic.py index 969f0694..110ae877 100644 --- a/src/Exscript/protocols/drivers/generic.py +++ b/src/Exscript/protocols/drivers/generic.py @@ -28,5 +28,14 @@ class GenericDriver(Driver): - def __init__(self): + """ + The ``generic`` driver, which uses the patterns of the base class + unchanged. It is used when the OS guesser does not recognize the device; + ``unknown`` is another name for it. + """ + + def __init__(self) -> None: + """ + Constructor. Keeps the patterns of the base class. + """ Driver.__init__(self, 'generic') diff --git a/src/Exscript/protocols/drivers/hp_pro_curve.py b/src/Exscript/protocols/drivers/hp_pro_curve.py index e439cfaa..69358ed3 100644 --- a/src/Exscript/protocols/drivers/hp_pro_curve.py +++ b/src/Exscript/protocols/drivers/hp_pro_curve.py @@ -25,9 +25,13 @@ """ import re +from typing import TYPE_CHECKING from .driver import Driver +if TYPE_CHECKING: + from ..protocol import Protocol + _user_re = [re.compile(r'(?:\r\n|[\r\n])Username: ?$')] _password_re = [re.compile(r'(?:\r\n|[\r\n])Password: ?$')] _prompt_re = [re.compile(r'(?:\r\n|[\r\n])[\-\w+\.:/]+[>#] ?$')] @@ -44,7 +48,18 @@ class HPProCurveDriver(Driver): - def __init__(self): + """ + The ``hp_pro_curve`` driver for HP ProCurve switches. Its prompt is a + name followed by ">" or "#"; the escape sequences around it are removed + before the prompt is matched. + """ + + def __init__(self) -> None: + """ + Constructor. Sets the patterns of the user name prompt "Username:", + the password prompt, the command prompt, the error messages, such as + "Invalid input:", and the login errors, such as "Invalid password". + """ Driver.__init__(self, 'hp_pro_curve') self.user_re = _user_re self.password_re = _password_re @@ -52,14 +67,30 @@ def __init__(self): self.error_re = _error_re self.login_error_re = _login_fail_re - def check_head_for_os(self, string): + def check_head_for_os(self, string: str) -> int: + """ + Returns 95 if the data contains "ProCurve", else 50 if it contains + "Hewlett-Packard", else 0. + """ if 'ProCurve' in string: return 95 if 'Hewlett-Packard' in string: return 50 return 0 - def clean_response_for_re_match(self, response): + def clean_response_for_re_match(self, response: str) -> tuple[str, str]: + """ + Removes the escape sequences from the end of the received data before + the prompt patterns are matched. If one of the first ten characters is + an escape character, the data before it is dropped. Then "ESC E" + becomes a line break, and the control sequences that start with + "ESC [" or the character U+009B, e.g. cursor movements, are removed. + + :param response: The end of the received data. + :return: The cleaned data up to the first escape character that is + left, and the rest from that character on, e.g. an incomplete + escape sequence. + """ start = response[:10].find('\x1b') if start != -1: response = response[start:] @@ -70,5 +101,8 @@ def clean_response_for_re_match(self, response): return response[:i], response[i:] return response, '' - def init_terminal(self, conn): + def init_terminal(self, conn: 'Protocol') -> None: + """ + Sends an empty command line and waits for the prompt. + """ conn.execute('\r\n') diff --git a/src/Exscript/protocols/drivers/icotera.py b/src/Exscript/protocols/drivers/icotera.py index a20aa1b3..8e3d3264 100644 --- a/src/Exscript/protocols/drivers/icotera.py +++ b/src/Exscript/protocols/drivers/icotera.py @@ -21,7 +21,7 @@ # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ -A driver for Icotera CPE +A driver for Icotera CPE. """ import re @@ -30,9 +30,16 @@ class IcoteraDriver(Driver): - def __init__(self): + """ + The ``icotera`` driver for Icotera CPE. It takes any line that ends with + ">" for a prompt. + """ + + def __init__(self) -> None: """ - Constructor of the IcoteraDriver. + Constructor of the IcoteraDriver. Sets the patterns of the user name + prompt "Username:", the password prompts, the command prompt and the + error message "ERROR". """ Driver.__init__(self, 'icotera') self.user_re = [re.compile(r'user ?name: ?$', re.I)] @@ -40,7 +47,10 @@ def __init__(self): self.prompt_re = [re.compile(r'.*?>\s*$')] self.error_re = [re.compile(r'ERROR')] - def check_head_for_os(self, string): + def check_head_for_os(self, string: str) -> int: + """ + Returns 80 if the data contains "ICOTERA", else 0. + """ if 'ICOTERA' in string: return 80 return 0 diff --git a/src/Exscript/protocols/drivers/ios.py b/src/Exscript/protocols/drivers/ios.py index e7fdb92d..826b0561 100644 --- a/src/Exscript/protocols/drivers/ios.py +++ b/src/Exscript/protocols/drivers/ios.py @@ -25,10 +25,16 @@ """ import re +from typing import TYPE_CHECKING from ..exception import InvalidCommandException from .driver import Driver +if TYPE_CHECKING: + from ...account import Account, AccountProxy + from ...util.impl import _Context + from ..protocol import Protocol + _user_re = [re.compile(r'user ?name: ?$', re.I)] _password_re = [re.compile(r'(?:(?:\r\n|[\r\n])[Pp]assword: ?|last resort password:)$')] _tacacs_re = re.compile(r'(?:(?:\r\n|[\r\n])s|^s)\/key[\S ]+\r?%s' % _password_re[0].pattern) @@ -43,7 +49,20 @@ class IOSDriver(Driver): - def __init__(self): + """ + The ``ios`` driver for Cisco IOS (not IOS XR). Its prompt is a name with + an optional mode in parentheses, followed by ">" or "#", e.g. + "hostname#" or "hostname(config)#". + """ + + def __init__(self) -> None: + """ + Constructor. Sets the patterns of the user name prompt "Username:", + the password prompts, the command prompt and the error messages, such + as "%Error" and "Invalid input". It also makes the SSH adapter + reconnect between the authentication methods, as some IOS devices + accept no further login attempt after a failed one. + """ Driver.__init__(self, 'ios') self.user_re = _user_re self.password_re = _password_re @@ -54,14 +73,23 @@ def __init__(self): # in this hack, we re-connect after each attempt... self.reconnect_between_auth_methods = True - def check_protocol_for_os(self, string): + def check_protocol_for_os(self, string: str) -> int: + """ + Returns 80 if the protocol information, e.g. the SSH version string, + contains "Cisco", else 0. + """ if not string: return 0 if 'Cisco' in string: return 80 return 0 - def check_head_for_os(self, string): + def check_head_for_os(self, string: str) -> int: + """ + Returns 60 if the data contains "User Access Verification", else 50 + if it contains an S/Key challenge before a password prompt, else 30 + if it ends with the user name prompt, else 0. + """ if 'User Access Verification' in string: return 60 if _tacacs_re.search(string): @@ -70,7 +98,12 @@ def check_head_for_os(self, string): return 30 return 0 - def init_terminal(self, conn): + def init_terminal(self, conn: 'Protocol') -> None: + """ + Turns off the pager with "term len 0" and sets the terminal width + with "term width 0". If the device rejects one of them, as the Cisco + ASA does, it executes "term pager 0" instead. + """ # Try the standard term len/width ios commands try: conn.execute('term len 0') @@ -79,6 +112,15 @@ def init_terminal(self, conn): # Deal with corner cases like the Cisco ASA conn.execute('term pager 0') - def auto_authorize(self, conn, account, flush, bailout): + def auto_authorize( + self, + conn: 'Protocol', + account: 'Account | AccountProxy | _Context | None', + flush: bool, + bailout: bool, + ) -> None: + """ + Sends "enable", then authorizes with conn.app_authorize(). + """ conn.send('enable\r') conn.app_authorize(account, flush, bailout) diff --git a/src/Exscript/protocols/drivers/ios_xr.py b/src/Exscript/protocols/drivers/ios_xr.py index 03b11727..b7dfba9c 100644 --- a/src/Exscript/protocols/drivers/ios_xr.py +++ b/src/Exscript/protocols/drivers/ios_xr.py @@ -25,9 +25,13 @@ """ import re +from typing import TYPE_CHECKING from .driver import Driver +if TYPE_CHECKING: + from ..protocol import Protocol + _user_re = [re.compile(r'(?:\r\n|[\r\n])Username: $')] _password_re = [re.compile(r'(?:\r\n|[\r\n])Password: $')] _prompt_re = [ @@ -37,19 +41,37 @@ class IOSXRDriver(Driver): - def __init__(self): + """ + The ``ios_xr`` driver for Cisco IOS XR. It recognizes prompts like + "RP/0/RP0/CPU0:hostname#" and NETCONF capabilities that contain + "Cisco-IOS-XR". + """ + + def __init__(self) -> None: + """ + Constructor. Sets the patterns of the user name prompt "Username:", + the password prompt, the command prompt and the NETCONF capabilities. + """ Driver.__init__(self, 'ios_xr') self.user_re = _user_re self.password_re = _password_re self.prompt_re = _prompt_re self.netconf_capabilities_re = _netconf_caps_re - def check_response_for_os(self, string): + def check_response_for_os(self, string: str) -> int: + """ + Returns 95 if the data ends with an IOS XR prompt, else 0. + """ if _prompt_re[0].search(string): return 95 return 0 - def init_terminal(self, conn): + def init_terminal(self, conn: 'Protocol') -> None: + """ + Turns off the timestamps with "terminal exec prompt no-timestamp", + turns off the pager with "terminal len 0" and sets the terminal width + with "terminal width 0". + """ conn.execute('terminal exec prompt no-timestamp') conn.execute('terminal len 0') conn.execute('terminal width 0') diff --git a/src/Exscript/protocols/drivers/isam.py b/src/Exscript/protocols/drivers/isam.py index dedf14ae..8af2a41e 100644 --- a/src/Exscript/protocols/drivers/isam.py +++ b/src/Exscript/protocols/drivers/isam.py @@ -44,19 +44,38 @@ class IsamDriver(Driver): - def __init__(self): + """ + The ``isam`` driver for devices running ISAM (runs on Alcatel ISAM). Its + prompt is a name with an optional part in parentheses, followed by ">" + or "#", e.g. "hostname>#". + """ + + def __init__(self) -> None: + """ + Constructor. Sets the patterns of the user name prompt "login:", the + password prompt, the command prompt and the error messages, such as + "invalid token" and "command is not complete". + """ Driver.__init__(self, 'isam') self.user_re = _user_re self.password_re = _password_re self.prompt_re = _prompt_re self.error_re = _error_re - def check_response_for_os(self, string): + def check_response_for_os(self, string: str) -> int: + """ + Returns 20 if the data ends with a prompt that ends with ">#", e.g. + "typ:isadmin>#", else 0. + """ if _isam_prompt_re.search(string): return 20 return 0 - def check_head_for_os(self, string): + def check_head_for_os(self, string: str) -> int: + """ + Returns 90 if the data contains "last login : " with a date and a + time, e.g. "last login : 13/05/15 13:49:40", else 0. + """ if _isam_re.search(string): return 90 return 0 diff --git a/src/Exscript/protocols/drivers/junos.py b/src/Exscript/protocols/drivers/junos.py index 5ef1b8d9..0a011149 100644 --- a/src/Exscript/protocols/drivers/junos.py +++ b/src/Exscript/protocols/drivers/junos.py @@ -25,9 +25,13 @@ """ import re +from typing import TYPE_CHECKING from .driver import Driver +if TYPE_CHECKING: + from ..protocol import Protocol + # JunOS prompt examples: # sab@DD-EA3> # @@ -68,7 +72,19 @@ class JunOSDriver(Driver): - def __init__(self): + """ + The ``junos`` driver for devices running JunOS (by Juniper). Its prompt + is "user@host" followed by ">", "%" or "#" and a space, e.g. + "user@router> ", maybe after lines like "{master}" or "[edit]". + """ + + def __init__(self) -> None: + """ + Constructor. Sets the patterns of the user name prompt "login:", the + password prompt, the command prompt, the error messages, which start + with "unknown", "invalid", "error" or "syntax error", and the NETCONF + capabilities. + """ Driver.__init__(self, 'junos') self.user_re = _user_re self.password_re = _password_re @@ -76,14 +92,24 @@ def __init__(self): self.error_re = _error_re self.netconf_capabilities_re = _netconf_caps_re - def check_head_for_os(self, string): + def check_head_for_os(self, string: str) -> int: + """ + Returns 80 if the data contains the word "JUNOS" in any case, else 35 + if it contains a BSD terminal name like "(ttyp0)", which JunOS prints + before "login:", else 0. + """ if _junos_re.search(string): return 80 if _bsd_tty_re.search(string): return 35 return 0 - def init_terminal(self, conn): + def init_terminal(self, conn: 'Protocol') -> None: + """ + Turns off the pager with "set cli screen-length 0", sets the screen + width with "set cli screen-width 0" and the terminal type with + "set cli terminal ansi". + """ conn.execute('set cli screen-length 0') conn.execute('set cli screen-width 0') conn.execute('set cli terminal ansi') diff --git a/src/Exscript/protocols/drivers/junos_erx.py b/src/Exscript/protocols/drivers/junos_erx.py index aca78263..c27071eb 100644 --- a/src/Exscript/protocols/drivers/junos_erx.py +++ b/src/Exscript/protocols/drivers/junos_erx.py @@ -25,9 +25,15 @@ """ import re +from typing import TYPE_CHECKING from .driver import Driver +if TYPE_CHECKING: + from ...account import Account, AccountProxy + from ...util.impl import _Context + from ..protocol import Protocol + _user_re = [re.compile(r'(?:\r\n|[\r\n])User: $')] _password_re = [re.compile(r'(?:\r\n|[\r\n])(Telnet password:|Password:) $')] _prompt_re = [re.compile(r'(?:\r\n|[\r\n])[\-\w+\.:/]+(?:\([^\)]+\))?[>#] ?$')] @@ -35,21 +41,49 @@ class JunOSERXDriver(Driver): - def __init__(self): + """ + The ``junos_erx`` driver for devices running Juniper ERX OS. Its prompt + is a name with an optional mode in parentheses, followed by ">" or "#", + e.g. "hostname-erx>". + """ + + def __init__(self) -> None: + """ + Constructor. Sets the patterns of the user name prompt "User:", the + password prompts "Telnet password:" and "Password:", and the command + prompt. + """ Driver.__init__(self, 'junos_erx') self.user_re = _user_re self.password_re = _password_re self.prompt_re = _prompt_re - def check_head_for_os(self, string): + def check_head_for_os(self, string: str) -> int: + """ + Returns 75 if the data contains "Juniper Networks" in any case, + else 0. + """ if _junos_re.search(string): return 75 return 0 - def init_terminal(self, conn): + def init_terminal(self, conn: 'Protocol') -> None: + """ + Turns off the pager with "terminal length 0" and sets the terminal + width with "terminal width 512". + """ conn.execute('terminal length 0') conn.execute('terminal width 512') - def auto_authorize(self, conn, account, flush, bailout): + def auto_authorize( + self, + conn: 'Protocol', + account: 'Account | AccountProxy | _Context | None', + flush: bool, + bailout: bool, + ) -> None: + """ + Sends "enable 15", then authorizes with conn.app_authorize(). + """ conn.send('enable 15\r') conn.app_authorize(account, flush, bailout) diff --git a/src/Exscript/protocols/drivers/mrv.py b/src/Exscript/protocols/drivers/mrv.py index 3b978a99..78f4edd0 100644 --- a/src/Exscript/protocols/drivers/mrv.py +++ b/src/Exscript/protocols/drivers/mrv.py @@ -25,9 +25,13 @@ """ import re +from typing import TYPE_CHECKING from .driver import Driver +if TYPE_CHECKING: + from ..protocol import Protocol + _user_re = [re.compile(r'(?:\r\n|[\r\n])Authorized Login: $')] _password_re = [re.compile(r'(?:\r\n|[\r\n])Password: $')] _mrv_re = re.compile(r'\bMRV\b') @@ -35,19 +39,36 @@ class MRVDriver(Driver): - def __init__(self): + """ + The ``mrv`` driver for MRV Linux. Its prompt is a line that ends with + "$", e.g. "hostname$ ". + """ + + def __init__(self) -> None: + """ + Constructor. Sets the patterns of the user name prompt + "Authorized Login:", the password prompt and the command prompt. + """ Driver.__init__(self, 'mrv') self.user_re = _user_re self.password_re = _password_re self.prompt_re = _prompt_re - def check_head_for_os(self, string): + def check_head_for_os(self, string: str) -> int: + """ + Returns 20 if the data ends with the user name prompt + "Authorized Login: ", else 75 if it contains the word "MRV", else 0. + """ if _user_re[0].search(string): return 20 if _mrv_re.search(string): return 75 return 0 - def init_terminal(self, conn): + def init_terminal(self, conn: 'Protocol') -> None: + """ + Turns off the pager with "terminal length 0" and sets the terminal + width with "terminal width 512". + """ conn.execute('terminal length 0') conn.execute('terminal width 512') diff --git a/src/Exscript/protocols/drivers/nxos.py b/src/Exscript/protocols/drivers/nxos.py index ec925dfa..6cb85222 100644 --- a/src/Exscript/protocols/drivers/nxos.py +++ b/src/Exscript/protocols/drivers/nxos.py @@ -25,9 +25,15 @@ """ import re +from typing import TYPE_CHECKING from .driver import Driver +if TYPE_CHECKING: + from ...account import Account, AccountProxy + from ...util.impl import _Context + from ..protocol import Protocol + _user_re = [re.compile(r'user ?name: ?$', re.I)] _password_re = [re.compile(r'(?:(?:\r\n|[\r\n])Password: ?|last resort password:)$')] _prompt_re = [re.compile(r'(?:\r\n\x00?|[\r\n\x00])[\-\w+\.:/]+(?:\([^\)]+\))?[>#] ?$')] @@ -41,20 +47,47 @@ class NXOSDriver(Driver): - def __init__(self): + """ + The ``nxos`` driver for Cisco Nexus OS (NXOS). Its prompt is a name with + an optional mode in parentheses, followed by ">" or "#", e.g. + "hostname#". + """ + + def __init__(self) -> None: + """ + Constructor. Sets the patterns of the user name prompt "Username:", + the password prompts, the command prompt and the error messages, such + as "%Error" and "Invalid input". + """ Driver.__init__(self, 'nxos') self.user_re = _user_re self.password_re = _password_re self.prompt_re = _prompt_re self.error_re = _error_re - def check_head_for_os(self, string): + def check_head_for_os(self, string: str) -> int: + """ + Returns 95 if the data contains + "Cisco Nexus Operating System (NX-OS) Software", else 0. + """ if 'Cisco Nexus Operating System (NX-OS) Software' in string: return 95 return 0 - def init_terminal(self, conn): + def init_terminal(self, conn: 'Protocol') -> None: + """ + Turns off the pager with "term len 0". + """ conn.execute('term len 0') - def auto_authorize(self, conn, account, flush, bailout): + def auto_authorize( + self, + conn: 'Protocol', + account: 'Account | AccountProxy | _Context | None', + flush: bool, + bailout: bool, + ) -> None: + """ + Does nothing, i.e. skips the authorization. + """ pass diff --git a/src/Exscript/protocols/drivers/one_os.py b/src/Exscript/protocols/drivers/one_os.py index 157d644e..158762e8 100644 --- a/src/Exscript/protocols/drivers/one_os.py +++ b/src/Exscript/protocols/drivers/one_os.py @@ -25,9 +25,15 @@ """ import re +from typing import TYPE_CHECKING from .driver import Driver +if TYPE_CHECKING: + from ...account import Account, AccountProxy + from ...util.impl import _Context + from ..protocol import Protocol + _user_re = [re.compile(r'(?:\r\n|[\r\n])Username: ?$')] _password_re = [re.compile(r'(?:\r\n|[\r\n])Password: ?$')] _first_prompt_re = re.compile(r'\r?\n\r?\n[\-\w+\.]+[>#]$') @@ -35,18 +41,36 @@ class OneOSDriver(Driver): - def __init__(self): + """ + The ``one_os`` driver for OneOS (OneAccess). Its prompt is a name with an + optional mode in parentheses, followed by ">" or "#", e.g. "hostname#". + """ + + def __init__(self) -> None: + """ + Constructor. Sets the patterns of the user name prompt "Username:", + the password prompt and the command prompt. + """ Driver.__init__(self, 'one_os') self.user_re = _user_re self.password_re = _password_re self.prompt_re = _prompt_re - def check_head_for_os(self, string): + def check_head_for_os(self, string: str) -> int: + """ + Returns 40 if the data ends with an empty line and a prompt like + "hostname>", else 0. + """ if _first_prompt_re.search(string): return 40 return 0 - def init_terminal(self, conn): + def init_terminal(self, conn: 'Protocol') -> None: + """ + Turns off the pager with "term len 0" and sets the terminal width + with "stty columns 255" for OneOS 5; it ignores the errors of the + latter. + """ conn.execute('term len 0') # TERMINAL WIDTH ONEOS5 try: @@ -54,6 +78,15 @@ def init_terminal(self, conn): except Exception: # noqa: BLE001, S110 pass - def auto_authorize(self, conn, account, flush, bailout): + def auto_authorize( + self, + conn: 'Protocol', + account: 'Account | AccountProxy | _Context | None', + flush: bool, + bailout: bool, + ) -> None: + """ + Sends "enable", then authorizes with conn.app_authorize(). + """ conn.send('enable\r') conn.app_authorize(account, flush, bailout) diff --git a/src/Exscript/protocols/drivers/rios.py b/src/Exscript/protocols/drivers/rios.py index a6a3e150..903681e2 100644 --- a/src/Exscript/protocols/drivers/rios.py +++ b/src/Exscript/protocols/drivers/rios.py @@ -25,9 +25,15 @@ """ import re +from typing import TYPE_CHECKING from .driver import Driver +if TYPE_CHECKING: + from ...account import Account, AccountProxy + from ...util.impl import _Context + from ..protocol import Protocol + _user_re = [re.compile(r'user ?name: ?$', re.I)] _password_re = [re.compile(r'(?:(?:\r\n|[\r\n])Password: ?|last resort password:)$')] _prompt_re = [re.compile(r'(?:\r\n|[\r\n])[\-\w+\.:/]+\s?(?:\([^\)]+\))?\s+[>#]\s+$')] @@ -39,16 +45,40 @@ class RIOSDriver(Driver): - def __init__(self): + """ + The ``rios`` driver for Riverbed RIOS. Its prompt is a name with an + optional mode in parentheses, followed by ">" or "#" with spaces before + and after it, e.g. "hostname > ". The OS guesser does not recognize RIOS. + """ + + def __init__(self) -> None: + """ + Constructor. Sets the patterns of the user name prompt "Username:", + the password prompts, the command prompt and the error messages, such + as "%Error" and "% Unrecognized". + """ Driver.__init__(self, 'rios') self.user_re = _user_re self.password_re = _password_re self.prompt_re = _prompt_re self.error_re = _error_re - def init_terminal(self, conn): + def init_terminal(self, conn: 'Protocol') -> None: + """ + Turns off the pager with "term len 0" and executes "enable". + """ conn.execute('term len 0') conn.execute('enable') - def auto_authorize(self, conn, account, flush, bailout): + def auto_authorize( + self, + conn: 'Protocol', + account: 'Account | AccountProxy | _Context | None', + flush: bool, + bailout: bool, + ) -> None: + """ + Does nothing, i.e. skips the authorization; init_terminal() executes + "enable". + """ pass diff --git a/src/Exscript/protocols/drivers/shell.py b/src/Exscript/protocols/drivers/shell.py index 85476f79..b46041b2 100644 --- a/src/Exscript/protocols/drivers/shell.py +++ b/src/Exscript/protocols/drivers/shell.py @@ -42,19 +42,37 @@ class ShellDriver(Driver): - def __init__(self): + """ + The ``shell`` driver, a generic driver for Unix shells. It uses the + generic command prompt pattern. + """ + + def __init__(self) -> None: + """ + Constructor. Sets the patterns of the user name prompts "user:" and + "login:" and of the password prompt. + """ Driver.__init__(self, 'shell') self.user_re = _user_re self.password_re = _password_re - def check_head_for_os(self, string): + def check_head_for_os(self, string: str) -> int: + """ + Returns 70 if the data contains the word "Linux" in any case, else 20 + if it ends with "user: " or "login: ", but not with "Last login: ", + else 0. + """ if _linux_re.search(string): return 70 if _login_prompt_re.search(string): return 20 return 0 - def check_response_for_os(self, string): + def check_response_for_os(self, string: str) -> int: + """ + Returns 30 if the data ends with a shell prompt like "router1:~$ " or + "admin@router1:/etc# ", maybe after escape sequences, else 0. + """ if _shell_prompt_re.search(string): return 30 return 0 diff --git a/src/Exscript/protocols/drivers/smart_edge_os.py b/src/Exscript/protocols/drivers/smart_edge_os.py index dde18c30..e126f43f 100644 --- a/src/Exscript/protocols/drivers/smart_edge_os.py +++ b/src/Exscript/protocols/drivers/smart_edge_os.py @@ -25,9 +25,15 @@ """ import re +from typing import TYPE_CHECKING from .driver import Driver +if TYPE_CHECKING: + from ...account import Account, AccountProxy + from ...util.impl import _Context + from ..protocol import Protocol + _user_re = [re.compile(r'(?:\r\n|[\r\n])login: ')] _password_re = [re.compile(r'(?:\r\n|[\r\n])Password: $')] _prompt_re = [ @@ -37,23 +43,50 @@ class SmartEdgeOSDriver(Driver): - def __init__(self): + """ + The ``smart_edge_os`` driver for Redback Smart Edge OS. Its prompt starts + with a name in brackets and ends with ">" or "#", e.g. + "[local]hostname#". + """ + + def __init__(self) -> None: + """ + Constructor. Sets the patterns of the user name prompt "login:", the + password prompt and the command prompt. + """ Driver.__init__(self, 'smart_edge_os') self.user_re = _user_re self.password_re = _password_re self.prompt_re = _prompt_re - def check_head_for_os(self, string): + def check_head_for_os(self, string: str) -> int: + """ + Returns 60 if the data contains a line that ends with "-se800", else + 20 if it contains the user name prompt "login: ", else 0. + """ if _model_re.search(string): return 60 if self.user_re[0].search(string): return 20 return 0 - def init_terminal(self, conn): + def init_terminal(self, conn: 'Protocol') -> None: + """ + Turns off the pager with "terminal length 0" and sets the terminal + width with "terminal width 65536". + """ conn.execute('terminal length 0') conn.execute('terminal width 65536') - def auto_authorize(self, conn, account, flush, bailout): + def auto_authorize( + self, + conn: 'Protocol', + account: 'Account | AccountProxy | _Context | None', + flush: bool, + bailout: bool, + ) -> None: + """ + Sends "enable", then authorizes with conn.app_authorize(). + """ conn.send('enable\r') conn.app_authorize(account, flush, bailout) diff --git a/src/Exscript/protocols/drivers/sp_vxoa.py b/src/Exscript/protocols/drivers/sp_vxoa.py index e670e447..f1635992 100644 --- a/src/Exscript/protocols/drivers/sp_vxoa.py +++ b/src/Exscript/protocols/drivers/sp_vxoa.py @@ -25,9 +25,15 @@ """ import re +from typing import TYPE_CHECKING from .driver import Driver +if TYPE_CHECKING: + from ...account import Account, AccountProxy + from ...util.impl import _Context + from ..protocol import Protocol + _user_re = [re.compile(r'(?:\r\n|[\r\n])login ?:as :')] _password_re = [re.compile(r'(?:\r\n|[\r\n])[Pp]assword: ?$')] _prompt_re = [re.compile(r'(?:\r\n|[\r\n]).*?[>#] ?$')] @@ -36,21 +42,47 @@ class SilverPeakVXOADriver(Driver): - def __init__(self): + """ + The ``sp_vxoa`` driver for devices running VXOA (by Silver Peak). It takes + any line that ends with ">" or "#" for a prompt, e.g. "hostname >". + """ + + def __init__(self) -> None: + """ + Constructor. Sets the patterns of the user name, password and command + prompts and of the error messages, which start with "unknown", + "invalid", "error" or "syntax error". + """ Driver.__init__(self, 'sp_vxoa') self.user_re = _user_re self.password_re = _password_re self.prompt_re = _prompt_re self.error_re = _error_re - def check_head_for_os(self, string): + def check_head_for_os(self, string: str) -> int: + """ + Returns 90 if the data contains "Silver Peak Systems Inc.", also with + a line break between the words, else 0. + """ if _banner_re.search(string): return 90 return 0 - def init_terminal(self, conn): + def init_terminal(self, conn: 'Protocol') -> None: + """ + Sets the terminal length with "terminal length 65536". + """ conn.execute('terminal length 65536') - def auto_authorize(self, conn, account, flush, bailout): + def auto_authorize( + self, + conn: 'Protocol', + account: 'Account | AccountProxy | _Context | None', + flush: bool, + bailout: bool, + ) -> None: + """ + Sends "enable", then authorizes with conn.app_authorize(). + """ conn.send('enable\r') conn.app_authorize(account, flush, bailout) diff --git a/src/Exscript/protocols/drivers/sros.py b/src/Exscript/protocols/drivers/sros.py index 8551901f..a7e56694 100644 --- a/src/Exscript/protocols/drivers/sros.py +++ b/src/Exscript/protocols/drivers/sros.py @@ -25,26 +25,48 @@ """ import re +from typing import TYPE_CHECKING from .driver import Driver +if TYPE_CHECKING: + from ..protocol import Protocol + _prompt_re = [re.compile(r'(?:\r\n|[\r\n])[*]?(?:A|B):[\-\w\.>]+[#\$] ?$')] class SROSDriver(Driver): - def __init__(self): + """ + The ``sros`` driver for Alcatel SROS. Its prompt starts with "A:" or + "B:", maybe after "*", and ends with "#" or "$", e.g. "A:hostname# ". + """ + + def __init__(self) -> None: + """ + Constructor. Sets the pattern of the command prompt. + """ Driver.__init__(self, 'sros') self.prompt_re = _prompt_re - def check_head_for_os(self, string): + def check_head_for_os(self, string: str) -> int: + """ + Returns 95 if the data ends with an SROS prompt, else 0. + """ if _prompt_re[0].search(string): return 95 return 0 - def init_terminal(self, conn): + def init_terminal(self, conn: 'Protocol') -> None: + """ + Turns off the pager with "environment no more", and executes + "environment reduced-prompt 2" and "environment no saved-ind-prompt". + """ conn.execute('environment no more') conn.execute('environment reduced-prompt 2') conn.execute('environment no saved-ind-prompt') - def logout(self, conn): + def logout(self, conn: 'Protocol') -> None: + """ + Ends the session with "logout". + """ conn.send('logout\r') diff --git a/src/Exscript/protocols/drivers/vrp.py b/src/Exscript/protocols/drivers/vrp.py index e20ca144..e6d7f279 100644 --- a/src/Exscript/protocols/drivers/vrp.py +++ b/src/Exscript/protocols/drivers/vrp.py @@ -25,9 +25,13 @@ """ import re +from typing import TYPE_CHECKING from .driver import Driver +if TYPE_CHECKING: + from ..protocol import Protocol + _user_re = [re.compile(r'user ?name: ', re.I)] _password_re = [re.compile(r'(?:\r\n|[\r\n])Password: $')] _prompt_re = [re.compile(r'(?:\r\n|[\r\n])[\<\[]\S+[\>\]] ?$', re.I)] @@ -38,13 +42,27 @@ class VRPDriver(Driver): - def __init__(self): + """ + The ``vrp`` driver for devices running VRP (by Huawei). Its prompt is a + name in angle or square brackets, e.g. "" or "[hostname]". + """ + + def __init__(self) -> None: + """ + Constructor. Sets the patterns of the user name prompt "Username:", + the password prompt and the command prompt. + """ Driver.__init__(self, 'vrp') self.user_re = _user_re self.password_re = _password_re self.prompt_re = _prompt_re - def check_head_for_os(self, string): + def check_head_for_os(self, string: str) -> int: + """ + Returns 80 if the data contains the word "Huawei" in any case, else 70 + if it contains "The current login time is", else 60 if it ends with + a VRP prompt, else 0. + """ if _huawei_re[0].search(string): return 80 elif _huawei_re[1].search(string): @@ -53,8 +71,14 @@ def check_head_for_os(self, string): return 60 return 0 - def init_terminal(self, conn): + def init_terminal(self, conn: 'Protocol') -> None: + """ + Turns off the pager with "screen-length 0 temporary". + """ conn.execute('screen-length 0 temporary') - def logout(self, conn): + def logout(self, conn: 'Protocol') -> None: + """ + Ends the session with "quit". + """ conn.send('quit\r') From 7ef0aa7305a268bba95bd71ec1e94b3061ac0e6f Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 10 Nov 2026 15:37:28 +0100 Subject: [PATCH 071/250] Annotate the functions of the template language They take the scope and lists of template values, and return lists. Apart from four parameters that are no longer reassigned with another type, only annotations, docstrings and typing.cast() are added. --- src/Exscript/stdlib/__init__.py | 22 +++++- src/Exscript/stdlib/connection.py | 126 +++++++++++++++++------------- src/Exscript/stdlib/crypt.py | 12 ++- src/Exscript/stdlib/file.py | 42 ++++++---- src/Exscript/stdlib/ipv4.py | 45 +++++++---- src/Exscript/stdlib/list.py | 30 +++++-- src/Exscript/stdlib/mysys.py | 28 +++++-- src/Exscript/stdlib/string.py | 22 ++++-- src/Exscript/stdlib/util.py | 21 ++++- 9 files changed, 236 insertions(+), 112 deletions(-) diff --git a/src/Exscript/stdlib/__init__.py b/src/Exscript/stdlib/__init__.py index 3268cf5c..a3c698fd 100644 --- a/src/Exscript/stdlib/__init__.py +++ b/src/Exscript/stdlib/__init__.py @@ -20,9 +20,29 @@ # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" +The standard library of the template language: the functions that templates +call, such as ``connection.sendline()``, ``list.length()`` or ``sys.exec()``. + +``functions`` maps the name that a template uses to the function. The +interpreter calls the function with the scope of the call, followed by the +values of the arguments. Template values are lists: a string argument, for +example, arrives as a list with one string. A regular expression, such as +``/[Ff]ailed/``, arrives compiled. Where the documentation of a function gives +the type of an argument or of the result, such as string, it means the type +of the items. The functions return a list, or True if they have no result. + +Functions that run local commands or access local files are not secure (see +:func:`Exscript.stdlib.util.secure_function`): a template that is compiled +in secure mode cannot call them. +""" + +from collections.abc import Callable +from typing import Any + from . import connection, crypt, file, ipv4, list, mysys, string -functions = { +functions: dict[str, Callable[..., Any]] = { 'connection.add_error': connection.add_error, 'connection.add_prompt': connection.add_prompt, 'connection.authenticate': connection.authenticate, diff --git a/src/Exscript/stdlib/connection.py b/src/Exscript/stdlib/connection.py index 63652cb1..5d6b74d7 100644 --- a/src/Exscript/stdlib/connection.py +++ b/src/Exscript/stdlib/connection.py @@ -20,12 +20,24 @@ # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" +The functions of the template language that use the connection, such as +``connection.exec()`` or ``connection.set_prompt()``. +""" + +import re +from typing import TYPE_CHECKING + from ..account import Account from .util import secure_function +if TYPE_CHECKING: + from ..interpreter.scope import Scope + from ..protocols import Protocol + @secure_function -def authenticate(scope): +def authenticate(scope: 'Scope') -> bool: """ Looks for any username/password prompts on the current connection and logs in using the login information that was passed to Exscript. @@ -35,7 +47,9 @@ def authenticate(scope): @secure_function -def authenticate_user(scope, user=[None], password=[None]): +def authenticate_user( + scope: 'Scope', user: list[str | None] = [None], password: list[str | None] = [None] +) -> bool: """ Like authenticate(), but logs in using the given user and password. If a user and password are not given, the function uses the same @@ -47,18 +61,18 @@ def authenticate_user(scope, user=[None], password=[None]): :type password: string :param password: A password. """ - conn = scope.get('__connection__') - user = user[0] - if user is None: + conn: Protocol = scope.get('__connection__') + username = user[0] + if username is None: conn.app_authenticate() else: - account = Account(user, password[0]) + account = Account(username, password[0]) conn.app_authenticate(account) return True @secure_function -def authorize(scope, password=[None]): +def authorize(scope: 'Scope', password: list[str | None] = [None]) -> bool: """ Looks for a password prompt on the current connection and enters the given password. @@ -69,45 +83,43 @@ def authorize(scope, password=[None]): :type password: string :param password: A password. """ - conn = scope.get('__connection__') - password = password[0] - if password is None: + conn: Protocol = scope.get('__connection__') + secret = password[0] + if secret is None: conn.app_authorize() else: - account = Account('', password) + account = Account('', secret) conn.app_authorize(account) return True @secure_function -def auto_authorize(scope, password=[None]): +def auto_authorize(scope: 'Scope', password: list[str | None] = [None]) -> bool: """ Executes a command on the remote host that causes an authorization procedure to be started, then authorizes using the given password in the same way in which authorize() works. - Depending on the detected operating system of the remote host the - following commands are started: - - - on IOS, the "enable" command is executed. - - nothing on other operating systems yet. + The command depends on the driver of the connection; on IOS, for + example, it is "enable". Drivers that have no such command only + authorize, like authorize(). :type password: string :param password: A password. """ - conn = scope.get('__connection__') - password = password[0] - if password is None: + conn: Protocol = scope.get('__connection__') + secret = password[0] + if secret is None: conn.auto_app_authorize() else: - account = Account('', password) + account = Account('', secret) conn.auto_app_authorize(account) return True @secure_function -def autoinit(scope): +def autoinit(scope: 'Scope') -> bool: """ - Make the remote host more script-friendly by automatically executing + Makes the remote host more script-friendly by automatically executing one or more commands on it. The commands executed depend on the currently used driver. For example, the driver for Cisco IOS would execute the @@ -121,7 +133,7 @@ def autoinit(scope): @secure_function -def close(scope, force=[False]): +def close(scope: 'Scope', force: list[bool] = [False]) -> bool: """ Closes the existing connection with the remote host gracefully, sending the driver-specific logout command first. This function is rarely used @@ -131,14 +143,14 @@ def close(scope, force=[False]): :type force: boolean :param force: Whether the connection is closed without logout. """ - conn = scope.get('__connection__') + conn: Protocol = scope.get('__connection__') conn.close(force=force[0]) scope.define(__response__=conn.get_response().split('\n')) return True @secure_function -def exec_(scope, data): +def exec_(scope: 'Scope', data: list[str]) -> bool: """ Sends the given data to the remote host and waits until the host has responded with a prompt. @@ -151,7 +163,7 @@ def exec_(scope, data): :type data: string :param data: The data that is sent. """ - conn = scope.get('__connection__') + conn: Protocol = scope.get('__connection__') response = [] for line in data: conn.send(line) @@ -162,14 +174,14 @@ def exec_(scope, data): @secure_function -def execline(scope, data): +def execline(scope: 'Scope', data: list[str]) -> bool: """ Like exec(), but appends a newline to the data before sending it. :type data: string :param data: The data that is sent. """ - conn = scope.get('__connection__') + conn: Protocol = scope.get('__connection__') response = [] for line in data: conn.send(line + '\r') @@ -180,7 +192,7 @@ def execline(scope, data): @secure_function -def get_protocol(scope): +def get_protocol(scope: 'Scope') -> list[str]: """ Returns the name of the connection protocol. @@ -190,12 +202,12 @@ def get_protocol(scope): :rtype: string :return: The connection protocol. """ - conn = scope.get('__connection__') + conn: Protocol = scope.get('__connection__') return [conn.get_protocol()] @secure_function -def guess_os(scope): +def guess_os(scope: 'Scope') -> list[str]: """ Guesses the operating system of the connected host. @@ -206,12 +218,12 @@ def guess_os(scope): :rtype: string :return: The operating system. """ - conn = scope.get('__connection__') + conn: Protocol = scope.get('__connection__') return [conn.guess_os()] @secure_function -def send(scope, data): +def send(scope: 'Scope', data: list[str]) -> bool: """ Like exec(), but does not wait for a response of the remote host after sending the data. @@ -219,7 +231,7 @@ def send(scope, data): :type data: string :param data: The data that is sent. """ - conn = scope.get('__connection__') + conn: Protocol = scope.get('__connection__') for line in data: conn.send(line) scope.define(__response__=conn.get_response().split('\n')) @@ -227,7 +239,7 @@ def send(scope, data): @secure_function -def sendline(scope, data): +def sendline(scope: 'Scope', data: list[str]) -> bool: """ Like execline(), but does not wait for a response of the remote host after sending the data. @@ -235,7 +247,7 @@ def sendline(scope, data): :type data: string :param data: The data that is sent. """ - conn = scope.get('__connection__') + conn: Protocol = scope.get('__connection__') for line in data: conn.send(line + '\r') scope.define(__response__=conn.get_response().split('\n')) @@ -243,23 +255,23 @@ def sendline(scope, data): @secure_function -def expect(scope, prompt): +def expect(scope: 'Scope', prompt: re.Pattern[str] | list[str | re.Pattern[str]]) -> bool: """ Waits until the response of the remote host contains the given pattern. - Does consume the prompt, can be run only once, then it fails with a timeouts. + Does consume the prompt, can be run only once, then it fails with a timeout. :type prompt: regex :param prompt: The prompt pattern. """ - conn = scope.get('__connection__') + conn: Protocol = scope.get('__connection__') conn.expect(prompt) scope.define(__response__=conn.get_response().split('\n')) return True @secure_function -def wait_for(scope, prompt): +def wait_for(scope: 'Scope', prompt: re.Pattern[str] | list[str | re.Pattern[str]]) -> bool: """ Waits until the response of the remote host contains the given pattern. @@ -268,14 +280,16 @@ def wait_for(scope, prompt): :type prompt: regex :param prompt: The prompt pattern. """ - conn = scope.get('__connection__') + conn: Protocol = scope.get('__connection__') conn.waitfor(prompt) scope.define(__response__=conn.get_response().split('\n')) return True @secure_function -def set_prompt(scope, prompt=None): +def set_prompt( + scope: 'Scope', prompt: re.Pattern[str] | list[str | re.Pattern[str]] | None = None +) -> bool: """ Defines the pattern that is recognized at any future time when Exscript needs to wait for a prompt. @@ -285,18 +299,19 @@ def set_prompt(scope, prompt=None): Exscript waits for a prompt whenever it sends a command (unless the send() method was used). set_prompt() redefines as to what is recognized as a - prompt. + prompt. Without a pattern, the default pattern of the driver is used + again. :type prompt: regex :param prompt: The prompt pattern. """ - conn = scope.get('__connection__') + conn: Protocol = scope.get('__connection__') conn.set_prompt(prompt, extend=False) return True @secure_function -def add_prompt(scope, prompt): +def add_prompt(scope: 'Scope', prompt: re.Pattern[str] | list[str | re.Pattern[str]]) -> bool: """ Extends the current list of patterns, that are recognized at any future time when Exscript needs to wait for a prompt, each time when called. Use @@ -305,31 +320,34 @@ def add_prompt(scope, prompt): :type prompt: regex :param prompt: The additional prompt pattern. """ - conn = scope.get('__connection__') + conn: Protocol = scope.get('__connection__') conn.set_prompt(prompt, extend=True) return True @secure_function -def set_error(scope, error_re=None): +def set_error( + scope: 'Scope', error_re: re.Pattern[str] | list[str | re.Pattern[str]] | None = None +) -> bool: """ Defines a pattern that, whenever detected in the response of the remote host, causes an error to be raised. In other words, whenever Exscript waits for a prompt, it searches the response of the host for the given pattern and raises an error if the - pattern is found. + pattern is found. Without a pattern, the default pattern of the driver + is used again. :type error_re: regex :param error_re: The error pattern. """ - conn = scope.get('__connection__') + conn: Protocol = scope.get('__connection__') conn.set_error_prompt(error_re, extend=False) return True @secure_function -def add_error(scope, error_re): +def add_error(scope: 'Scope', error_re: re.Pattern[str] | list[str | re.Pattern[str]]) -> bool: """ Extends the list of patterns, that, whenever detected in the response of the remote host, cause an error to be raised. Use set_error() instead to replace @@ -338,13 +356,13 @@ def add_error(scope, error_re): :type error_re: regex :param error_re: The additional error pattern. """ - conn = scope.get('__connection__') + conn: Protocol = scope.get('__connection__') conn.set_error_prompt(error_re, extend=True) return True @secure_function -def set_timeout(scope, timeout): +def set_timeout(scope: 'Scope', timeout: list[int | str]) -> bool: """ Defines the time after which Exscript fails if it does not receive a prompt from the remote host. @@ -352,6 +370,6 @@ def set_timeout(scope, timeout): :type timeout: int :param timeout: The timeout in seconds. """ - conn = scope.get('__connection__') + conn: Protocol = scope.get('__connection__') conn.set_timeout(int(timeout[0])) return True diff --git a/src/Exscript/stdlib/crypt.py b/src/Exscript/stdlib/crypt.py index 3b2f6095..41d84023 100644 --- a/src/Exscript/stdlib/crypt.py +++ b/src/Exscript/stdlib/crypt.py @@ -20,12 +20,22 @@ # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" +The cryptographic functions of the template language, such as +``crypt.otp()``. +""" + +from typing import TYPE_CHECKING + from ..util import crypt from .util import secure_function +if TYPE_CHECKING: + from ..interpreter.scope import Scope + @secure_function -def otp(scope, password, seed, seqs): +def otp(scope: 'Scope', password: list[str], seed: list[str], seqs: list[int | str]) -> list[str]: """ Calculates a one-time password hash using the given password, seed, and sequence number and returns it. diff --git a/src/Exscript/stdlib/file.py b/src/Exscript/stdlib/file.py index b9b22c8b..b0735d6a 100644 --- a/src/Exscript/stdlib/file.py +++ b/src/Exscript/stdlib/file.py @@ -20,12 +20,23 @@ # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" +The functions of the template language for local files, such as +``file.read()`` or ``file.write()``. Except for ``file.exists()``, they are +not secure: a template that is compiled in secure mode cannot call them +(see :func:`Exscript.stdlib.util.secure_function`). +""" + import os +from typing import TYPE_CHECKING from Exscript.stdlib.util import secure_function +if TYPE_CHECKING: + from Exscript.interpreter.scope import Scope + -def chmod(scope, filename, mode): +def chmod(scope: 'Scope', filename: list[str], mode: list[int]) -> bool: """ Changes the permissions of the given file (or list of files) to the given mode. You probably want to use an octal representation @@ -41,9 +52,9 @@ def chmod(scope, filename, mode): return True -def clear(scope, filename): +def clear(scope: 'Scope', filename: list[str]) -> bool: """ - Clear the contents of the given file. The file is created if it does + Clears the contents of the given file. The file is created if it does not exist. :type filename: string @@ -55,28 +66,27 @@ def clear(scope, filename): @secure_function -def exists(scope, filename): +def exists(scope: 'Scope', filename: list[str]) -> list[bool]: """ - Returns True if the file with the given name exists, False otherwise. - If a list of files is given, the function returns True only if ALL of - the files exist. + Returns, for each of the given files, True if the file exists and False + otherwise. :type filename: string - :param filename: A filename. + :param filename: A filename, or a list of filenames. :rtype: bool - :return: The operating system of the remote device. + :return: True or False for each file. """ return [os.path.exists(f) for f in filename] -def mkdir(scope, dirname, mode=None): +def mkdir(scope: 'Scope', dirname: list[str], mode: list[int] | None = None) -> bool: """ Creates the given directory (or directories). The optional access permissions are set to the given mode, and default to whatever is the umask on your system defined. :type dirname: string - :param dirname: A filename, or a list of dirnames. + :param dirname: A directory name, or a list of directory names. :type mode: int :param mode: The access permissions. """ @@ -88,7 +98,7 @@ def mkdir(scope, dirname, mode=None): return True -def read(scope, filename): +def read(scope: 'Scope', filename: list[str]) -> list[str]: """ Reads the given file and returns the result. The result is also stored in the built-in __response__ variable. @@ -96,7 +106,7 @@ def read(scope, filename): :type filename: string :param filename: A filename. :rtype: string - :return: The content of the file. + :return: The content of the file, one line per item. """ with open(filename[0], encoding='utf-8') as fp: lines = fp.readlines() @@ -104,7 +114,7 @@ def read(scope, filename): return lines -def rm(scope, filename): +def rm(scope: 'Scope', filename: list[str]) -> bool: """ Deletes the given file (or files) from the file system. @@ -116,7 +126,7 @@ def rm(scope, filename): return True -def write(scope, filename, lines, mode=['a']): +def write(scope: 'Scope', filename: list[str], lines: list[str], mode: list[str] = ['a']) -> bool: """ Writes the given string into the given file. The following modes are supported: @@ -124,6 +134,8 @@ def write(scope, filename, lines, mode=['a']): - 'a': Append to the file if it already exists. - 'w': Replace the file if it already exists. + Each line is written without trailing whitespace, followed by a newline. + :type filename: string :param filename: A filename. :type lines: string diff --git a/src/Exscript/stdlib/ipv4.py b/src/Exscript/stdlib/ipv4.py index 37bb2bd4..26e5f314 100644 --- a/src/Exscript/stdlib/ipv4.py +++ b/src/Exscript/stdlib/ipv4.py @@ -20,12 +20,24 @@ # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" +The functions of the template language for IPv4 addresses, such as +``ipv4.in_network()`` or ``ipv4.mask2pfxlen()``. +""" + +from typing import TYPE_CHECKING + from Exscript.stdlib.util import secure_function from Exscript.util import ipv4 +if TYPE_CHECKING: + from Exscript.interpreter.scope import Scope + @secure_function -def in_network(scope, prefixes, destination, default_pfxlen=[24]): +def in_network( + scope: 'Scope', prefixes: list[str], destination: list[str], default_pfxlen: list[int] = [24] +) -> list[bool]: """ Returns True if the given destination is in the network range that is defined by the given prefix (e.g. 10.0.0.1/22). If the given prefix @@ -42,7 +54,7 @@ def in_network(scope, prefixes, destination, default_pfxlen=[24]): :param destination: An IP address. :type default_pfxlen: int :param default_pfxlen: The default prefix length. - :rtype: True + :rtype: bool :return: Whether the given destination is in the given network. """ needle = ipv4.ip2int(destination[0]) @@ -55,37 +67,37 @@ def in_network(scope, prefixes, destination, default_pfxlen=[24]): @secure_function -def mask(scope, ips, mask): +def mask(scope: 'Scope', ips: list[str], mask: list[str]) -> list[str]: """ Applies the given IP mask (e.g. 255.255.255.0) to the given IP address (or list of IP addresses) and returns it. :type ips: string - :param ips: A prefix, or a list of IP prefixes. + :param ips: An IP address, or a list of IP addresses. :type mask: string :param mask: An IP mask. :rtype: string :return: The network(s) that result(s) from applying the mask. """ - mask = ipv4.ip2int(mask[0]) - return [ipv4.int2ip(ipv4.ip2int(ip) & mask) for ip in ips] + mask_int = ipv4.ip2int(mask[0]) + return [ipv4.int2ip(ipv4.ip2int(ip) & mask_int) for ip in ips] @secure_function -def mask2pfxlen(scope, masks): +def mask2pfxlen(scope: 'Scope', masks: list[str]) -> list[int]: """ Converts the given IP mask(s) (e.g. 255.255.255.0) to prefix length(s). :type masks: string :param masks: An IP mask, or a list of masks. - :rtype: string + :rtype: int :return: The prefix length(s) that result(s) from converting the mask. """ return [ipv4.mask2pfxlen(mask) for mask in masks] @secure_function -def pfxlen2mask(scope, pfxlen): +def pfxlen2mask(scope: 'Scope', pfxlen: list[int]) -> list[str]: """ Converts the given prefix length(s) (e.g. 30) to IP mask(s). @@ -98,33 +110,33 @@ def pfxlen2mask(scope, pfxlen): @secure_function -def network(scope, prefixes): +def network(scope: 'Scope', prefixes: list[str]) -> list[str]: """ Given a prefix, this function returns the corresponding network address. :type prefixes: string :param prefixes: An IP prefix. :rtype: string - :return: The network address(es) of the prefix length(s). + :return: The network address(es) of the prefix(es). """ return [ipv4.network(pfx) for pfx in prefixes] @secure_function -def broadcast(scope, prefixes): +def broadcast(scope: 'Scope', prefixes: list[str]) -> list[str]: """ Given a prefix, this function returns the corresponding broadcast address. :type prefixes: string :param prefixes: An IP prefix. :rtype: string - :return: The broadcast address(es) of the prefix length(s). + :return: The broadcast address(es) of the prefix(es). """ return [ipv4.broadcast(pfx) for pfx in prefixes] @secure_function -def pfxmask(scope, ips, pfxlen): +def pfxmask(scope: 'Scope', ips: list[str], pfxlen: list[int]) -> list[str]: """ Applies the given prefix length to the given ips, resulting in a (list of) IP network addresses. @@ -134,14 +146,15 @@ def pfxmask(scope, ips, pfxlen): :type pfxlen: int :param pfxlen: An IP prefix length. :rtype: string - :return: The mask(s) that result(s) from converting the prefix length. + :return: The network address(es) that result(s) from applying the + prefix length. """ mask = ipv4.pfxlen2mask_int(pfxlen[0]) return [ipv4.int2ip(ipv4.ip2int(ip) & mask) for ip in ips] @secure_function -def remote_ip(scope, local_ips): +def remote_ip(scope: 'Scope', local_ips: list[str]) -> list[str]: """ Given an IP address, this function calculates the remaining available IP address under the assumption that it is a /30 network. diff --git a/src/Exscript/stdlib/list.py b/src/Exscript/stdlib/list.py index edc55b55..c344765d 100644 --- a/src/Exscript/stdlib/list.py +++ b/src/Exscript/stdlib/list.py @@ -20,14 +20,28 @@ # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" +The functions of the template language for lists, such as ``list.new()`` +or ``list.length()``. +""" + +from typing import TYPE_CHECKING, Any + from .util import secure_function +if TYPE_CHECKING: + from ..interpreter.scope import Scope + +# The functions take and return lists of any values: a template list can +# hold strings, numbers, and the values that functions return. + @secure_function -def new(scope, *args): +def new(scope: 'Scope', *args: list[Any]) -> list[Any]: """ Returns a new list with optional initial values. + :param args: The initial values; the first item of each is added. :rtype: list :return: A new list with optional initial values. """ @@ -35,7 +49,7 @@ def new(scope, *args): @secure_function -def length(scope, mylist): +def length(scope: 'Scope', mylist: list[Any]) -> list[int]: """ Returns the number of items in the list. @@ -48,7 +62,7 @@ def length(scope, mylist): @secure_function -def get(scope, source, index): +def get(scope: 'Scope', source: list[Any], index: list[int | str]) -> list[Any]: """ Returns a copy of the list item with the given index. It is an error if an item with the given index does not exist. @@ -61,19 +75,19 @@ def get(scope, source, index): :return: The string at the given index. """ try: - index = int(index[0]) + position = int(index[0]) except IndexError: raise ValueError('index variable is required') except ValueError: raise ValueError('index is not an integer') try: - return [source[index]] + return [source[position]] except IndexError: raise ValueError('no such item in the list') @secure_function -def sort(scope, source): +def sort(scope: 'Scope', source: list[Any]) -> list[Any]: """ Returns a copy of the given list sorted in ascending order. @@ -86,7 +100,7 @@ def sort(scope, source): @secure_function -def reverse(scope, source): +def reverse(scope: 'Scope', source: list[Any]) -> list[Any]: """ Returns a copy of the given list with the items in reverse order. @@ -99,7 +113,7 @@ def reverse(scope, source): @secure_function -def unique(scope, source): +def unique(scope: 'Scope', source: list[Any]) -> list[Any]: """ Returns a copy of the given list in which all duplicates are removed such that one of each item remains in the list. diff --git a/src/Exscript/stdlib/mysys.py b/src/Exscript/stdlib/mysys.py index 9e07416d..2d199d17 100644 --- a/src/Exscript/stdlib/mysys.py +++ b/src/Exscript/stdlib/mysys.py @@ -20,30 +20,42 @@ # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" +The system functions of the template language, which templates call as +``sys.env()``, ``sys.exec()``, ``sys.message()`` and ``sys.wait()``. +``sys.env()`` and ``sys.exec()`` are not secure: a template that is compiled +in secure mode cannot call them. +""" + import os import shlex import sys import time from subprocess import DEVNULL, PIPE, STDOUT, run +from typing import TYPE_CHECKING from .util import secure_function +if TYPE_CHECKING: + from ..interpreter.scope import Scope + -def env(scope, varname): +def env(scope: 'Scope', varname: list[str]) -> list[str]: """ Returns the value of the environment variable with the given - name. + name, or an empty string if it is not set. - :type varnames: string - :param varnames: A variable name. + :type varname: string + :param varname: A variable name. """ return [os.environ.get(varname[0], '')] -def execute(scope, command): +def execute(scope: 'Scope', command: list[str]) -> bool: """ Executes the given command locally, without a shell, and waits for it to - finish. Its output is stored in the __response__ variable. + finish. Its output, including the error output, is stored in the + __response__ variable, one line per item. :type command: string :param command: A command line. On POSIX systems it is split like a shell @@ -57,7 +69,7 @@ def execute(scope, command): @secure_function -def message(scope, string): +def message(scope: 'Scope', string: list[str]) -> bool: """ Writes the given string to stdout. @@ -69,7 +81,7 @@ def message(scope, string): @secure_function -def wait(scope, seconds): +def wait(scope: 'Scope', seconds: list[int | str]) -> bool: """ Waits for the given number of seconds. diff --git a/src/Exscript/stdlib/string.py b/src/Exscript/stdlib/string.py index 4f5bec1b..cd6ec84e 100644 --- a/src/Exscript/stdlib/string.py +++ b/src/Exscript/stdlib/string.py @@ -20,11 +20,21 @@ # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" +The functions of the template language for strings, such as +``string.replace()`` or ``string.split()``. +""" + +from typing import TYPE_CHECKING + from .util import secure_function +if TYPE_CHECKING: + from ..interpreter.scope import Scope + @secure_function -def replace(scope, strings, source, dest): +def replace(scope: 'Scope', strings: list[str], source: list[str], dest: list[str]) -> list[str]: """ Returns a copy of the given string (or list of strings) in which all occurrences of the given source are replaced by the given dest. @@ -42,7 +52,7 @@ def replace(scope, strings, source, dest): @secure_function -def split(scope, strings, separator): +def split(scope: 'Scope', strings: list[str], separator: list[str]) -> list[str]: """ Returns a list with the split values of the given string (or list of strings). The values are split at the separator. @@ -52,7 +62,7 @@ def split(scope, strings, separator): :type separator: string :param separator: The separator to split the string at. :rtype: string - :return: The resulting string, or list of strings. + :return: The resulting list of strings. """ result = [] for s in strings: @@ -61,7 +71,7 @@ def split(scope, strings, separator): @secure_function -def strip(scope, strings, chars=[None]): +def strip(scope: 'Scope', strings: list[str], chars: list[str | None] = [None]) -> list[str]: """ Returns a copy of the given string (or list of strings) with leading and trailing whitespace or custom characters removed. @@ -77,7 +87,7 @@ def strip(scope, strings, chars=[None]): @secure_function -def tolower(scope, strings): +def tolower(scope: 'Scope', strings: list[str]) -> list[str]: """ Returns the given string in lower case. @@ -90,7 +100,7 @@ def tolower(scope, strings): @secure_function -def toupper(scope, strings): +def toupper(scope: 'Scope', strings: list[str]) -> list[str]: """ Returns the given string in upper case. diff --git a/src/Exscript/stdlib/util.py b/src/Exscript/stdlib/util.py index b73c8796..41b821c9 100644 --- a/src/Exscript/stdlib/util.py +++ b/src/Exscript/stdlib/util.py @@ -20,19 +20,34 @@ # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" +The decorator that marks functions of the template language as secure. +""" + +from collections.abc import Callable from functools import wraps +from typing import Any, cast -def secure_function(function): +def secure_function[**P, R](function: Callable[P, R]) -> Callable[P, R]: """Decorator to indicate a secure function in scripts. The function is secure, if it does not have any side effects on and does not read data from the system where the interpreter is running. + + The decorator returns a wrapper that calls the function and has the + attribute ``_is_secure``. A template that is compiled in secure mode + may only call functions that have this attribute. + + :param function: The function that is marked as secure. + :return: The wrapper, with the name, docstring and signature of the + function. """ @wraps(function) - def wrapper(*args, **kwargs): + def wrapper(*args: P.args, **kwargs: P.kwargs) -> R: return function(*args, **kwargs) - wrapper._is_secure = True + # The type of the wrapper (functools._Wrapped) does not declare the attribute. + cast(Any, wrapper)._is_secure = True return wrapper From 8a17753b172ceec9e59f7d6377f536161b6ed923 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 10 Nov 2026 15:37:28 +0100 Subject: [PATCH 072/250] Annotate file, interact and log in Exscript.util Only annotations, docstrings and typing.cast() are added. --- src/Exscript/util/file.py | 55 ++++++++++++++++++-------- src/Exscript/util/interact.py | 73 +++++++++++++++++++++-------------- src/Exscript/util/log.py | 30 ++++++++++++-- 3 files changed, 110 insertions(+), 48 deletions(-) diff --git a/src/Exscript/util/file.py b/src/Exscript/util/file.py index cdd324fd..50bb2253 100644 --- a/src/Exscript/util/file.py +++ b/src/Exscript/util/file.py @@ -31,12 +31,15 @@ import importlib.util import os import re +from typing import Any, cast -from .. import Account +from .. import Account, Host from .cast import to_host -def get_accounts_from_file(filename, encoding='utf-8', password_encoding='utf-8'): +def get_accounts_from_file( + filename: str | os.PathLike[str], encoding: str = 'utf-8', password_encoding: str = 'utf-8' +) -> list[Account]: """ Reads a list of user/password combinations from the given file and returns a list of Account instances. The file content @@ -55,7 +58,9 @@ def get_accounts_from_file(filename, encoding='utf-8', password_encoding='utf-8' python3 -c 'import base64; print(base64.b64encode("thepassword".encode()).decode())' - :type filename: string + Raises an Exception if a password can not be decoded. + + :type filename: str|os.PathLike :param filename: The name of the file containing the list of accounts. :type encoding: str :param encoding: The encoding of the file. @@ -64,10 +69,13 @@ def get_accounts_from_file(filename, encoding='utf-8', password_encoding='utf-8' :rtype: list[Account] :return: The newly created account instances. """ - accounts = [] + accounts: list[Account] = [] parser = configparser.RawConfigParser() - parser.optionxform = str + # str keeps the case of the user names. The stub declares optionxform as + # a method, which may not be replaced; the cast is for the type checker. + cast(Any, parser).optionxform = str parser.read(filename, encoding) + password: str | bytes # read as a string, then decoded to bytes for user, password in parser.items('account-pool'): try: password = base64.decodebytes(password.encode('ascii')) @@ -79,12 +87,19 @@ def get_accounts_from_file(filename, encoding='utf-8', password_encoding='utf-8' def get_hosts_from_file( - filename, default_protocol='ssh', default_domain='', remove_duplicates=False, encoding='utf-8' -): + filename: str | os.PathLike[str], + default_protocol: str = 'ssh', + default_domain: str = '', + remove_duplicates: bool = False, + encoding: str = 'utf-8', +) -> list[Host]: """ Reads a list of hostnames from the file with the given name. + The file contains one hostname per line, which may be URL formatted + (see :meth:`Exscript.host.Host.set_uri`). Empty lines and everything after + a ``#`` are ignored. Raises OSError if the file does not exist. - :type filename: string + :type filename: str|os.PathLike :param filename: A full filename. :type default_protocol: str :param default_protocol: Passed to the Host constructor. @@ -102,8 +117,8 @@ def get_hosts_from_file( raise OSError('No such file: %s' % filename) # Read the hostnames. - have = set() - hosts = [] + have: set[str] = set() + hosts: list[Host] = [] with open(filename, encoding=encoding) as file_handle: for line in file_handle: hostname = line.split('#')[0].strip() @@ -117,7 +132,12 @@ def get_hosts_from_file( return hosts -def get_hosts_from_csv(filename, default_protocol='ssh', default_domain='', encoding='utf-8'): +def get_hosts_from_csv( + filename: str | os.PathLike[str], + default_protocol: str = 'ssh', + default_domain: str = '', + encoding: str = 'utf-8', +) -> list[Host]: """ Reads a list of hostnames and variables from the tab-separated .csv file with the given name. The first line of the file must contain the column @@ -141,7 +161,7 @@ def get_hosts_from_csv(filename, default_protocol='ssh', default_domain='', enco 10.0.0.1 myhost value 10.0.0.2 otherhost othervalue - :type filename: string + :type filename: str|os.PathLike :param filename: A full filename. :type default_protocol: str :param default_protocol: Passed to the Host constructor. @@ -174,7 +194,7 @@ def get_hosts_from_csv(filename, default_protocol='ssh', default_domain='', enco # definitions. last_uri = '' line_re = re.compile(r'[\r\n]*$') - hosts = [] + hosts: list[Host] = [] for line in file_handle: if line.strip() == '': continue @@ -204,7 +224,9 @@ def get_hosts_from_csv(filename, default_protocol='ssh', default_domain='', enco return hosts -def load_lib(filename): +# Any: the values are the objects in the __lib__ variable of the loaded file, +# which are not checked. +def load_lib(filename: str) -> dict[str, Any]: """ Loads a Python file containing functions, and returns the content of the __lib__ variable. The __lib__ variable must contain @@ -212,7 +234,7 @@ def load_lib(filename): Returns a dictionary mapping the namespaced function names to callables. The namespace is the basename of the file, without file - extension. + extension. Raises OSError if the file does not exist. :type filename: string :param filename: A full filename. @@ -227,7 +249,8 @@ def load_lib(filename): # load_module() was removed in Python 3.15; an explicit loader also accepts other suffixes. loader = importlib.machinery.SourceFileLoader(name, filename) spec = importlib.util.spec_from_file_location(name, filename, loader=loader) - module = importlib.util.module_from_spec(spec) + # With a loader given, spec_from_file_location() does not return None. + module = importlib.util.module_from_spec(cast(importlib.machinery.ModuleSpec, spec)) loader.exec_module(module) return dict((name + '.' + k, v) for (k, v) in list(module.__lib__.items())) diff --git a/src/Exscript/util/interact.py b/src/Exscript/util/interact.py index c8d3a87a..81908fb3 100644 --- a/src/Exscript/util/interact.py +++ b/src/Exscript/util/interact.py @@ -28,6 +28,7 @@ import getpass import os import sys +from collections.abc import Callable, Iterable from .. import Account from .cast import to_list @@ -41,21 +42,21 @@ class InputHistory: allow for such functionality. """ - def __init__(self, filename='~/.exscript_history', section=os.path.basename(sys.argv[0])): + def __init__( + self, filename: str = '~/.exscript_history', section: str = os.path.basename(sys.argv[0]) + ) -> None: """ - Constructor. The filename argument allows for listing on or - more config files, and is passed to Python's RawConfigParser; please - consult the documentation of RawConfigParser.read() if you require - more information. + Constructor. The filename argument is the name of the config + file; a leading ``~`` is expanded to the home directory. The optional section argument allows to specify a section under which the input is stored in the config file. The section defaults to the name of the running script. - Silently creates a tempfile if the given file can not be opened, - such that the object behavior does not change, but the history - is not remembered across instances. + If the file can not be read, the history starts empty. Saving + the input with set() raises OSError if the file can not be + written. - :type filename: str|list(str) + :type filename: str :param filename: The config file. :type section: str :param section: The section in the configfile. @@ -68,7 +69,7 @@ def __init__(self, filename='~/.exscript_history', section=os.path.basename(sys. if not self.parser.has_section(self.section): self.parser.add_section(self.section) - def get(self, key, default=None): + def get[T](self, key: str, default: T | None = None) -> str | T | None: """ Returns the input with the given key from the section that was passed to the constructor. If either the section or the key @@ -88,7 +89,7 @@ def get(self, key, default=None): except (configparser.NoSectionError, configparser.NoOptionError): return default - def set(self, key, value): + def set(self, key: str, value: str | None) -> str | None: """ Saves the input with the given key in the section that was passed to the constructor. If either the section or the key @@ -112,10 +113,18 @@ def set(self, key, value): return value -def prompt(key, message, default=None, doverh=True, strip=True, check=None, history=None): +def prompt( + key: str, + message: str, + default: str | None = None, + doverh: bool = True, + strip: bool = True, + check: Callable[[str], str | Iterable[str] | None] | None = None, + history: InputHistory | None = None, +) -> str: """ - Prompt the user for input. This function is similar to Python's built - in raw_input, with the following differences: + Prompts the user for input. This function is similar to Python's built + in input(), with the following differences: - You may specify a default value that is returned if the user presses "enter" without entering anything. @@ -129,8 +138,9 @@ def prompt(key, message, default=None, doverh=True, strip=True, check=None, hist The key argument specifies under which name the input is saved in the config file. - The given default value is only used if a default was not found in the - history. + If no default value is given, or if doverh is False, the value from the + history is offered as the default; the given default value is only used + if the history has no value. The strip argument specifies that the returned value should be stripped of whitespace (default). @@ -142,19 +152,19 @@ def prompt(key, message, default=None, doverh=True, strip=True, check=None, hist def validate(input): if len(input) < 4: return 'Please enter at least 4 characters!' - value = prompt('test', 'Enter a value', 'My Default', check = validate) + value = prompt('test', 'Enter a value', 'My Default', doverh=False, check=validate) print('You entered:', value) This leads to the following output:: - Please enter a value [My Default]: abc + Enter a value [My Default]: abc Please enter at least 4 characters! - Please enter a value [My Default]: Foobar + Enter a value [My Default]: Foobar You entered: Foobar The next time the same code is started, the input 'Foobar' is remembered:: - Please enter a value [Foobar]: (enters nothing) + Enter a value [Foobar]: (enters nothing) You entered: Foobar :type key: str @@ -162,15 +172,18 @@ def validate(input): :type message: str :param message: The user prompt. :type default: str|None - :param default: The offered default if none was found in the history. + :param default: The offered default value, see above. :type doverh: bool :param doverh: Whether to prefer default value over history value. :type strip: bool :param strip: Whether to remove whitespace from the input. :type check: callable - :param check: A function that is called for validating the input. + :param check: A function that is called for validating the input. It + returns None if the input is valid, and an error message or a list + of error messages otherwise. :type history: :class:`InputHistory` or None :param history: The history used for recording default values, or None. + :return: The input, or the default value if the user entered nothing. """ if history is None: history = InputHistory() @@ -194,7 +207,9 @@ def validate(input): return value -def get_filename(key, message, default=None, history=None): +def get_filename( + key: str, message: str, default: str | None = None, history: InputHistory | None = None +) -> str: """ Like :meth:`prompt`, but only accepts the name of an existing file as an input. @@ -204,19 +219,21 @@ def get_filename(key, message, default=None, history=None): :type message: str :param message: The user prompt. :type default: str|None - :param default: The offered default if none was found in the history. + :param default: The offered default value, see :meth:`prompt`. :type history: :class:`InputHistory` or None :param history: The history used for recording default values, or None. + :return: The name of the file. """ - def _validate(string): + def _validate(string: str) -> str | None: if not os.path.isfile(string): return 'File not found. Please enter a filename.' + return None return prompt(key, message, default, check=_validate, history=history) -def get_user(prompt=None): +def get_user(prompt: str | None = None) -> str: """ Prompts the user for his login name, defaulting to the USER environment variable. Returns a string containing the username. @@ -243,7 +260,7 @@ def get_user(prompt=None): return user -def get_login(): +def get_login() -> tuple[str, str]: """ Prompts the user for the login name using get_user(), and also asks for the password. @@ -258,7 +275,7 @@ def get_login(): return user, password -def read_login(): +def read_login() -> Account: """ Like get_login(), but returns an Account object. diff --git a/src/Exscript/util/log.py b/src/Exscript/util/log.py index b2926a2e..b469576e 100644 --- a/src/Exscript/util/log.py +++ b/src/Exscript/util/log.py @@ -24,35 +24,57 @@ Logging utilities. """ -from .. import FileLogger +from collections.abc import Callable +from typing import Any + +from .. import FileLogger, Logger from .impl import add_label _loggers: list[FileLogger] = [] -def log_to(logger): +def log_to[F: Callable[..., Any]](logger: Logger) -> Callable[[F], F]: """ Wraps a function that has a connection passed such that everything that happens on the connection is logged using the given logger. + The returned decorator labels the function and returns it unchanged; + when the :class:`Exscript.Queue` runs the function for a host, it logs + the conversation with the host to a new log of the logger. + :type logger: Logger :param logger: The logger that handles the logging. + :return: A decorator for the function that the queue runs. """ logger_id = id(logger) - def decorator(function): + def decorator(function: F) -> F: func = add_label(function, 'log_to', logger_id=logger_id) return func return decorator -def log_to_file(logdir, mode='a', delete=False, clearmem=True): +def log_to_file[F: Callable[..., Any]]( + logdir: str, mode: str = 'a', delete: bool = False, clearmem: bool = True +) -> Callable[[F], F]: """ Like :func:`log_to`, but automatically creates a new FileLogger instead of having one passed. Note that the logger stays alive (in memory) forever. If you need to control the lifetime of a logger, use :func:`log_to` instead. + + :type logdir: str + :param logdir: The directory of the log files, see + :class:`Exscript.FileLogger`. + :type mode: str + :param mode: The mode in which the log files are opened for writing, + e.g. 'a' to append to existing logs. + :type delete: bool + :param delete: Whether to delete the log of a job that succeeds. + :type clearmem: bool + :param clearmem: Whether to forget the logs of finished jobs in memory. + :return: A decorator for the function that the queue runs. """ logger = FileLogger(logdir, mode, delete, clearmem) _loggers.append(logger) From ac9a72693fab03b3007f271a4ea3b61c8b8f85c1 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 10 Nov 2026 15:40:00 +0100 Subject: [PATCH 073/250] Check the newly annotated modules strictly 53 more modules leave the relaxed mypy override; 18 are left. pre-commit fails below 87 percent docstring coverage instead of 72. --- pyproject.toml | 54 +------------------------------------------------- 1 file changed, 1 insertion(+), 53 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 25b791a3..1a7f4674 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -132,65 +132,13 @@ ignore_missing_imports = true # ncclient and others ship no type hints # it passes the strict checks; the list only shrinks. module = [ "Exscript.contrib.otel", - "Exscript.emulators.command", - "Exscript.emulators.iosemu", - "Exscript.emulators.vdevice", - "Exscript.protocols", - "Exscript.protocols.drivers.ace", - "Exscript.protocols.drivers.adtran", - "Exscript.protocols.drivers.aironet", - "Exscript.protocols.drivers.aix", - "Exscript.protocols.drivers.arbor_peakflow", - "Exscript.protocols.drivers.aruba", - "Exscript.protocols.drivers.bigip", - "Exscript.protocols.drivers.brocade", - "Exscript.protocols.drivers.cienasaos", - "Exscript.protocols.drivers.enterasys", - "Exscript.protocols.drivers.enterasys_wc", - "Exscript.protocols.drivers.eos", - "Exscript.protocols.drivers.ericsson_ban", - "Exscript.protocols.drivers.fortios", - "Exscript.protocols.drivers.generic", - "Exscript.protocols.drivers.hp_pro_curve", - "Exscript.protocols.drivers.icotera", - "Exscript.protocols.drivers.ios", - "Exscript.protocols.drivers.ios_xr", - "Exscript.protocols.drivers.isam", - "Exscript.protocols.drivers.junos", - "Exscript.protocols.drivers.junos_erx", - "Exscript.protocols.drivers.mrv", - "Exscript.protocols.drivers.nxos", - "Exscript.protocols.drivers.one_os", - "Exscript.protocols.drivers.rios", - "Exscript.protocols.drivers.shell", - "Exscript.protocols.drivers.smart_edge_os", - "Exscript.protocols.drivers.sp_vxoa", - "Exscript.protocols.drivers.sros", - "Exscript.protocols.drivers.vrp", "Exscript.protocols.drivers.vxworks", "Exscript.protocols.drivers.zte", "Exscript.protocols.drivers.zyxel", - "Exscript.protocols.dummy", "Exscript.protocols.netconf", "Exscript.protocols.ssh2", - "Exscript.protocols.telnet", - "Exscript.protocols.telnetlib", "Exscript.queue", "Exscript.servers.httpd", - "Exscript.servers.server", - "Exscript.servers.sshd", - "Exscript.servers.telnetd", - "Exscript.stdlib.connection", - "Exscript.stdlib.crypt", - "Exscript.stdlib.file", - "Exscript.stdlib.ipv4", - "Exscript.stdlib.list", - "Exscript.stdlib.mysys", - "Exscript.stdlib.string", - "Exscript.stdlib.util", - "Exscript.util.file", - "Exscript.util.interact", - "Exscript.util.log", "Exscript.util.mail", "Exscript.util.report", "Exscript.util.sigint", @@ -218,7 +166,7 @@ extra_checks = false # Docstring coverage (P8). Private functions count; other dunder methods than # __init__, nested functions and property setters do not. Raise fail-under as # docstrings are added; the goal is 100. -fail-under = 72 +fail-under = 87 ignore-magic = true ignore-nested-functions = true ignore-setters = true From eaef21db8d16882dae8927e40fecad37bd25a000 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 10 Nov 2026 15:59:13 +0100 Subject: [PATCH 074/250] Annotate template, start, mail and report in Exscript.util The four modules leave the relaxed mypy override; start() casts the queue to Any until the queue is annotated. Only annotations, docstrings and typing.cast() are added. --- pyproject.toml | 4 - src/Exscript/util/mail.py | 187 +++++++++++++++++++++++----------- src/Exscript/util/report.py | 39 +++++-- src/Exscript/util/start.py | 69 +++++++++---- src/Exscript/util/template.py | 114 +++++++++++++++++---- 5 files changed, 305 insertions(+), 108 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 1a7f4674..1f5d9f0f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -139,11 +139,7 @@ module = [ "Exscript.protocols.ssh2", "Exscript.queue", "Exscript.servers.httpd", - "Exscript.util.mail", - "Exscript.util.report", "Exscript.util.sigint", - "Exscript.util.start", - "Exscript.util.template", "Exscript.workqueue.job", "Exscript.workqueue.mainloop", "Exscript.workqueue.pipeline", diff --git a/src/Exscript/util/mail.py b/src/Exscript/util/mail.py index 8c78aa0f..8923009e 100644 --- a/src/Exscript/util/mail.py +++ b/src/Exscript/util/mail.py @@ -37,6 +37,7 @@ from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText from getpass import getuser +from typing import Any, cast from Exscript.util.event import Event @@ -49,16 +50,29 @@ class _TemplateParser: """ - This exists for backward compatibility; Python 2.3 does not come - with a similar way for string substitution yet. + Replaces the placeholders of the form ``{name}`` in a mail template by + the values of the variables with the same name. """ - def __init__(self): - self.tmpl_vars = None + def __init__(self) -> None: + """ + Constructor. + """ + # The values of the variables may have any type; they are converted + # to strings. parse() sets the variables before they are used; the + # cast is for the type checker. + self.tmpl_vars: dict[str, Any] = cast(dict[str, Any], None) # Tokens that include variables in a string may use this callback to # substitute the variable against its value. - def _variable_sub_cb(self, match): + def _variable_sub_cb(self, match: re.Match[str]) -> str: + """ + Returns the value of the variable that the given placeholder names, + as a string; the items of a list are put on separate lines. + A placeholder that is escaped with a backslash is replaced by + ``$name``. Raises an Exception if the name is not a valid variable + name, or if the variable is not defined. + """ escape = match.group(1) varname = match.group(2) if escape == '\\': @@ -72,7 +86,12 @@ def _variable_sub_cb(self, match): value = '\n'.join([str(v) for v in value]) return str(value) - def parse(self, template, **kwargs): + def parse(self, template: str, **kwargs: Any) -> str: + """ + Returns the given template with the placeholders replaced by the + values of the given variables. A line that ends with a space is + joined with the next line. + """ self.tmpl_vars = kwargs output = '' for line in template.split('\n'): @@ -83,23 +102,44 @@ def parse(self, template, **kwargs): return _string_re.sub(self._variable_sub_cb, output) -def _render_template(string, **vars): +def _render_template(string: str, **vars: Any) -> str: + """ + Returns the given mail template with the placeholders replaced by the + given variables. The variables ``date`` (the current date, as + YYYY-MM-DD) and ``user`` (the name of the current user) are defined by + default. + """ default = {'date': time.strftime('%Y-%m-%d'), 'user': getuser()} default.update(vars) parser = _TemplateParser() return parser.parse(string, **default) -def _is_header_line(line): +def _is_header_line(line: str) -> bool: + """ + Returns True if the given line of a mail template is a header line, + such as ``Subject: Hello``. + """ return re.match(r'^\w+: .+$', line) is not None -def _get_var_from_header_line(line): - match = re.match(r'^(\w+): (.+)$', line) +def _get_var_from_header_line(line: str) -> tuple[str, str]: + """ + Returns the field name, in lower case, and the value of the given + header line. + """ + # The caller checks the line with _is_header_line(), so it matches; the + # cast is for the type checker. + match = cast(re.Match[str], re.match(r'^(\w+): (.+)$', line)) return match.group(1).strip().lower(), match.group(2).strip() -def _cleanup_mail_addresses(recipients): +def _cleanup_mail_addresses(recipients: str | list[str]) -> list[str]: + """ + Returns the given email addresses in lower case, sorted and without + duplicates. The addresses are given as a list, or as a string in which + they are separated by commas, semicolons or line breaks. + """ if isinstance(recipients, list): recipients = ','.join(recipients) rcpt = re.split(r'\s*[,;\r\n]\s*', recipients.lower()) @@ -116,13 +156,21 @@ class Mail: Represents an email. """ - def __init__(self, sender=None, to='', cc='', bcc='', subject='', body=''): + def __init__( + self, + sender: str | None = None, + to: str | list[str] = '', + cc: str | list[str] = '', + bcc: str | list[str] = '', + subject: str = '', + body: str = '', + ) -> None: """ Creates a new email with the given values. - If the given sender is None, one will be automatically chosen - using getpass.getuser(). + If the given sender is None or empty, the sender is the name of + the current user (see getpass.getuser()) at the local host. - :type sender: string + :type sender: string|None :param sender: The email address of the sender. :type to: string|list(string) :param to: A list of email addresses, passed to set_to(). @@ -136,13 +184,14 @@ def __init__(self, sender=None, to='', cc='', bcc='', subject='', body=''): :param body: The email body, passed to set_body(). """ self.changed_event = Event() - self.files = [] - self.sender = None - self.cc = None - self.bcc = None - self.to = None - self.subject = None - self.body = None + self.files: list[str] = [] + # The setters below set the fields; the casts are for the type checker. + self.sender: str = cast(str, None) + self.cc: list[str] = cast(list[str], None) + self.bcc: list[str] = cast(list[str], None) + self.to: list[str] = cast(list[str], None) + self.subject: str = cast(str, None) + self.body: str = cast(str, None) if not sender: domain = socket.getfqdn('localhost') sender = getuser() + '@' + domain @@ -153,10 +202,11 @@ def __init__(self, sender=None, to='', cc='', bcc='', subject='', body=''): self.set_subject(subject) self.set_body(body) - def set_from_template_string(self, string): + def set_from_template_string(self, string: str) -> None: """ Reads the given template (SMTP formatted) and sets all fields - accordingly. + accordingly. Raises an Exception if the template has a header + field other than From, To, Cc, Bcc and Subject. :type string: string :param string: The template. @@ -186,7 +236,7 @@ def set_from_template_string(self, string): raise Exception('Invalid header field "%s"' % key) self.set_body(body.strip()) - def set_sender(self, sender): + def set_sender(self, sender: str) -> None: """ Defines the value of the "From:" field. @@ -196,7 +246,7 @@ def set_sender(self, sender): self.sender = sender self.changed_event() - def get_sender(self): + def get_sender(self) -> str: """ Returns the value of the "From:" field. @@ -205,7 +255,7 @@ def get_sender(self): """ return self.sender - def set_to(self, to): + def set_to(self, to: str | list[str]) -> None: """ Replaces the current list of recipients in the 'to' field by the given value. The value may be one of the following: @@ -219,7 +269,7 @@ def set_to(self, to): self.to = _cleanup_mail_addresses(to) self.changed_event() - def add_to(self, to): + def add_to(self, to: str | list[str]) -> None: """ Adds the given list of recipients to the 'to' field. Accepts the same argument types as set_to(). @@ -230,7 +280,7 @@ def add_to(self, to): self.to += _cleanup_mail_addresses(to) self.changed_event() - def get_to(self): + def get_to(self) -> list[str]: """ Returns the value of the "to" field. @@ -239,7 +289,7 @@ def get_to(self): """ return self.to - def set_cc(self, cc): + def set_cc(self, cc: str | list[str]) -> None: """ Like set_to(), but for the 'cc' field. @@ -249,7 +299,7 @@ def set_cc(self, cc): self.cc = _cleanup_mail_addresses(cc) self.changed_event() - def add_cc(self, cc): + def add_cc(self, cc: str | list[str]) -> None: """ Like add_to(), but for the 'cc' field. @@ -259,7 +309,7 @@ def add_cc(self, cc): self.cc += _cleanup_mail_addresses(cc) self.changed_event() - def get_cc(self): + def get_cc(self) -> list[str]: """ Returns the value of the "cc" field. @@ -268,7 +318,7 @@ def get_cc(self): """ return self.cc - def set_bcc(self, bcc): + def set_bcc(self, bcc: str | list[str]) -> None: """ Like set_to(), but for the 'bcc' field. @@ -278,7 +328,7 @@ def set_bcc(self, bcc): self.bcc = _cleanup_mail_addresses(bcc) self.changed_event() - def add_bcc(self, bcc): + def add_bcc(self, bcc: str | list[str]) -> None: """ Like add_to(), but for the 'bcc' field. @@ -288,7 +338,7 @@ def add_bcc(self, bcc): self.bcc += _cleanup_mail_addresses(bcc) self.changed_event() - def get_bcc(self): + def get_bcc(self) -> list[str]: """ Returns the value of the "bcc" field. @@ -297,7 +347,7 @@ def get_bcc(self): """ return self.bcc - def get_recipients(self): + def get_recipients(self) -> list[str]: """ Returns a list of all recipients (to, cc, and bcc). @@ -306,7 +356,7 @@ def get_recipients(self): """ return self.get_to() + self.get_cc() + self.get_bcc() - def set_subject(self, subject): + def set_subject(self, subject: str) -> None: """ Defines the subject line. @@ -316,7 +366,7 @@ def set_subject(self, subject): self.subject = subject self.changed_event() - def get_subject(self): + def get_subject(self) -> str: """ Returns the subject line. @@ -325,7 +375,7 @@ def get_subject(self): """ return self.subject - def set_body(self, body): + def set_body(self, body: str) -> None: """ Defines the body of the mail. @@ -335,7 +385,7 @@ def set_body(self, body): self.body = body self.changed_event() - def get_body(self): + def get_body(self) -> str: """ Returns the body of the mail. @@ -344,9 +394,9 @@ def get_body(self): """ return self.body - def get_smtp_header(self): + def get_smtp_header(self) -> str: """ - Returns the SMTP formatted header of the line. + Returns the SMTP formatted header of the mail. :rtype: string :return: The SMTP header. @@ -358,7 +408,7 @@ def get_smtp_header(self): header += "Subject: %s\r\n" % self.get_subject() return header - def get_smtp_mail(self): + def get_smtp_mail(self) -> str: """ Returns the SMTP formatted email, as it may be passed to sendmail. @@ -369,7 +419,7 @@ def get_smtp_mail(self): body = self.get_body().replace('\n', '\r\n') return header + '\r\n' + body + '\r\n' - def add_attachment(self, filename): + def add_attachment(self, filename: str) -> None: """ Adds the file with the given name as an attachment. @@ -378,7 +428,7 @@ def add_attachment(self, filename): """ self.files.append(filename) - def get_attachments(self): + def get_attachments(self) -> list[str]: """ Returns a list of attached files. @@ -388,14 +438,24 @@ def get_attachments(self): return self.files -def from_template_string(string, **kwargs): +def from_template_string(string: str, **kwargs: Any) -> Mail: """ Reads the given SMTP formatted template, and creates a new Mail object using the information. + Placeholders of the form ``{name}`` are replaced by the value of the + variable with that name; the items of a list are put on separate lines. + The variables ``date`` (the current date) and ``user`` (the name of the + current user) are defined by default. A line that ends with a space is + joined with the next line. The header fields From, To, Cc, Bcc and + Subject at the start of the template set the fields of the mail; the + rest of the template is the body. Raises an Exception if a placeholder + names an undefined variable, or if the template has another header + field. + :type string: str :param string: The SMTP formatted template. - :type kwargs: str + :type kwargs: dict :param kwargs: Variables to replace in the template. :rtype: Mail :return: The resulting mail. @@ -406,14 +466,14 @@ def from_template_string(string, **kwargs): return mail -def from_template(filename, **kwargs): +def from_template(filename: str | os.PathLike[str], **kwargs: Any) -> Mail: """ - Like from_template_string(), but reads the template from the file with - the given name instead. + Like from_template_string(), but reads the template from the UTF-8 file + with the given name instead. - :type filename: string + :type filename: str|os.PathLike :param filename: The name of the template file. - :type kwargs: str + :type kwargs: dict :param kwargs: Variables to replace in the template. :rtype: Mail :return: The resulting mail. @@ -422,15 +482,20 @@ def from_template(filename, **kwargs): return from_template_string(fp.read(), **kwargs) -def _get_mime_object(filename): - # Guess the content type based on the file's extension. Encoding - # is ignored, although we should check for simple things like - # gzip'd or compressed files. +def _get_mime_object(filename: str) -> MIMEBase: + """ + Returns a MIME part with the content of the given file, attached under + its base name. The content type is guessed from the file name + (application/octet-stream for unknown and compressed files); text files + are read as UTF-8. + """ + # Guess the content type based on the file's extension. ctype, encoding = mimetypes.guess_type(filename) if ctype is None or encoding is not None: ctype = 'application/octet-stream' maintype, subtype = ctype.split('/', 1) + msg: MIMEBase # MIMEText, MIMEImage and MIMEAudio are MIMEBase, too if maintype == 'text': with open(filename, encoding='utf-8') as fp: msg = MIMEText(fp.read(), _subtype=subtype) @@ -451,9 +516,11 @@ def _get_mime_object(filename): return msg -def send(mail, server='localhost'): +def send(mail: Mail, server: str = 'localhost') -> None: """ - Sends the given mail. + Sends the given mail through the SMTP server with the given address. + The body is sent as UTF-8 text, and each attachment as a MIME part. + Raises smtplib.SMTPException or OSError if sending fails. :type mail: Mail :param mail: The mail object. @@ -470,7 +537,9 @@ def send(mail, server='localhost'): message['Cc'] = ', '.join(mail.get_cc()) message.preamble = 'Your mail client is not MIME aware.' - body = MIMEText(mail.get_body().encode("utf-8"), "plain", "utf-8") + # MIMEText also takes the body as bytes, which its stub does not declare; + # the cast is for the type checker. + body = MIMEText(cast(Any, mail.get_body().encode("utf-8")), "plain", "utf-8") body.add_header('Content-Disposition', 'inline') message.attach(body) diff --git a/src/Exscript/util/report.py b/src/Exscript/util/report.py index cf762ad4..8e58469c 100644 --- a/src/Exscript/util/report.py +++ b/src/Exscript/util/report.py @@ -24,12 +24,20 @@ Formatting logs into human readable reports. """ +from typing import cast -def _underline(text, line='-'): +from .. import Logger + + +def _underline(text: str, line: str = '-') -> list[str]: + """ + Returns the given text, and below it the given character repeated to + the length of the text, as a list of two lines. + """ return [text, line * len(text)] -def status(logger): +def status(logger: Logger) -> str: """ Creates a one-line summary on the actions that were logged by the given Logger. @@ -57,10 +65,11 @@ def status(logger): return msg % (total, aborted, succeeded) -def summarize(logger): +def summarize(logger: Logger) -> str: """ Creates a short summary on the actions that were logged by the given - Logger. + Logger: one line for each log, with the name of the log and ``ok``, or + the error message (the name of the exception if it has no message). :type logger: Logger :param logger: The logger that recorded what happened in the queue. @@ -75,14 +84,26 @@ def summarize(logger): return '\n'.join(summary) -def format(logger, show_successful=True, show_errors=True, show_traceback=True): +def format( + logger: Logger, + show_successful: bool = True, + show_errors: bool = True, + show_traceback: bool = True, +) -> str: """ - Prints a report of the actions that were logged by the given Logger. + Formats a report of the actions that were logged by the given Logger. The report contains a list of successful actions, as well as the full error message on failed actions. :type logger: Logger :param logger: The logger that recorded what happened in the queue. + :type show_successful: bool + :param show_successful: Whether to list the successful actions. + :type show_errors: bool + :param show_errors: Whether to list the failed actions. + :type show_traceback: bool + :param show_traceback: Whether to show the traceback of each failed + action, or only the error message. :rtype: string :return: A string summarizing the status of every performed task. """ @@ -92,12 +113,14 @@ def format(logger, show_successful=True, show_errors=True, show_traceback=True): errors = logger.get_aborted_actions() if show_errors and errors: output += _underline('Failed actions:') + # The logs ended with an error, so get_error() returns a string; the + # casts are for the type checker. for log in logger.get_aborted_logs(): if show_traceback: output.append(log.get_name() + ':') - output.append(log.get_error()) + output.append(cast(str, log.get_error())) else: - output.append(log.get_name() + ': ' + log.get_error(False)) + output.append(log.get_name() + ': ' + cast(str, log.get_error(False))) output.append('') # Print successful actions. diff --git a/src/Exscript/util/start.py b/src/Exscript/util/start.py index 3078bee1..d0df9595 100644 --- a/src/Exscript/util/start.py +++ b/src/Exscript/util/start.py @@ -24,17 +24,27 @@ Quickstart methods for the Exscript queue. """ -from .. import Queue +from collections.abc import Callable, Iterable +from typing import Any, cast + +from .. import Account, Host, Queue from .decorator import autoauthenticate, autologin from .interact import read_login -def run(users, hosts, func, **kwargs): +def run( + users: Account | Iterable[Account], + hosts: str | Host | Iterable[str | Host], + func: Callable[..., Any], + **kwargs: Any, +) -> None: """ Convenience function that creates an Exscript.Queue instance, adds the given accounts, and calls Queue.run() with the given hosts and function as an argument. + The queue opens a connection to each host and calls the function as + ``func(job, host, conn)``; the connection is not logged in yet. If you also want to pass arguments to the given function, you may use util.decorator.bind() like this:: @@ -48,52 +58,65 @@ def my_callback(job, host, conn, my_arg, **kwargs): :type users: Account|list[Account] :param users: The account(s) to use for logging in. - :type hosts: Host|list[Host] - :param hosts: A list of Host objects. + :type hosts: str|Host|list(str|Host) + :param hosts: A hostname or Host object, or a list of them. :type func: function :param func: The callback function. :type kwargs: dict - :param kwargs: Passed to the Exscript.Queue constructor. + :param kwargs: Passed to the Exscript.Queue constructor, except for + ``attempts``, the number of attempts for each host (default 1), + which is passed to Queue.run(). """ attempts = kwargs.get("attempts", 1) if "attempts" in kwargs: del kwargs["attempts"] - queue = Queue(**kwargs) + # The queue is not annotated (it will be reworked); the cast is for the + # type checker. + queue = cast(Any, Queue)(**kwargs) queue.add_account(users) queue.run(hosts, func, attempts) queue.destroy() -def quickrun(hosts, func, **kwargs): +def quickrun( + hosts: str | Host | Iterable[str | Host], func: Callable[..., Any], **kwargs: Any +) -> None: """ A wrapper around run() that creates the account by asking the user for entering his login information. - :type hosts: Host|list[Host] - :param hosts: A list of Host objects. + :type hosts: str|Host|list(str|Host) + :param hosts: A hostname or Host object, or a list of them. :type func: function :param func: The callback function. :type kwargs: dict - :param kwargs: Passed to the Exscript.Queue constructor. + :param kwargs: Passed to run(). """ run(read_login(), hosts, func, **kwargs) -def start(users, hosts, func, only_authenticate=False, **kwargs): +def start( + users: Account | Iterable[Account], + hosts: str | Host | Iterable[str | Host], + func: Callable[..., Any], + only_authenticate: bool = False, + **kwargs: Any, +) -> None: """ Like run(), but automatically logs into the host before passing the host to the callback function. :type users: Account|list[Account] :param users: The account(s) to use for logging in. - :type hosts: Host|list[Host] - :param hosts: A list of Host objects. + :type hosts: str|Host|list(str|Host) + :param hosts: A hostname or Host object, or a list of them. :type func: function :param func: The callback function. :type only_authenticate: bool - :param only_authenticate: don't authorize, just authenticate? + :param only_authenticate: Whether to only authenticate, without + authorizing (see :func:`Exscript.util.decorator.autoauthenticate`). :type kwargs: dict - :param kwargs: Passed to the Exscript.Queue constructor. + :param kwargs: Passed to run(). """ if only_authenticate: run(users, hosts, autoauthenticate()(func), **kwargs) @@ -101,19 +124,25 @@ def start(users, hosts, func, only_authenticate=False, **kwargs): run(users, hosts, autologin()(func), **kwargs) -def quickstart(hosts, func, only_authenticate=False, **kwargs): +def quickstart( + hosts: str | Host | Iterable[str | Host], + func: Callable[..., Any], + only_authenticate: bool = False, + **kwargs: Any, +) -> None: """ Like quickrun(), but automatically logs into the host before passing the connection to the callback function. - :type hosts: Host|list[Host] - :param hosts: A list of Host objects. + :type hosts: str|Host|list(str|Host) + :param hosts: A hostname or Host object, or a list of them. :type func: function :param func: The callback function. :type only_authenticate: bool - :param only_authenticate: don't authorize, just authenticate? + :param only_authenticate: Whether to only authenticate, without + authorizing (see :func:`Exscript.util.decorator.autoauthenticate`). :type kwargs: dict - :param kwargs: Passed to the Exscript.Queue constructor. + :param kwargs: Passed to run(). """ if only_authenticate: quickrun(hosts, autoauthenticate()(func), **kwargs) diff --git a/src/Exscript/util/template.py b/src/Exscript/util/template.py index 21319352..4ade58c0 100644 --- a/src/Exscript/util/template.py +++ b/src/Exscript/util/template.py @@ -31,21 +31,59 @@ allowed. Pass ``secure=False`` to allow all functions. """ +from typing import TYPE_CHECKING, Any, cast + from Exscript import stdlib from Exscript.interpreter import Parser from Exscript.stdlib.util import secure_function +if TYPE_CHECKING: + from Exscript.interpreter.program import Program + from Exscript.protocols import Protocol + debug = 0 -def _trusted(value): +def _trusted(value: object) -> object: + """ + Returns the given value of a template variable. A function that is not + marked as secure is returned wrapped by secure_function(), so that a + template that is compiled in secure mode may call it. + """ # Functions passed by the caller are allowed in secure mode. if callable(value) and not hasattr(value, '_is_secure'): return secure_function(value) return value -def _compile(conn, filename, template, parser_kwargs, **kwargs): +# Any: the variables of a template are lists whose items depend on the +# template, functions, or the connection; the parser options have +# different types. +def _compile( + conn: 'Protocol | None', + filename: str | None, + template: str, + parser_kwargs: dict[str, Any], + **kwargs: Any, +) -> 'Program': + """ + Compiles the given template with a new parser. The parser defines the + built-in variables (``__filename__``, ``__username__``, + ``__hostname__``, ``__response__`` and ``__connection__``), the + functions of the standard library, and the given variables, which may + override the built-in ones; the functions among them may be called in + secure mode. + + :param conn: The connection on which the template runs, or None if the + template is only compiled. + :param filename: The name of the template file, or None. + :param template: The template. + :param parser_kwargs: The options of the parser; ``debug`` is added + with the value of the debug variable of this module if it is + missing. + :param kwargs: Variables to define in the template. + :return: The compiled template. + """ undefined = 'undefined' if conn: hostname = conn.get_host() @@ -74,16 +112,29 @@ def _compile(conn, filename, template, parser_kwargs, **kwargs): # Allow for overriding built-in variables. parser.define(**{name: _trusted(value) for name, value in kwargs.items()}) - # Compile the template. - return parser.parse(template, parser.variables.get('__filename__')[0]) + # Compile the template. __filename__ is a list, as defined above or by + # define(); the cast is for the type checker. + return parser.parse(template, cast(list[str], parser.variables.get('__filename__'))[0]) -def _run(conn, filename, template, parser_kwargs, **kwargs): +def _run( + conn: 'Protocol', + filename: str | None, + template: str, + parser_kwargs: dict[str, Any], + **kwargs: Any, +) -> dict[str, Any]: + """ + Compiles the given template and runs it on the given connection, see + _compile(). + + :return: The variables that are defined after execution of the template. + """ compiled = _compile(conn, filename, template, parser_kwargs, **kwargs) return compiled.execute() -def test(string, secure=True, **kwargs): +def test(string: str, secure: bool = True, **kwargs: Any) -> None: """ Compiles the given template, and raises an exception if that failed. Does nothing otherwise. @@ -100,10 +151,10 @@ def test(string, secure=True, **kwargs): _compile(None, None, string, parser_args, **kwargs) -def test_file(filename, secure=True, **kwargs): +def test_file(filename: str, secure: bool = True, **kwargs: Any) -> None: """ - Convenience wrapper around test() that reads the template from a file - instead. + Convenience wrapper around test() that reads the template from a UTF-8 + file instead. :type filename: string :param filename: The name of the template file. @@ -117,7 +168,15 @@ def test_file(filename, secure=True, **kwargs): _compile(None, filename, fp.read(), parser_args, **kwargs) -def eval(conn, string, strip_command=True, no_escaping=False, *, secure=True, **kwargs): +def eval( + conn: 'Protocol', + string: str, + strip_command: bool = True, + no_escaping: bool = False, + *, + secure: bool = True, + **kwargs: Any, +) -> dict[str, Any]: r""" Compiles the given template and executes it on the given connection. Raises an exception if the compilation fails. @@ -163,10 +222,18 @@ def eval(conn, string, strip_command=True, no_escaping=False, *, secure=True, ** return _run(conn, None, string, parser_args, **kwargs) -def eval_file(conn, filename, strip_command=True, no_escaping=False, *, secure=True, **kwargs): +def eval_file( + conn: 'Protocol', + filename: str, + strip_command: bool = True, + no_escaping: bool = False, + *, + secure: bool = True, + **kwargs: Any, +) -> dict[str, Any]: """ - Convenience wrapper around eval() that reads the template from a file - instead. + Convenience wrapper around eval() that reads the template from a UTF-8 + file instead. :type conn: Exscript.protocols.Protocol :param conn: The connection on which to run the template. @@ -182,13 +249,17 @@ def eval_file(conn, filename, strip_command=True, no_escaping=False, *, secure=T :param secure: Whether to forbid the insecure functions. :type kwargs: dict :param kwargs: Variables to define in the template. + :rtype: dict + :return: The variables that are defined after execution of the script. """ parser_args = {'strip_command': strip_command, 'no_escaping': no_escaping, 'secure': secure} with open(filename, encoding='utf-8') as fp: return _run(conn, filename, fp.read(), parser_args, **kwargs) -def paste(conn, string, no_escaping=False, *, secure=True, **kwargs): +def paste( + conn: 'Protocol', string: str, no_escaping: bool = False, *, secure: bool = True, **kwargs: Any +) -> dict[str, Any]: """ Compiles the given template and executes it on the given connection. This function differs from eval() such that it does not @@ -220,10 +291,17 @@ def paste(conn, string, no_escaping=False, *, secure=True, **kwargs): return _run(conn, None, string, parser_args, **kwargs) -def paste_file(conn, filename, no_escaping=False, *, secure=True, **kwargs): +def paste_file( + conn: 'Protocol', + filename: str, + no_escaping: bool = False, + *, + secure: bool = True, + **kwargs: Any, +) -> dict[str, Any]: """ - Convenience wrapper around paste() that reads the template from a file - instead. + Convenience wrapper around paste() that reads the template from a UTF-8 + file instead. :type conn: Exscript.protocols.Protocol :param conn: The connection on which to run the template. @@ -237,6 +315,8 @@ def paste_file(conn, filename, no_escaping=False, *, secure=True, **kwargs): :param secure: Whether to forbid the insecure functions. :type kwargs: dict :param kwargs: Variables to define in the template. + :rtype: dict + :return: The variables that are defined after execution of the script. """ parser_args = {'no_prompt': True, 'no_escaping': no_escaping, 'secure': secure} with open(filename, encoding='utf-8') as fp: From 0622fe252e094497be1f246691f4b43e0c76d2a8 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 10 Nov 2026 16:02:56 +0100 Subject: [PATCH 075/250] Recognize the prompt of the Junos root shell The junos prompt pattern did not allow the path in csh prompts such as "root@router:~ # ", so a login as root waited until the timeout. The banner from pull request #211 by yoleri joins the OS-guess corpus. --- CHANGES.md | 4 ++ src/Exscript/protocols/drivers/junos.py | 6 +- tests/Exscript/protocols/PromptRegexTest.py | 33 ++++++++++ tests/Exscript/protocols/banners/junos.4 | 4 ++ .../Exscript/protocols/osguess-baseline.json | 60 +++++++++++++++++++ tools/osguess_report.py | 2 +- 6 files changed, 107 insertions(+), 2 deletions(-) create mode 100644 tests/Exscript/protocols/banners/junos.4 diff --git a/CHANGES.md b/CHANGES.md index ff09481e..86af05a6 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -285,6 +285,10 @@ marked with **BREAKING**; a migration guide will accompany the release. local shell) before the connection is closed gracefully, otherwise the logout command may fail. +- **Junos Root Shell:** The `junos` driver recognizes the prompt of the shell that + root gets, e.g. `root@router:~ # ` (csh) or `root@router:/var/tmp # `; before, a + login as root waited until the timeout. Thanks to yoleri (pull request #211), whose + banner is now part of the test corpus. - **Login Errors of `ericsson_ban`:** A login over the `ericsson_ban` driver no longer fails with `AttributeError: 'str' object has no attribute 'search'`; its login error pattern was a string instead of a compiled pattern. A test checks the diff --git a/src/Exscript/protocols/drivers/junos.py b/src/Exscript/protocols/drivers/junos.py index 0a011149..3a8fd0df 100644 --- a/src/Exscript/protocols/drivers/junos.py +++ b/src/Exscript/protocols/drivers/junos.py @@ -53,6 +53,10 @@ # {primary:node0} # pheller@fw1> # +# The shell of root: csh on older versions, the RE number on newer ones: +# root@router:~ # +# root@router:RE:0% +# _user_re = [re.compile(r'(?:\r\n|[\r\n])(?:[\-\w+]+ )?login: $')] _password_re = [re.compile(r'(?:\r\n|[\r\n])(Local )?[Pp]assword: ?$')] @@ -61,7 +65,7 @@ _re_re = r'(?:' + _mb + r'|' + _ps + r')' _edit = r'(?:\[edit[^\]\r\n]*\])' _prefix = r'(?:(?:\r\n|[\r\n])+' + _re_re + r'?' + _edit + r'?)' -_prompt = r'(?:\r\n|[\r\n])+[\w\-\.]+@[\-\w+\.:]+[%>#] $' +_prompt = r'(?:\r\n|[\r\n])+[\w\-\.]+@[\-\w+\.:]++(?:[~/][\w\-\./]*+ ?)?[%>#] $' _prompt_re = [re.compile(_prefix + r'?' + _prompt)] _error_re = [re.compile(r'^(unknown|invalid|error|syntax error)', re.I)] _junos_re = re.compile(r'\bjunos\b', re.I) diff --git a/tests/Exscript/protocols/PromptRegexTest.py b/tests/Exscript/protocols/PromptRegexTest.py index fb2d64a0..21c85752 100644 --- a/tests/Exscript/protocols/PromptRegexTest.py +++ b/tests/Exscript/protocols/PromptRegexTest.py @@ -60,6 +60,39 @@ def testGenericPrompts(self): for text in 'a b c#', 'ab.cd@host#', 'a@host#', 'a:b:c:d#', 'line without prompt': self.assertIsNone(_prompt_re[0].search('output\r\n' + text), text) + def testJunosPrompts(self): + regex = driver_map['junos'].prompt_re[0] + prompts = [ + 'sab@DD-EA3> ', + 'pheller@sw3# ', + '{master:3}\r\npheller@sw3> ', + '{primary:node0}[edit interfaces]\r\npheller@fw1# ', + 'root@router% ', + 'root@router:RE:0% ', + 'root@testhost:~ # ', # csh, the shell of root (upstream pull request #211) + 'root@router:/var/tmp # ', + 'root@router:~# ', + ] + for prompt in prompts: + self.assertTrue(regex.search('output\r\n' + prompt), prompt) + for text in ( + 'sab@DD-EA3>', + 'show route | match 10.0.0.1 ', + 'a@b c # ', + 'line without prompt', + ): + self.assertIsNone(regex.search('output\r\n' + text), text) + + # Everything that the pattern of 2.x matched still matches, at the same position. + old = re.compile(regex.pattern.replace(r'++(?:[~/][\w\-\./]*+ ?)?', '+')) + self.assertNotEqual(old.pattern, regex.pattern) + alphabet = list('ab_1.-@:~/ #>%{}[]') + ['\r\n'] + rng = random.Random(20261110) + for _ in range(20000): + text = '\r\n' + ''.join(rng.choice(alphabet) for _ in range(rng.randint(0, 16))) + if old.search(text): + self.assertEqual(span(old, text), span(regex, text), repr(text)) + def testAllRegexesNeedLinearTime(self): # Lines that are no prompt but contain long runs of the characters that # prompts consist of. A regex with catastrophic backtracking needs diff --git a/tests/Exscript/protocols/banners/junos.4 b/tests/Exscript/protocols/banners/junos.4 new file mode 100644 index 00000000..dba38bf1 --- /dev/null +++ b/tests/Exscript/protocols/banners/junos.4 @@ -0,0 +1,4 @@ +Password: +Last login: Mon Sep 14 17:57:32 2020 from (x.x.x.x) +--- JUNOS 18.4R3.3 Kernel 64-bit JNPR-11.0-20191211.fa5e90e_buil +root@testhost:~ # diff --git a/tests/Exscript/protocols/osguess-baseline.json b/tests/Exscript/protocols/osguess-baseline.json index 6f3b7092..e5ee38ee 100644 --- a/tests/Exscript/protocols/osguess-baseline.json +++ b/tests/Exscript/protocols/osguess-baseline.json @@ -2579,6 +2579,66 @@ 80 ] }, + "junos.4|-|64": { + "after_auth": [ + "junos", + 80 + ], + "final": [ + "junos", + 80 + ] + }, + "junos.4|-|65536": { + "after_auth": [ + "junos", + 80 + ], + "final": [ + "junos", + 80 + ] + }, + "junos.4|SSH-2.0-Cisco-1.25|64": { + "after_auth": [ + "ios", + 80 + ], + "final": [ + "ios", + 80 + ] + }, + "junos.4|SSH-2.0-Cisco-1.25|65536": { + "after_auth": [ + "ios", + 80 + ], + "final": [ + "ios", + 80 + ] + }, + "junos.4|SSH-2.0-OpenSSH_9.6|64": { + "after_auth": [ + "junos", + 80 + ], + "final": [ + "junos", + 80 + ] + }, + "junos.4|SSH-2.0-OpenSSH_9.6|65536": { + "after_auth": [ + "junos", + 80 + ], + "final": [ + "junos", + 80 + ] + }, "junos_erx|-|64": { "after_auth": [ "junos_erx", diff --git a/tools/osguess_report.py b/tools/osguess_report.py index 12aacde3..74bbd5a5 100644 --- a/tools/osguess_report.py +++ b/tools/osguess_report.py @@ -28,7 +28,7 @@ from tests.Exscript.protocols import ReplayTest as replays # noqa: E402 from tests.Exscript.protocols.replay import Recording # noqa: E402 -MIN_CORRECT = 213 # 2026-11-03; raise it when a change guesses more sessions right +MIN_CORRECT = 217 # 2026-11-10; raise it when a change guesses more sessions right def banner_sessions(): From 227f37de84ac324845cd78cb81269b8067cd88ce Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 10 Nov 2026 16:04:48 +0100 Subject: [PATCH 076/250] Recognize the NX-OS banner of newer versions Newer versions print "Cisco NX-OS Software" instead of "Cisco Nexus Operating System (NX-OS) Software" (upstream issue #200). --- CHANGES.md | 2 ++ src/Exscript/protocols/drivers/nxos.py | 5 ++++- tests/Exscript/protocols/OsGuesserTest.py | 6 ++++++ 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 86af05a6..fd443959 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -289,6 +289,8 @@ marked with **BREAKING**; a migration guide will accompany the release. root gets, e.g. `root@router:~ # ` (csh) or `root@router:/var/tmp # `; before, a login as root waited until the timeout. Thanks to yoleri (pull request #211), whose banner is now part of the test corpus. +- **NX-OS Banners:** The `nxos` driver also recognizes the banner "Cisco NX-OS + Software" of newer versions, e.g. nx-osv9000 (upstream issue #200). - **Login Errors of `ericsson_ban`:** A login over the `ericsson_ban` driver no longer fails with `AttributeError: 'str' object has no attribute 'search'`; its login error pattern was a string instead of a compiled pattern. A test checks the diff --git a/src/Exscript/protocols/drivers/nxos.py b/src/Exscript/protocols/drivers/nxos.py index 6cb85222..f6ab4f02 100644 --- a/src/Exscript/protocols/drivers/nxos.py +++ b/src/Exscript/protocols/drivers/nxos.py @@ -68,10 +68,13 @@ def __init__(self) -> None: def check_head_for_os(self, string: str) -> int: """ Returns 95 if the data contains - "Cisco Nexus Operating System (NX-OS) Software", else 0. + "Cisco Nexus Operating System (NX-OS) Software" or, as newer + versions print, "Cisco NX-OS Software", else 0. """ if 'Cisco Nexus Operating System (NX-OS) Software' in string: return 95 + if 'Cisco NX-OS Software' in string: + return 95 return 0 def init_terminal(self, conn: 'Protocol') -> None: diff --git a/tests/Exscript/protocols/OsGuesserTest.py b/tests/Exscript/protocols/OsGuesserTest.py index a8fa42e5..179fa636 100644 --- a/tests/Exscript/protocols/OsGuesserTest.py +++ b/tests/Exscript/protocols/OsGuesserTest.py @@ -134,6 +134,12 @@ def testShellPrompt(self): for prompt in 'router#', 'RP/0/RSP0/CPU0:xr-pe1#', 'A:sros-pe1#', 'admin@junos-mx> ': self.assertNotEqual(guess(response='\r\nshow\r\n' + prompt), 'shell', prompt) + def testNxosBanner(self): + # Newer NX-OS versions, e.g. nx-osv9000, print the short name. + self.assertEqual(guess(head='Cisco NX-OS Software\r\n'), 'nxos') + banner = 'Cisco Nexus Operating System (NX-OS) Software\r\n' + self.assertEqual(guess(head=banner), 'nxos') + def testIsamPrompt(self): self.assertEqual(guess(response='\r\ninfo\r\ntyp:isadmin>#'), 'isam') for prompt in 'router#', 'switch>', 'hostname >', 'S>* 10.0.0.0/8 is directly connected': From 36c9e5655ef2ad26502f019f24f2a106752f75f8 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 10 Nov 2026 16:07:56 +0100 Subject: [PATCH 077/250] Annotate the SSH2 and NETCONF adapters Both modules leave the relaxed mypy override. Only annotations, docstrings and typing.cast() are added; lxml and ncclient are still imported only for NETCONF. --- pyproject.toml | 2 - src/Exscript/protocols/netconf.py | 275 +++++++++++++++++++++++------ src/Exscript/protocols/ssh2.py | 284 +++++++++++++++++++++++------- 3 files changed, 445 insertions(+), 116 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 1f5d9f0f..c581d063 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -135,8 +135,6 @@ module = [ "Exscript.protocols.drivers.vxworks", "Exscript.protocols.drivers.zte", "Exscript.protocols.drivers.zyxel", - "Exscript.protocols.netconf", - "Exscript.protocols.ssh2", "Exscript.queue", "Exscript.servers.httpd", "Exscript.util.sigint", diff --git a/src/Exscript/protocols/netconf.py b/src/Exscript/protocols/netconf.py index 5652161e..e30328f7 100644 --- a/src/Exscript/protocols/netconf.py +++ b/src/Exscript/protocols/netconf.py @@ -31,6 +31,8 @@ import logging import re import time +from collections.abc import Callable, Iterator, Mapping +from typing import TYPE_CHECKING, Any, NoReturn, Self, cast from paramiko.ssh_exception import SSHException @@ -42,6 +44,13 @@ ) from .ssh2 import SSH2 +if TYPE_CHECKING: + from lxml.etree import _Element + from paramiko import Channel, Transport + + from ..account import Account, AccountProxy + from ..util.impl import _Context + # ncclient and lxml are imported when the first NETCONF session starts, not # with Exscript: lxml enables the GIL of free-threaded Python. @@ -61,16 +70,21 @@ @functools.cache -def _install_to_xml(): +def _install_to_xml() -> Callable[..., str]: + """ + Replaces to_xml() in the two ncclient modules that serialize the + and the RPCs with a wrapper that pretty-prints within _pretty(). + Runs once per process, and returns the wrapper. + """ import ncclient.operations.rpc import ncclient.transport.session from ncclient.xml_ import to_xml @functools.wraps(to_xml) - def _to_xml(*args, **kwargs): + def _to_xml(*args: Any, **kwargs: Any) -> str: if _pretty_print.get(): kwargs['pretty_print'] = True - return to_xml(*args, **kwargs) + return cast(str, to_xml(*args, **kwargs)) # ncclient is untyped ncclient.operations.rpc.to_xml = _to_xml ncclient.transport.session.to_xml = _to_xml @@ -78,7 +92,11 @@ def _to_xml(*args, **kwargs): @contextlib.contextmanager -def _pretty(): +def _pretty() -> Iterator[None]: + """ + Makes the to_xml() wrapper of _install_to_xml() pretty-print within this + context, in the current thread or task only. + """ token = _pretty_print.set(True) try: yield @@ -91,21 +109,24 @@ class Netconf(SSH2): The SSH Netconf protocol adapter, based on the SSH2 adapter and ncclient. """ - def __init__(self, **kwargs): + def __init__(self, **kwargs: Any) -> None: + """ + Constructor. Takes the keyword arguments of :class:`Protocol`. + """ SSH2.__init__(self, **kwargs) # Module-Level self._logger = _logger - # Internals - self._ncc_device_handler = None - self._ncc_session = None - self._ncc_manager = None - self._ncc_response = None + # Internals; the ncclient objects are Any, because ncclient is untyped + self._ncc_device_handler: Any = None + self._ncc_session: Any = None + self._ncc_manager: Any = None + self._ncc_response: Any = None self._huge_tree = False self._receives_echo = False # Netconf does not echo the sent RPC. self._replicate = False @property - def huge_tree(self): + def huge_tree(self) -> bool: """Whether huge_tree XML parsing is enabled for RPC replies (default: False). Enabling this disables lxml's security limits for very large XML documents. @@ -114,44 +135,63 @@ def huge_tree(self): return self._huge_tree @huge_tree.setter - def huge_tree(self, value): + def huge_tree(self, value: bool) -> None: self._huge_tree = value if self._ncc_manager is not None: self._ncc_manager.huge_tree = value - def autoinit(self): + def autoinit(self) -> None: """Netconf does not support terminal/shell initialization.""" pass - def get_prompt(self): + def get_prompt(self) -> list[re.Pattern[str]]: + """ + Returns the regular expressions that match the end of a NETCONF + message: the prompt set with set_prompt(), else the netconf_prompt_re + of the driver. + """ if self._manual_prompt_re: return self._manual_prompt_re return self.get_driver().netconf_prompt_re - def get_error_prompt(self): + def get_error_prompt(self) -> list[re.Pattern[str]]: + """ + Returns the regular expressions that are used to monitor the response + for errors: the pattern set with set_error_prompt(), else the + netconf_error_re of the driver, which matches an `` tag. + """ if self._manual_error_re: return self._manual_error_re return self.get_driver().netconf_error_re - def get_client_capabilities(self): - """Returns the client capabilities sent in the `` message.""" + def get_client_capabilities(self) -> list[str] | None: + """ + Returns the client capabilities sent in the `` message, or + None if no NETCONF session is open. + """ if self._ncc_manager is None: return None return list(self._ncc_manager.client_capabilities) - def get_server_capabilities(self): - """Returns the server capabilities received in the `` message.""" + def get_server_capabilities(self) -> list[str] | None: + """ + Returns the server capabilities received in the `` message, or + None if no NETCONF session is open. + """ if self._ncc_manager is None: return None return list(self._ncc_manager.server_capabilities) - def get_session_id(self): - """Returns the session ID received in the `` message.""" + def get_session_id(self) -> str | None: + """ + Returns the session ID received in the `` message, or None + if no NETCONF session is open. + """ if self._ncc_manager is None: return None return str(self._ncc_manager.session_id) - def get_response_element(self): + def get_response_element(self) -> '_Element | None': """ Returns the parsed XML element tree of the latest RPC reply. This might be also some transformed XML structure, depending on specific device @@ -167,7 +207,7 @@ def get_response_element(self): is by design. Please use the get_server_capabilities() method to access a parsed list of hello capabilities instead. - :rtype: lxml.etree.Element | None + :rtype: lxml.etree._Element | None """ from ncclient.operations.rpc import RPCReply from ncclient.xml_ import NCElement @@ -182,8 +222,17 @@ def get_response_element(self): @staticmethod def cleanup_response( - response, *, strip_command=False, strip_chars='\x00\x07\x08', strip_ansi=True - ): + response: str | None, + *, + strip_command: bool = False, + strip_chars: str = '\x00\x07\x08', + strip_ansi: bool = True, + ) -> str: + """ + Returns a clean copy of the given plaintext response, like + :meth:`Protocol.cleanup_response`, but strip_command is ignored: the + first line is never removed, because NETCONF does not echo the RPC. + """ # Netconf does not echo any commands, so there is nothing to strip. return SSH2.cleanup_response( response, @@ -192,7 +241,14 @@ def cleanup_response( strip_ansi=strip_ansi, ) - def _paramiko_channel(self): + def _paramiko_channel(self) -> None: + """ + Opens a channel with the NETCONF subsystem. Tries the subsystem names + of the ncclient device handler for the netconf_device_params of the + driver, in order, and keeps the handler for the session. + + :raises LoginFailure: If no subsystem can be opened. + """ # This is a bit different from the SSH2 adapter. Here we need to init # the Netconf subsystem. We integrate ncclient's device params in the # drivers to support customization. @@ -203,7 +259,7 @@ def _paramiko_channel(self): subsystems = device_handler.get_ssh_subsystem_names() for subname in subsystems: try: - self._channel = self._transport.open_session() + self._channel = cast('Transport', self._transport).open_session() self._channel.invoke_subsystem(subname) self._channel.set_name("%s-subsystem-%s" % (subname, self._channel.get_id())) except SSHException as e: @@ -218,7 +274,16 @@ def _paramiko_channel(self): msg = 'Failed to open a channel with any subsystem: ' + ', '.join(subsystems) raise LoginFailure(msg) - def _init_ncclient(self): + def _init_ncclient(self) -> None: + """ + Starts the NETCONF session with ncclient on the authenticated + transport and the open channel: exchanges the `` messages, + starts the IO thread of ncclient and creates its manager. recv() and + send() of the channel are wrapped, so the data of the session also + reaches the buffer, the logs and the events. Then updates the driver + from the server capabilities and consumes the server `` from + the buffer. Does nothing if the manager exists already. + """ # Bypasses ncclient's default SSH connection flow by injecting the # already-authenticated paramiko transport and the open NETCONF channel # directly into a fresh SSHSession, then calling _post_connect() to @@ -238,8 +303,8 @@ def _init_ncclient(self): session._host = self.host session._transport = self._transport session._channel = self._channel - session._channel_id = self._channel.get_id() - session._channel_name = self._channel.get_name() + session._channel_id = cast('Channel', self._channel).get_id() + session._channel_name = cast('Channel', self._channel).get_name() session._connected = True session._closing.clear() @@ -250,10 +315,10 @@ def _init_ncclient(self): self._replicate = True # Setup the channel receive wrapper. - recv = self._channel.recv + recv = cast('Channel', self._channel).recv @functools.wraps(recv) - def recv_wrapper(*args, **kwargs): + def recv_wrapper(*args: Any, **kwargs: Any) -> bytes: data = recv(*args, **kwargs) if not self._replicate: return data @@ -266,15 +331,15 @@ def recv_wrapper(*args, **kwargs): self._replicate = False return data - self._channel.recv = recv_wrapper + cast(Any, self._channel).recv = recv_wrapper # Any: replaces a method # Setup the channel send wrapper. This is needed as Netconf does not # echo the sent data back to the client known from CLI shells. - send = self._channel.send + send = cast('Channel', self._channel).send param = next(iter(inspect.signature(send).parameters)) # first param @functools.wraps(send) - def send_wrapper(*args, **kwargs): + def send_wrapper(*args: Any, **kwargs: Any) -> int: count = send(*args, **kwargs) if not self._replicate: return count @@ -290,7 +355,7 @@ def send_wrapper(*args, **kwargs): self._replicate = False return count - self._channel.send = send_wrapper + cast(Any, self._channel).send = send_wrapper # Any: replaces a method # Starts IO thread, sends client , consumes server . self._buffer.clear() # keeps the first self.response clean @@ -315,7 +380,24 @@ def send_wrapper(*args, **kwargs): self._update_auto_driver() self.expect_prompt() # consumes the server from the buffer - def login(self, account=None, app_account=None, flush=True): + def login( + self, + account: 'Account | AccountProxy | _Context | None' = None, + app_account: 'Account | AccountProxy | _Context | None' = None, + flush: bool = True, + ) -> None: + """ + Logs into the host like :meth:`Protocol.login`, with the account and + account factory rules described there. NETCONF authenticates at the + protocol level only and then starts the NETCONF session; there is no + app-level login, so the driver's auto-authorization is skipped and + app_account and flush have no effect. The app-level authentication + and authorization are marked as done. + + :param account: The account for protocol level authentication. + :param app_account: Not used by NETCONF. + :param flush: Not used by NETCONF. + """ # Simplified procedure for Netconf to suppress driver auto-authorize. with self._get_account(account) as account: if app_account is None: @@ -323,27 +405,67 @@ def login(self, account=None, app_account=None, flush=True): self.authenticate(account, app_account, flush=False) self.app_authorize() # just to set the flag - def authenticate(self, account=None, app_account=None, flush=True): + def authenticate( + self, + account: 'Account | AccountProxy | _Context | None' = None, + app_account: 'Account | AccountProxy | _Context | None' = None, + flush: bool = True, + ) -> None: + """ + Like :meth:`Protocol.authenticate`, and then starts the NETCONF + session on the authenticated connection: exchanges the `` + messages and consumes the server `` from the buffer. + app_account and flush have no effect, because NETCONF has no + app-level authentication. + + :param account: The account for protocol level authentication. + :param app_account: Not used by NETCONF. + :param flush: Not used by NETCONF. + """ super().authenticate(account, app_account, flush=flush) # Authentication is complete and the subsystem channel is open. The # server waits unread in the channel buffer. Hand the channel to # ncclient now — it will consume the and start its IO thread. self._init_ncclient() - def app_authenticate(self, account=None, flush=True, bailout=False): - """Netconf does not support app-authentication.""" + def app_authenticate( + self, + account: 'Account | AccountProxy | _Context | None' = None, + flush: bool = True, + bailout: bool = False, + ) -> None: + """ + Netconf does not support app-authentication. Only marks it as done, + so is_app_authenticated() returns True; the arguments are not used. + """ self._app_authenticated = True - def app_authorize(self, account=None, flush=True, bailout=False): - """Netconf does not support app-authorization.""" + def app_authorize( + self, + account: 'Account | AccountProxy | _Context | None' = None, + flush: bool = True, + bailout: bool = False, + ) -> None: + """ + Netconf does not support app-authorization. Only marks it as done, + so is_app_authorized() returns True; the arguments are not used. + """ self._app_authorized = True - def auto_app_authorize(self, account=None, flush=True, bailout=False): - """Netconf does not support app-authorization.""" + def auto_app_authorize( + self, + account: 'Account | AccountProxy | _Context | None' = None, + flush: bool = True, + bailout: bool = False, + ) -> None: + """ + Netconf does not support app-authorization. Like app_authorize(), + only marks it as done; the arguments are not used. + """ self.app_authorize() # just to set the flag @staticmethod - def reassemble_chunks(response): + def reassemble_chunks(response: str) -> str: """ Reassembles a NETCONF 1.1 chunked response into plain content. @@ -382,12 +504,22 @@ def reassemble_chunks(response): raise ValueError(msg % repr(response[:50] + '...')) return assembled - def _domatch(self, prompt, flush): + def _domatch(self, prompt: list[re.Pattern[str]], flush: bool) -> tuple[int, re.Match[str]]: + """ + Like Protocol._domatch(), but reassembles a chunked response (NETCONF + 1.1) into plain content. + """ n, match = super()._domatch(prompt, flush) - self.response = self.reassemble_chunks(self.response) + # super()._domatch() sets the response. + self.response = self.reassemble_chunks(cast(str, self.response)) return n, match - def _fill_buffer(self, timeout): + def _fill_buffer(self, timeout: float | None) -> bool | None: + """ + Waits until ncclient's IO thread has appended data to the buffer, for + up to the given time. Returns True if data arrived, None if the time + ran out, and False if the channel was closed. + """ # ncclient's thread reads the channel and appends to the buffer (see the # recv wrapper in _init_ncclient()), so this only waits for new data. size = self._buffer.size() @@ -400,7 +532,17 @@ def _fill_buffer(self, timeout): time.sleep(0.01) return True - def _send(self, data): + def _send(self, data: str) -> None: + """ + Validates the given RPC body, sends it as an RPC with ncclient, and + waits for the reply, which is kept for get_response_element(). The + recv() and send() wrappers of the channel pass the data to the buffer + and the events. + + :raises XMLValidationError: If the RPC body is not valid XML. + :raises TimeoutException: If no reply arrives in time. + :raises ProtocolException: On other errors of ncclient. + """ from ncclient import NCClientError from ncclient.operations.errors import TimeoutExpiredError @@ -421,7 +563,11 @@ def _send(self, data): raise self._dbg(4, 'Received response type: %s' % type(self._ncc_response).__name__) - def _validate_rpc_body(self, data): + def _validate_rpc_body(self, data: str) -> None: + """ + Checks that the given RPC body, wrapped in an `` element, is + valid XML, and raises XMLValidationError if not. + """ from lxml.etree import XMLSyntaxError _wrapper_start_tag = '\n' @@ -450,7 +596,12 @@ def _validate_rpc_body(self, data): ) raise err - def _raise_on_response_with_errors(self, line): + def _raise_on_response_with_errors(self, line: str) -> None: + """ + Raises RPCError if the latest RPC reply has an `` with the + severity error. The matched line is not used: the error comes from + the reply that ncclient parsed. + """ # For device handlers that apply XSLT (e.g. Junos) or any transformation, # dispatch() returns NCElement instead of RPCReply. response = self._ncc_response @@ -459,19 +610,35 @@ def _raise_on_response_with_errors(self, line): if error.severity == 'error': raise RPCError._from_ncclient(rpc_reply.errors) - def _interact(self, key_handlers=None, handle_window_size=True): + def _interact( + self, + key_handlers: Mapping[str, Callable[[Self], object]] | None = None, + handle_window_size: bool = True, + ) -> NoReturn: + """ + Raises NotImplementedError: NETCONF has no interactive shell. + """ # Netconf does not support interactive shells. User should use # native `ssh HOST -p 830 -s netconf` for an interactive session. raise NotImplementedError() - def _logout(self): + def _logout(self) -> None: + """ + Closes the NETCONF session with a `` RPC; errors are + ignored. + """ try: with _pretty(): self._ncc_manager.close_session() except Exception: pass - def _close(self, force=False): + def _close(self, force: bool = False) -> None: + """ + Tells ncclient's IO thread that the session is closed on purpose, and + closes the channel, the transport and the socket like SSH2._close(). + Then waits up to 0.3 seconds for the IO thread to end. + """ # Clear the ncclient manager reference. if self._ncc_manager: self._ncc_manager = None diff --git a/src/Exscript/protocols/ssh2.py b/src/Exscript/protocols/ssh2.py index ddf299a4..a2656a63 100644 --- a/src/Exscript/protocols/ssh2.py +++ b/src/Exscript/protocols/ssh2.py @@ -30,6 +30,8 @@ import socket import threading from binascii import hexlify +from collections.abc import Callable, Iterable, Mapping +from typing import Any, Self, cast import paramiko from paramiko.ssh_exception import ( @@ -51,8 +53,13 @@ _READ_SIZE = 65536 -def _prefer_key_type(options, keytype): - """Makes the server present a key of the known type, like paramiko's SSHClient.""" +def _prefer_key_type(options: paramiko.SecurityOptions, keytype: str | None) -> None: + """ + Makes the server present a key of the known type, like paramiko's + SSHClient: moves the type, or for 'ssh-rsa' the first offered of + rsa-sha2-512, rsa-sha2-256 and ssh-rsa, to the front of the host key + types that the client offers. Changes nothing if the type is not offered. + """ types = list(options.key_types) if keytype == 'ssh-rsa': # RSA keys are negotiated with SHA-2 signatures keytype = next((t for t in ('rsa-sha2-512', 'rsa-sha2-256', 'ssh-rsa') if t in types), None) @@ -61,7 +68,11 @@ def _prefer_key_type(options, keytype): # Register supported key types. -keymap = {'rsa': paramiko.RSAKey, 'ecdsa': paramiko.ECDSAKey, 'ed25519': paramiko.Ed25519Key} +keymap: dict[str, type[paramiko.PKey]] = { + 'rsa': paramiko.RSAKey, + 'ecdsa': paramiko.ECDSAKey, + 'ed25519': paramiko.Ed25519Key, +} for key in keymap: PrivateKey.keytypes.add(key) @@ -79,14 +90,19 @@ class SSH2(Protocol): KEEPALIVE_INTERVAL = 2.5 * 60 # Two and a half minutes - def __init__(self, **kwargs): + def __init__(self, **kwargs: Any) -> None: + """ + Constructor. Takes the keyword arguments of :class:`Protocol`. If + preferred_authentications is empty or not given, the methods + publickey, keyboard-interactive and password are tried, in this order. + """ Protocol.__init__(self, **kwargs) # Module-Level self._logger = _logger - # Internals - self._sock = None - self._transport = None - self._channel = None + # Internals; cast where a connection is required + self._sock: socket.socket | None = None + self._transport: paramiko.Transport | None = None + self._channel: paramiko.Channel | None = None # Set default preferred auth types, that are supported. if not self.preferred_authentications: @@ -99,16 +115,24 @@ def __init__(self, **kwargs): # Host keys accepted for unknown hosts, pinned for reconnects of this object. self._host_keys = paramiko.HostKeys() - def _host_key_name(self): + def _host_key_name(self) -> str: + """ + Returns the name of the host in known_hosts files: the host, or + "[host]:port" for other ports than 22. + """ # OpenSSH and paramiko store hosts on other ports than 22 as "[host]:port". if self.port in (None, 22): - return self.host + return cast(str, self.host) # connect() checks that the host is set return f'[{self.host}]:{self.port}' - def _known_hosts_file(self): + def _known_hosts_file(self) -> str: + """ + Returns the path of the known_hosts file: the known_hosts option, or + ~/.ssh/known_hosts, with ~ expanded. + """ return os.path.expanduser(self.known_hosts or '~/.ssh/known_hosts') - def _load_known_hosts(self): + def _load_known_hosts(self) -> paramiko.HostKeys: """Returns the keys in the known_hosts file; a missing file has none.""" keys = paramiko.HostKeys() try: @@ -117,10 +141,17 @@ def _load_known_hosts(self): pass return keys - def _check_host_key(self, key, name, known): + def _check_host_key( + self, key: paramiko.PKey, name: str, known: Mapping[str, paramiko.PKey] | None + ) -> None: """ Raises BadHostKeyException if the key differs from the known keys of - the host. If the host has no known keys, applies the host_key_policy. + the host. If the host has no known keys, applies the host_key_policy; + a key that it accepts is kept for reconnects of this object. + + :param key: The host key that the server presented. + :param name: The name of the host in known_hosts, see _host_key_name(). + :param known: The known keys of the host by key type, if any. """ if known: expected = known.get(key.get_name()) @@ -137,7 +168,12 @@ def _check_host_key(self, key, name, known): self._logger.warning('Accepting %s (not in known_hosts)', msg) self._host_keys.add(name, key.get_name(), key) - def _append_known_host(self, name, key): + def _append_known_host(self, name: str, key: paramiko.PKey) -> None: + """ + Appends the key of the host to the known_hosts file. Creates the file + and its directory if needed, and starts a new line if the file does + not end with one. + """ filename = self._known_hosts_file() line = f'{name} {key.get_name()} {key.get_base64()}\n' # shortcut: the lock serializes threads only; use separate files for parallel processes. @@ -150,14 +186,25 @@ def _append_known_host(self, name, key): line = '\n' + line fp.write(line.encode()) - def _paramiko_connect(self): + def _paramiko_connect(self) -> paramiko.Transport: + """ + Connects to the host, trying all of its addresses, and starts the SSH + transport. If verify_fingerprint is set, checks the host key before + any credentials are sent; if the check fails, closes the transport + and raises the error. + + :return: The transport. + """ # Tries all addresses of the host, e.g. IPv4 after IPv6. - self._sock = socket.create_connection((self.host, self.port), self.connect_timeout or None) + self._sock = socket.create_connection( + (self.host, cast(int, self.port)), # connect() checks that the port is set + self.connect_timeout or None, + ) # Init the paramiko protocol. t = paramiko.Transport(self._sock, disabled_algorithms=self.disabled_algorithms) t.banner_timeout = self.banner_timeout - t.name = self.host + t.name = cast(str, self.host) # connect() checks that the host is set if self.debug < 5: t.logger.disabled = True if self.debug > 5: @@ -166,7 +213,7 @@ def _paramiko_connect(self): # The known keys are loaded before the key exchange, so that the key is # checked before any credentials are sent. name = self._host_key_name() - known = None + known: Mapping[str, paramiko.PKey] | None = None if self.verify_fingerprint: known = self._load_known_hosts().lookup(name) or self._host_keys.lookup(name) if known: @@ -182,14 +229,26 @@ def _paramiko_connect(self): t.set_keepalive(self.KEEPALIVE_INTERVAL) return t - def _paramiko_auth_none(self, username, password=None): - self._transport.auth_none(username) + def _paramiko_auth_none(self, username: str, password: str | None = None) -> None: + """ + Authenticates with the "none" method; the password is not used. + """ + cast(paramiko.Transport, self._transport).auth_none(username) - def _paramiko_auth_interactive(self, username, password=None): + def _paramiko_auth_interactive(self, username: str, password: str | None = None) -> None: + """ + Authenticates with the keyboard-interactive method: answers S/Key and + OTP challenges with a one-time password computed from the password, + and password prompts with the password. Passes the answered prompts, + but not the answers, to the received data. Does nothing without a + password. + """ if password is None: return - def handler(title, instructions, prompt_list): + def handler( + title: str, instructions: str, prompt_list: list[tuple[str, bool]] + ) -> list[str]: if not prompt_list: return [] response = [] @@ -215,43 +274,66 @@ def handler(title, instructions, prompt_list): break return response - self._transport.auth_interactive(username, handler) + cast(paramiko.Transport, self._transport).auth_interactive(username, handler) - def _paramiko_auth_password(self, username, password): + def _paramiko_auth_password(self, username: str, password: str | None) -> None: + """ + Authenticates with the password method; None is sent as an empty + password. Passes a password prompt to the received data. + """ self._receive_cb('Password: \n') # Expose a prompt (hide the PW). - self._transport.auth_password(username, password or '') + cast(paramiko.Transport, self._transport).auth_password(username, password or '') - def _paramiko_auth_agent(self, username, password=None): + def _paramiko_auth_agent(self, username: str, password: str | None = None) -> None: + """ + Authenticates with the keys of the SSH agent, trying one after the + other until the server accepts one. Raises the error of the last key, + or AuthenticationException if the agent has no keys. The password is + not used. + """ keys = paramiko.Agent().get_keys() if not keys: raise AuthenticationException('auth agent found no keys') - saved_exception = AuthenticationException('Failed to authenticate with given username') + saved_exception: SSHException = AuthenticationException( + 'Failed to authenticate with given username' + ) for key in keys: try: - fp = hexlify(key.get_fingerprint()) + fp: object = hexlify(key.get_fingerprint()) # bytes; the log shows b'...' self._dbg(2, 'Trying SSH agent key %s' % fp) - self._transport.auth_publickey(username, key) + cast(paramiko.Transport, self._transport).auth_publickey(username, key) return except SSHException as e: saved_exception = e raise saved_exception - def _paramiko_auth_key(self, username, keys, password): + def _paramiko_auth_key( + self, username: str, keys: Iterable[tuple[type[paramiko.PKey], str]], password: str | None + ) -> None: + """ + Authenticates with the given private key files, trying one after the + other until the server accepts one. Raises the error of the last key + that failed, or AuthenticationException if there is no key. + + :param keys: The key class and the file name of each key. + :param password: The password of the key files; None is used as an + empty password. + """ if password is None: password = '' - saved_exception = AuthenticationException( + saved_exception: SSHException | OSError = AuthenticationException( 'Failed to authenticate with given username and password/key' ) for pkey_class, filename in keys: try: key = pkey_class.from_private_key_file(filename, password) - fp = hexlify(key.get_fingerprint()) + fp: object = hexlify(key.get_fingerprint()) # bytes; the log shows b'...' self._dbg(2, 'Trying key %s in %s' % (fp, filename)) - self._transport.auth_publickey(username, key) + cast(paramiko.Transport, self._transport).auth_publickey(username, key) return except SSHException as e: saved_exception = e @@ -259,7 +341,11 @@ def _paramiko_auth_key(self, username, keys, password): saved_exception = e raise saved_exception - def _paramiko_auth_autokey(self, username, password): + def _paramiko_auth_autokey(self, username: str, password: str | None) -> None: + """ + Like _paramiko_auth_key(), with the RSA, ECDSA and Ed25519 key files + in ~/.ssh (Unix) and ~/ssh (Windows) that exist. + """ keyfiles = [] for cls, file in ( (paramiko.RSAKey, '~/.ssh/id_rsa'), # Unix @@ -274,7 +360,19 @@ def _paramiko_auth_autokey(self, username, password): keyfiles.append((cls, file)) self._paramiko_auth_key(username, keyfiles, password) - def _get_auth_methods(self, allowed_types): + def _get_auth_methods( + self, allowed_types: list[str] + ) -> list[Callable[[str, str | None], None]]: + """ + Returns the methods for the authentication types to try, in order: the + preferred_authentications that the server allows, or all allowed + types if preferred_authentications is empty. Types without a method + in auth_types are skipped. + + :param allowed_types: The authentication types that the server allows. + :raises BadAuthenticationType: If the server allows none of the + preferred_authentications. + """ if self.preferred_authentications: self._dbg(3, 'Preferred auth. types: %s' % self.preferred_authentications) requested_types = [] @@ -300,12 +398,23 @@ def _get_auth_methods(self, allowed_types): auth_methods.append(getattr(self, type_name)) return auth_methods - def _paramiko_auth(self, username, password): + def _paramiko_auth(self, username: str, password: str | None) -> None: + """ + Authenticates the user. First tries the "none" method, and returns if + the server accepts it; else the server tells the authentication types + that it allows. The authentication banner of the server is passed to + the received data. Then tries the methods of _get_auth_methods() until + one succeeds. Before a method, reconnects if the transport is no + longer active, or if the driver sets reconnect_between_auth_methods + and a request reached the server since the last connect. + + :raises LoginFailure: If no method succeeds; lists the error of each. + """ # Try authentication using auth_none. This should (almost) always fail, # but provides us with info about allowed authentication types. try: self._dbg(2, 'Authenticating with client.auth_none().') - self._transport.auth_none(username) + cast(paramiko.Transport, self._transport).auth_none(username) except BadAuthenticationType as err: self._dbg( 2, 'Authentication with client.auth_none() failed, allowed: %s' % err.allowed_types @@ -328,7 +437,8 @@ def _paramiko_auth(self, username, password): # login attempts after failing one. So in this hack, we re-connect # after each attempt that reached the server, auth_none() included. reconnect = self.get_driver().reconnect_between_auth_methods and attempted - if reconnect or not self._transport.active: + # The stubs lack Transport.active. + if reconnect or not cast(Any, self._transport).active: if reconnect: self._dbg(1, 'Forced reconnect by driver.') else: @@ -342,7 +452,7 @@ def _paramiko_auth(self, username, password): self._dbg(2, 'Authenticating with %s().' % method.__name__) # paramiko creates a new auth handler for each request that it sends; # without keys, the public key methods send none. - handler = self._transport.auth_handler + handler = cast(paramiko.Transport, self._transport).auth_handler try: method(username, password) return @@ -358,12 +468,21 @@ def _paramiko_auth(self, username, password): msg = 'SSHException with %s(): %s' % (method.__name__, str(e)) self._dbg(1, msg) errors.append(msg) - attempted = attempted or self._transport.auth_handler is not handler + attempted = ( + attempted or cast(paramiko.Transport, self._transport).auth_handler is not handler + ) raise LoginFailure('Login failed: ' + '; '.join(errors)) - def _paramiko_channel(self): + def _paramiko_channel(self) -> None: + """ + Opens a session channel with a pseudo-terminal of the terminal type + and size of this object, and starts a shell on it. + + :raises LoginFailure: If the channel, the terminal or the shell + cannot be opened. + """ try: - self._channel = self._transport.open_session() + self._channel = cast(paramiko.Transport, self._transport).open_session() self._channel.get_pty(self.termtype, self.term_width, self.term_height) self._channel.invoke_shell() self._channel.set_name("pty-shell-%s" % self._channel.get_id()) @@ -374,17 +493,29 @@ def _paramiko_channel(self): # Decodes characters that span two reads, too. self._decoder = codecs.getincrementaldecoder(self.encoding)() - def _connect(self): + def _connect(self) -> bool: + """ + Connects to the host, starts the SSH transport, which checks the host + key, and returns True. + """ self._transport = self._paramiko_connect() return True - def _protocol_authenticate(self, user, password): + def _protocol_authenticate(self, user: str, password: str | None) -> None: + """ + Authenticates with the user name and the password, and opens the + shell channel. + """ self._paramiko_auth(user, password) self._paramiko_channel() - def _protocol_authenticate_by_key(self, user, key): + def _protocol_authenticate_by_key(self, user: str, key: PrivateKey) -> None: + """ + Authenticates with the user name and the key file of the given key, + decrypted with the password of the key, and opens the shell channel. + """ # Allow multiple key files. - key_file = key.get_filename() + key_file: str | list[str] | None = key.get_filename() if key_file is None: key_file = [] elif isinstance(key_file, str): @@ -399,7 +530,12 @@ def _protocol_authenticate_by_key(self, user, key): # Open the channel. self._paramiko_channel() - def get_banner(self): + def get_banner(self) -> str | None: + """ + Returns the authentication banner that the server sent, decoded with + the encoding of this object. Returns None if the server sent no + banner or if the adapter is not connected. + """ if not self._transport: return None banner = self._transport.get_banner() @@ -407,7 +543,11 @@ def get_banner(self): return banner.decode(self.encoding) return banner - def get_remote_version(self): + def get_remote_version(self) -> str | None: + """ + Returns the version string that the server sent, such as + 'SSH-2.0-OpenSSH_9.9', or None if the adapter is not connected. + """ if not self._transport: return None version = self._transport.remote_version @@ -415,35 +555,59 @@ def get_remote_version(self): return version.decode(self.encoding) return version - def _send(self, data): + def _send(self, data: str) -> None: + """ + Sends the given text, encoded with the encoding of this object. + """ try: - self._channel.sendall(data.encode(self.encoding)) + cast(paramiko.Channel, self._channel).sendall(data.encode(self.encoding)) self._send_cb(data) except Exception: self._dbg(1, 'Error while writing to connection') raise - def _fill_buffer(self, timeout): - self._channel.settimeout(timeout) + def _fill_buffer(self, timeout: float | None) -> bool | None: + """ + Waits for data from the host and reads it once. Decodes it + incrementally, so a character that spans two reads is decoded, too. + """ + cast(paramiko.Channel, self._channel).settimeout(timeout) try: - data = self._channel.recv(_READ_SIZE) + data = cast(paramiko.Channel, self._channel).recv(_READ_SIZE) except TimeoutError: return None finally: - self._channel.settimeout(None) # sendall() blocks without limit, as before + # sendall() blocks without limit + cast(paramiko.Channel, self._channel).settimeout(None) text = self._decoder.decode(data, final=not data) if text: self._receive_cb(text) self._buffer.append(text) return bool(data) - def _set_terminal_size(self, rows, cols): - self._channel.resize_pty(cols, rows) + def _set_terminal_size(self, rows: int, cols: int) -> None: + """ + Changes the size of the pseudo-terminal. + """ + cast(paramiko.Channel, self._channel).resize_pty(cols, rows) - def _interact(self, key_handlers=None, handle_window_size=True): - return self._open_shell(self._channel, key_handlers, handle_window_size) + def _interact( + self, + key_handlers: Mapping[str, Callable[[Self], object]] | None = None, + handle_window_size: bool = True, + ) -> None: + """ + Opens the interactive shell on the channel. + """ + return self._open_shell( + cast(paramiko.Channel, self._channel), key_handlers, handle_window_size + ) - def _close(self, force=False): + def _close(self, force: bool = False) -> None: + """ + Closes the channel, the transport and the socket. Unless forced, + first reads the data that is waiting on the channel, if any. + """ if self._channel: if not force: try: From d6bb1dc76f7128c4162a8019f682ffadfbc5872c Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 10 Nov 2026 16:23:37 +0100 Subject: [PATCH 078/250] Annotate the OpenTelemetry instrumentation and three drivers Also add docstrings, and check Exscript.contrib.otel and the vxworks, zte and zyxel drivers strictly. The OTel module may subclass BaseInstrumentor, which comes from a module that mypy skips. --- pyproject.toml | 9 ++-- src/Exscript/contrib/otel.py | 66 +++++++++++++++++++---- src/Exscript/protocols/drivers/vxworks.py | 46 ++++++++++++++-- src/Exscript/protocols/drivers/zte.py | 40 ++++++++++++-- src/Exscript/protocols/drivers/zyxel.py | 23 ++++++-- 5 files changed, 157 insertions(+), 27 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index c581d063..41927c24 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -131,10 +131,6 @@ ignore_missing_imports = true # ncclient and others ship no type hints # Modules that are not annotated yet (P8). Remove a module from this list when # it passes the strict checks; the list only shrinks. module = [ - "Exscript.contrib.otel", - "Exscript.protocols.drivers.vxworks", - "Exscript.protocols.drivers.zte", - "Exscript.protocols.drivers.zyxel", "Exscript.queue", "Exscript.servers.httpd", "Exscript.util.sigint", @@ -156,6 +152,11 @@ implicit_reexport = true strict_equality = false extra_checks = false +[[tool.mypy.overrides]] +# BaseInstrumentor comes from a module that mypy skips (it starts with "# type: ignore"). +module = "Exscript.contrib.otel" +disallow_subclassing_any = false + [tool.interrogate] # Docstring coverage (P8). Private functions count; other dunder methods than # __init__, nested functions and property setters do not. Raise fail-under as diff --git a/src/Exscript/contrib/otel.py b/src/Exscript/contrib/otel.py index 25eb6c6b..54a76aed 100644 --- a/src/Exscript/contrib/otel.py +++ b/src/Exscript/contrib/otel.py @@ -65,8 +65,8 @@ from opentelemetry.instrumentation.instrumentor import ( # type: ignore[attr-defined] BaseInstrumentor, # mypy cannot resolve it in this namespace package ) -from opentelemetry.metrics import get_meter -from opentelemetry.trace import Span, SpanKind, get_tracer +from opentelemetry.metrics import Histogram, get_meter +from opentelemetry.trace import Span, SpanKind, Tracer, get_tracer from Exscript.protocols import Protocol from Exscript.version import __version__ @@ -85,7 +85,7 @@ ) -def _protocol_classes(): +def _protocol_classes() -> list[type[Protocol]]: """Returns Protocol and its subclasses, which may override instrumented methods.""" classes, todo = [], [Protocol] while todo: @@ -95,12 +95,20 @@ def _protocol_classes(): return classes -def _truncate(value, limit): +def _truncate(value: object, limit: int) -> str: + """ + Returns the given value as a string, cut to the given number of + characters and "..." if it is longer. + """ value = str(value) return value if len(value) <= limit else value[:limit] + '...' -def _set_connection_attributes(span, conn): +def _set_connection_attributes(span: Span, conn: Protocol) -> None: + """ + Sets the address and the port of the given connection on the span, if + they are known, and the guessed OS, once it is known. + """ if conn.host: span.set_attribute('server.address', str(conn.host)) if conn.port: @@ -110,12 +118,33 @@ def _set_connection_attributes(span, conn): span.set_attribute('exscript.os', os_name) -def _make_wrapper(original, operation, tracer, histogram, options): +def _make_wrapper( + original: Callable[..., Any], + operation: str, + tracer: Tracer, + histogram: Histogram, + options: dict[str, Any], +) -> Callable[..., Any]: + """ + Returns a wrapper of the given method of the protocol adapters, which + calls the method in a client span and records its duration in the + histogram, as the module documentation describes. The wrapper calls the + method without a span if the instrumentation is disabled for the + adapter, if the same operation is in progress on the adapter (an + override that calls super()), or if the arguments do not fit the method. + + :param original: The method, as the class defines it. + :param operation: The name of the method, e.g. "execute". + :param tracer: The tracer that creates the spans. + :param histogram: The histogram of the durations. + :param options: The options of instrument(): capture_payload, + max_payload_length, request_hook and response_hook. + """ signature = inspect.signature(original) capture, limit = options['capture_payload'], options['max_payload_length'] @functools.wraps(original) - def wrapper(conn, *args, **kwargs): + def wrapper(conn: Protocol, *args: Any, **kwargs: Any) -> Any: key = (id(conn), operation) if getattr(conn, _DISABLED, False) or _active.get() == key: return original(conn, *args, **kwargs) # e.g. an override that calls super() @@ -166,9 +195,19 @@ class ExscriptInstrumentor(BaseInstrumentor): """Instruments all protocol adapters; see the module documentation.""" def instrumentation_dependencies(self) -> Collection[str]: + """ + Returns the packages that instrument() requires, unless its argument + skip_dep_check is true: exscript, in any version. + """ return ('exscript',) - def _instrument(self, **kwargs): + def _instrument(self, **kwargs: Any) -> None: + """ + Wraps the methods connect(), login(), authenticate(), send(), + execute() and close() of Protocol and its subclasses, where a class + defines them and they are not wrapped yet. instrument() calls it with + its keyword arguments, the options in the module documentation. + """ tracer = get_tracer(__name__, __version__, kwargs.get('tracer_provider')) meter = get_meter(__name__, __version__, kwargs.get('meter_provider')) histogram = meter.create_histogram( @@ -190,14 +229,19 @@ def _instrument(self, **kwargs): wrapper = _make_wrapper(method, operation, tracer, histogram, options) setattr(cls, operation, wrapper) - def _uninstrument(self, **kwargs): + def _uninstrument(self, **kwargs: Any) -> None: + """ + Restores the methods that _instrument() wrapped. uninstrument() calls + it with its keyword arguments, which it ignores. + """ for cls in _protocol_classes(): for operation in _OPERATIONS: - method = cls.__dict__.get(operation) + # Any: getattr() below does not narrow out the None of a missing name. + method: Any = cls.__dict__.get(operation) if getattr(method, _APPLIED, False): setattr(cls, operation, method.__wrapped__) @staticmethod - def uninstrument_protocol(protocol): + def uninstrument_protocol(protocol: Protocol) -> None: """Disables the instrumentation for the given protocol adapter.""" setattr(protocol, _DISABLED, True) diff --git a/src/Exscript/protocols/drivers/vxworks.py b/src/Exscript/protocols/drivers/vxworks.py index f0e0a92a..893b3b45 100644 --- a/src/Exscript/protocols/drivers/vxworks.py +++ b/src/Exscript/protocols/drivers/vxworks.py @@ -21,13 +21,19 @@ # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ -A driver for devices running Vxworks, which can be found on huawei5600T +A driver for devices running VxWorks, which can be found on the Huawei MA5600T. """ import re +from typing import TYPE_CHECKING from .driver import Driver +if TYPE_CHECKING: + from ...account import Account, AccountProxy + from ...util.impl import _Context + from ..protocol import Protocol + _user_re = [re.compile(r'user ?name:', re.I)] _password_re = [re.compile(r'(?:User )?Password:', re.I)] _prompt_re = [ @@ -44,23 +50,53 @@ class VxworksDriver(Driver): - def __init__(self): + """ + The ``vxworks`` driver for devices running VxWorks, such as the Huawei + MA5600T. Its prompt is a name with an optional mode in parentheses, + followed by ">" or "#", e.g. "hostname>" or "hostname(config)#"; the + question "(y/n)[n]" counts as a prompt, too. + """ + + def __init__(self) -> None: + """ + Constructor. Sets the patterns of the user name prompt "User name:", + the password prompts "User password:" and "Password:", the command + prompt and the error messages, such as "%Error", "Unknown command", + "Failure" and "Parameter error". + """ Driver.__init__(self, 'vxworks') self.user_re = _user_re self.password_re = _password_re self.prompt_re = _prompt_re self.error_re = _error_re - def auto_authorize(self, conn, account, flush, bailout): + def auto_authorize( + self, + conn: 'Protocol', + account: 'Account | AccountProxy | _Context | None', + flush: bool, + bailout: bool, + ) -> None: + """ + Sends "enable", then authorizes with conn.app_authorize(). + """ conn.send('enable\r\n') conn.app_authorize(account, flush, bailout) - def check_head_for_os(self, string): + def check_head_for_os(self, string: str) -> int: + """ + Returns 90 if the data contains "Huawei Integrated Access Software" in + any case, else 0. + """ if _huawei_re.search(string): return 90 return 0 - def check_response_for_os(self, string): + def check_response_for_os(self, string: str) -> int: + """ + Returns 90 if the data contains "User last login information" in any + case, else 0. + """ if _banner_re.search(string): return 90 return 0 diff --git a/src/Exscript/protocols/drivers/zte.py b/src/Exscript/protocols/drivers/zte.py index 5f2992ec..577f8d58 100644 --- a/src/Exscript/protocols/drivers/zte.py +++ b/src/Exscript/protocols/drivers/zte.py @@ -21,13 +21,19 @@ # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ -A driver for devices running Zte operating system. +A driver for devices running the ZTE operating system. """ import re +from typing import TYPE_CHECKING from .driver import Driver +if TYPE_CHECKING: + from ...account import Account, AccountProxy + from ...util.impl import _Context + from ..protocol import Protocol + _user_re = [re.compile(r'user ?name:', re.I), re.compile(r'login', re.I)] _password_re = [re.compile(r'(?:User )?Password:', re.I)] _prompt_re = [ @@ -44,18 +50,44 @@ class ZteDriver(Driver): - def __init__(self): + """ + The ``zte`` driver for devices running the ZTE operating system. Its + prompt is a name, which may contain parentheses, with an optional mode in + parentheses, followed by ">", "#" or "$", e.g. "hostname#"; the question + "(y/n)[n]" counts as a prompt, too, and so does the password prompt that + "enable" shows. + """ + + def __init__(self) -> None: + """ + Constructor. Sets the patterns of the user name prompt, "Username:" + or any text with "login", the password prompts "Password:" and "User + password:", the command prompts and the error messages, such as + "%Error", "Unrecognized command" and "Invalid input". + """ Driver.__init__(self, 'zte') self.user_re = _user_re self.password_re = _password_re self.prompt_re = _prompt_re self.error_re = _error_re - def auto_authorize(self, conn, account, flush, bailout): + def auto_authorize( + self, + conn: 'Protocol', + account: 'Account | AccountProxy | _Context | None', + flush: bool, + bailout: bool, + ) -> None: + """ + Sends "enable", then authorizes with conn.app_authorize(). + """ conn.send('enable\r\n') conn.app_authorize(account, flush, bailout) - def check_head_for_os(self, string): + def check_head_for_os(self, string: str) -> int: + """ + Returns 90 if the data contains "ZTE" in any case, else 0. + """ if _zte_re.search(string): return 90 return 0 diff --git a/src/Exscript/protocols/drivers/zyxel.py b/src/Exscript/protocols/drivers/zyxel.py index 4f62e40a..3e3e58b2 100644 --- a/src/Exscript/protocols/drivers/zyxel.py +++ b/src/Exscript/protocols/drivers/zyxel.py @@ -21,7 +21,7 @@ # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ -A driver for devices running Zyxel operating system. +A driver for devices running the Zyxel operating system. """ import re @@ -37,11 +37,28 @@ class ZyxelDriver(Driver): - def __init__(self): + """ + The ``zyxel`` driver for devices running the Zyxel operating system. Its + prompt is a name, which may contain parentheses, with an optional mode in + parentheses, followed by ">", "#" or "$" and optionally by a space and the + escape sequence ESC 7, e.g. "hostname#"; the question "(y/n)[n]" counts + as a prompt, too, and so does the password prompt that "enable" shows. + """ + + def __init__(self) -> None: + """ + Constructor. Sets the patterns of the command prompts; the user name, + password and error patterns are the generic ones. + """ Driver.__init__(self, 'zyxel') self.prompt_re = _prompt_re - def check_response_for_os(self, string): + def check_response_for_os(self, string: str) -> int: + """ + Returns 90 if the data contains a line with "Copyright" and "ZyXEL + Communications Corp." in any case, e.g. "Copyright (c) 1994 - 2013 + ZyXEL Communications Corp.", else 0. + """ if _zyxel_re.search(string): return 90 return 0 From 5faa87e85fedc8c1ef910cc0e09351956d62d252 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 10 Nov 2026 16:26:54 +0100 Subject: [PATCH 079/250] Log the fingerprints of SSH keys like OpenSSH The keys that SSH2 tries were logged with their MD5 fingerprint as a bytes literal, which raises BytesWarning under python -bb. Log the SHA256 fingerprint, as OpenSSH does. --- CHANGES.md | 3 +++ src/Exscript/protocols/ssh2.py | 7 ++----- tests/Exscript/protocols/SSH2Test.py | 23 +++++++++++++++++++++++ 3 files changed, 28 insertions(+), 5 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index fd443959..140b6b7f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -139,6 +139,9 @@ marked with **BREAKING**; a migration guide will accompany the release. `PreferredAuthentications`. The default order is `publickey`, `keyboard-interactive`, `password`; methods that the server does not offer are skipped. +- **SSH Key Fingerprints in Debug Logs:** The keys that are tried for a login are + logged with their SHA256 fingerprints, as `ssh-add -l` and `ssh-keygen -l` show + them. Before, the MD5 fingerprint was logged as a Python bytes literal. - **BREAKING - Terminal Size:** SSH and Telnet sessions get a terminal of 511 columns and 1000 lines; the new options `term_width` and `term_height` (also `Host` options) choose another size. Before, Exscript used the size of the terminal that diff --git a/src/Exscript/protocols/ssh2.py b/src/Exscript/protocols/ssh2.py index a2656a63..03a9ba74 100644 --- a/src/Exscript/protocols/ssh2.py +++ b/src/Exscript/protocols/ssh2.py @@ -29,7 +29,6 @@ import os import socket import threading -from binascii import hexlify from collections.abc import Callable, Iterable, Mapping from typing import Any, Self, cast @@ -301,8 +300,7 @@ def _paramiko_auth_agent(self, username: str, password: str | None = None) -> No for key in keys: try: - fp: object = hexlify(key.get_fingerprint()) # bytes; the log shows b'...' - self._dbg(2, 'Trying SSH agent key %s' % fp) + self._dbg(2, 'Trying SSH agent key %s' % key.fingerprint) cast(paramiko.Transport, self._transport).auth_publickey(username, key) return except SSHException as e: @@ -331,8 +329,7 @@ def _paramiko_auth_key( for pkey_class, filename in keys: try: key = pkey_class.from_private_key_file(filename, password) - fp: object = hexlify(key.get_fingerprint()) # bytes; the log shows b'...' - self._dbg(2, 'Trying key %s in %s' % (fp, filename)) + self._dbg(2, 'Trying key %s in %s' % (key.fingerprint, filename)) cast(paramiko.Transport, self._transport).auth_publickey(username, key) return except SSHException as e: diff --git a/tests/Exscript/protocols/SSH2Test.py b/tests/Exscript/protocols/SSH2Test.py index dabe6832..073bd89f 100644 --- a/tests/Exscript/protocols/SSH2Test.py +++ b/tests/Exscript/protocols/SSH2Test.py @@ -210,6 +210,28 @@ def testNoReconnectByDefault(self): self.assertEqual(self.handshakes('generic'), 1) +class FingerprintTest(unittest.TestCase): + """The keys that are tried are logged with their fingerprints in the format of OpenSSH.""" + + def logs(self, method, *args): + conn = SSH2(debug=2) + conn._transport = mock.Mock() + with self.assertLogs('Exscript.protocols.ssh2', 'DEBUG') as logs: + method(conn, 'admin', *args) + return logs.output + + def testKeyFile(self): + output = self.logs(SSH2._paramiko_auth_key, [(paramiko.RSAKey, keyfile)], None) + self.assertIn(paramiko.RSAKey(filename=keyfile).fingerprint, output[0]) + + def testAgentKey(self): + agent_key = paramiko.RSAKey(filename=keyfile) + with mock.patch('paramiko.Agent') as agent: + agent.return_value.get_keys.return_value = [agent_key] + output = self.logs(SSH2._paramiko_auth_agent) + self.assertIn(agent_key.fingerprint, output[0]) + + class ConnectTest(unittest.TestCase): def testTriesAllAddresses(self): server = KeyServer(self, paramiko.RSAKey(filename=keyfile)) @@ -268,6 +290,7 @@ def suite(): loader.loadTestsFromTestCase(SSH2Test), loader.loadTestsFromTestCase(HostKeyTest), loader.loadTestsFromTestCase(ReconnectTest), + loader.loadTestsFromTestCase(FingerprintTest), loader.loadTestsFromTestCase(ConnectTest), loader.loadTestsFromTestCase(TerminalSizeTest), ] From a52186e4f8079f4e3aa2c2cb5dd279029006e6d4 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 10 Nov 2026 16:29:51 +0100 Subject: [PATCH 080/250] Answer every keyboard-interactive prompt The handler answered only S/Key and password prompts, and OpenSSH closes the connection if a prompt, e.g. for a verification code, gets no answer. Answer other prompts with an empty string, and turn off paramiko's fallback that answers any such prompt with the password. --- CHANGES.md | 7 ++++++ src/Exscript/protocols/ssh2.py | 34 +++++++++++++--------------- tests/Exscript/protocols/SSH2Test.py | 34 ++++++++++++++++++++++++++++ 3 files changed, 57 insertions(+), 18 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 140b6b7f..248b3ea8 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -142,6 +142,13 @@ marked with **BREAKING**; a migration guide will accompany the release. - **SSH Key Fingerprints in Debug Logs:** The keys that are tried for a login are logged with their SHA256 fingerprints, as `ssh-add -l` and `ssh-keygen -l` show them. Before, the MD5 fingerprint was logged as a Python bytes literal. +- **Keyboard-Interactive Logins:** Every prompt gets an answer: password prompts the + password, S/Key challenges the one-time password, other prompts, e.g. for a + verification code, an empty string. Before, other prompts got no answer, and + OpenSSH servers closed the connection. All prompts are passed to the received data. + The password is also no longer sent to a keyboard-interactive prompt that does not + look like a password prompt when a server refuses the password method (paramiko's + fallback). - **BREAKING - Terminal Size:** SSH and Telnet sessions get a terminal of 511 columns and 1000 lines; the new options `term_width` and `term_height` (also `Host` options) choose another size. Before, Exscript used the size of the terminal that diff --git a/src/Exscript/protocols/ssh2.py b/src/Exscript/protocols/ssh2.py index 03a9ba74..30486dd6 100644 --- a/src/Exscript/protocols/ssh2.py +++ b/src/Exscript/protocols/ssh2.py @@ -238,9 +238,9 @@ def _paramiko_auth_interactive(self, username: str, password: str | None = None) """ Authenticates with the keyboard-interactive method: answers S/Key and OTP challenges with a one-time password computed from the password, - and password prompts with the password. Passes the answered prompts, - but not the answers, to the received data. Does nothing without a - password. + password prompts with the password, and other prompts, such as for a + verification code, with an empty string. Passes the prompts, but not + the answers, to the received data. Does nothing without a password. """ if password is None: return @@ -248,29 +248,24 @@ def _paramiko_auth_interactive(self, username: str, password: str | None = None) def handler( title: str, instructions: str, prompt_list: list[tuple[str, bool]] ) -> list[str]: - if not prompt_list: - return [] response = [] for prompt, visible in prompt_list: + self._receive_cb(prompt + '\n') # Expose the prompt (hide the answer). match = _skey_re.search(prompt) if match is not None: - self._receive_cb(prompt + '\n') # Expose the prompt (hide the PW). self._dbg(2, "S/Key prompt received.") seq = int(match.group(1)) seed = match.group(2) self.otp_requested_event(Account(username, password), seq, seed) self._dbg(3, "Seq: %s, Seed: %s" % (seq, seed)) - skey_password = otp(password, seed, seq) - response.append(skey_password) - continue - - for regex in self.get_password_prompt(): - match = regex.search(prompt) - if match is not None: - self._receive_cb(prompt + '\n') # Expose the prompt (hide the PW). - self._dbg(2, "Password prompt received.") - response.append(password) - break + response.append(otp(password, seed, seq)) + elif any(regex.search(prompt) for regex in self.get_password_prompt()): + self._dbg(2, "Password prompt received.") + response.append(password) + else: + # Servers such as OpenSSH close the connection unless every prompt is answered. + self._dbg(2, "Unknown prompt %r, answered with an empty string." % prompt) + response.append('') return response cast(paramiko.Transport, self._transport).auth_interactive(username, handler) @@ -281,7 +276,10 @@ def _paramiko_auth_password(self, username: str, password: str | None) -> None: password. Passes a password prompt to the received data. """ self._receive_cb('Password: \n') # Expose a prompt (hide the PW). - cast(paramiko.Transport, self._transport).auth_password(username, password or '') + # Without fallback, paramiko would answer any single keyboard-interactive + # prompt with the password if the server refused the password method. + transport = cast(paramiko.Transport, self._transport) + transport.auth_password(username, password or '', fallback=False) def _paramiko_auth_agent(self, username: str, password: str | None = None) -> None: """ diff --git a/tests/Exscript/protocols/SSH2Test.py b/tests/Exscript/protocols/SSH2Test.py index 073bd89f..67a883cd 100644 --- a/tests/Exscript/protocols/SSH2Test.py +++ b/tests/Exscript/protocols/SSH2Test.py @@ -1,3 +1,4 @@ +import io import os import socket import tempfile @@ -232,6 +233,38 @@ def testAgentKey(self): self.assertIn(agent_key.fingerprint, output[0]) +class _TwoPrompts(paramiko.ServerInterface): + """Asks for a verification code and the password, and records the answers.""" + + answers: list[list[str]] = [] + + def get_allowed_auths(self, username): + return 'keyboard-interactive' + + def check_auth_interactive(self, username, submethods): + prompts = ('Verification code: ', False), ('Password: ', False) + return paramiko.InteractiveQuery('', '', *prompts) + + def check_auth_interactive_response(self, responses): + self.answers.append(list(responses)) + return paramiko.AUTH_SUCCESSFUL if len(responses) == 2 else paramiko.AUTH_FAILED + + +class KeyboardInteractiveTest(unittest.TestCase): + """Every prompt gets an answer: password prompts the password, others an empty string.""" + + def testUnknownPrompt(self): + interface = type('TwoPrompts', (_TwoPrompts,), {'answers': []}) + server = KeyServer(self, paramiko.RSAKey(filename=keyfile), interface=interface) + stdout = io.StringIO() + conn = SSH2(timeout=1, connect_rate_limit=0, verify_fingerprint=False, stdout=stdout) + self.addCleanup(conn.close, force=True) + conn.connect(HOST, server.port) + conn._paramiko_auth('admin', 'S3cret') + self.assertEqual(interface.answers, [['', 'S3cret']]) + self.assertEqual(stdout.getvalue(), 'Verification code: \nPassword: \n') + + class ConnectTest(unittest.TestCase): def testTriesAllAddresses(self): server = KeyServer(self, paramiko.RSAKey(filename=keyfile)) @@ -291,6 +324,7 @@ def suite(): loader.loadTestsFromTestCase(HostKeyTest), loader.loadTestsFromTestCase(ReconnectTest), loader.loadTestsFromTestCase(FingerprintTest), + loader.loadTestsFromTestCase(KeyboardInteractiveTest), loader.loadTestsFromTestCase(ConnectTest), loader.loadTestsFromTestCase(TerminalSizeTest), ] From 07d614ca1b0295d56901c9facc34a6088f661cc4 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 10 Nov 2026 16:37:29 +0100 Subject: [PATCH 081/250] Continue SSH logins after a partial authentication Servers can require several methods, e.g. OpenSSH with AuthenticationMethods publickey,password, but SSH2 ignored a partial success. Continue with the methods that the server allows next. Based on the idea of upstream PR #203 by u1735067. --- CHANGES.md | 8 ++ src/Exscript/protocols/ssh2.py | 109 +++++++++++++++++---------- tests/Exscript/protocols/SSH2Test.py | 100 ++++++++++++++++++++++++ 3 files changed, 179 insertions(+), 38 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 248b3ea8..3749b620 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -149,6 +149,14 @@ marked with **BREAKING**; a migration guide will accompany the release. The password is also no longer sent to a keyboard-interactive prompt that does not look like a password prompt when a server refuses the password method (paramiko's fallback). +- **SSH Logins With Several Methods:** Servers that require more than one + authentication method, e.g. OpenSSH's `AuthenticationMethods publickey,password`, + are supported: after a partial success, the login continues with the methods that + the server allows next, without reconnecting. Before, it stopped after the first + method and then failed to open the shell. The key of an account + (`Account(..., key=...)`) is tried without the password of the account, so such a + login fails with a `LoginFailure` that names the required methods. Thanks to + u1735067 (pull request #203), whose rewrite of the authentication included this. - **BREAKING - Terminal Size:** SSH and Telnet sessions get a terminal of 511 columns and 1000 lines; the new options `term_width` and `term_height` (also `Host` options) choose another size. Before, Exscript used the size of the terminal that diff --git a/src/Exscript/protocols/ssh2.py b/src/Exscript/protocols/ssh2.py index 30486dd6..53d53d08 100644 --- a/src/Exscript/protocols/ssh2.py +++ b/src/Exscript/protocols/ssh2.py @@ -228,22 +228,20 @@ def _paramiko_connect(self) -> paramiko.Transport: t.set_keepalive(self.KEEPALIVE_INTERVAL) return t - def _paramiko_auth_none(self, username: str, password: str | None = None) -> None: - """ - Authenticates with the "none" method; the password is not used. - """ - cast(paramiko.Transport, self._transport).auth_none(username) - - def _paramiko_auth_interactive(self, username: str, password: str | None = None) -> None: + def _paramiko_auth_interactive(self, username: str, password: str | None = None) -> list[str]: """ Authenticates with the keyboard-interactive method: answers S/Key and OTP challenges with a one-time password computed from the password, password prompts with the password, and other prompts, such as for a verification code, with an empty string. Passes the prompts, but not - the answers, to the received data. Does nothing without a password. + the answers, to the received data. + + :raises AuthenticationException: Without a password. + :return: After a partial success, the authentication types that the + server allows next; else an empty list. """ if password is None: - return + raise AuthenticationException('keyboard-interactive requires a password') def handler( title: str, instructions: str, prompt_list: list[tuple[str, bool]] @@ -268,25 +266,31 @@ def handler( response.append('') return response - cast(paramiko.Transport, self._transport).auth_interactive(username, handler) + return cast(paramiko.Transport, self._transport).auth_interactive(username, handler) - def _paramiko_auth_password(self, username: str, password: str | None) -> None: + def _paramiko_auth_password(self, username: str, password: str | None) -> list[str]: """ Authenticates with the password method; None is sent as an empty password. Passes a password prompt to the received data. + + :return: After a partial success, the authentication types that the + server allows next; else an empty list. """ self._receive_cb('Password: \n') # Expose a prompt (hide the PW). # Without fallback, paramiko would answer any single keyboard-interactive # prompt with the password if the server refused the password method. transport = cast(paramiko.Transport, self._transport) - transport.auth_password(username, password or '', fallback=False) + return transport.auth_password(username, password or '', fallback=False) - def _paramiko_auth_agent(self, username: str, password: str | None = None) -> None: + def _paramiko_auth_agent(self, username: str, password: str | None = None) -> list[str]: """ - Authenticates with the keys of the SSH agent, trying one after the - other until the server accepts one. Raises the error of the last key, - or AuthenticationException if the agent has no keys. The password is - not used. + Authenticates with the keys of the SSH agent, one after the other + until the server accepts one; the password is not used. Raises the + error of the last key, or AuthenticationException if the agent has no + keys. + + :return: After a partial success, the authentication types that the + server allows next; else an empty list. """ keys = paramiko.Agent().get_keys() if not keys: @@ -299,15 +303,14 @@ def _paramiko_auth_agent(self, username: str, password: str | None = None) -> No for key in keys: try: self._dbg(2, 'Trying SSH agent key %s' % key.fingerprint) - cast(paramiko.Transport, self._transport).auth_publickey(username, key) - return + return cast(paramiko.Transport, self._transport).auth_publickey(username, key) except SSHException as e: saved_exception = e raise saved_exception def _paramiko_auth_key( self, username: str, keys: Iterable[tuple[type[paramiko.PKey], str]], password: str | None - ) -> None: + ) -> list[str]: """ Authenticates with the given private key files, trying one after the other until the server accepts one. Raises the error of the last key @@ -316,6 +319,8 @@ def _paramiko_auth_key( :param keys: The key class and the file name of each key. :param password: The password of the key files; None is used as an empty password. + :return: After a partial success, the authentication types that the + server allows next; else an empty list. """ if password is None: password = '' @@ -328,15 +333,14 @@ def _paramiko_auth_key( try: key = pkey_class.from_private_key_file(filename, password) self._dbg(2, 'Trying key %s in %s' % (key.fingerprint, filename)) - cast(paramiko.Transport, self._transport).auth_publickey(username, key) - return + return cast(paramiko.Transport, self._transport).auth_publickey(username, key) except SSHException as e: saved_exception = e except OSError as e: saved_exception = e raise saved_exception - def _paramiko_auth_autokey(self, username: str, password: str | None) -> None: + def _paramiko_auth_autokey(self, username: str, password: str | None) -> list[str]: """ Like _paramiko_auth_key(), with the RSA, ECDSA and Ed25519 key files in ~/.ssh (Unix) and ~/ssh (Windows) that exist. @@ -353,11 +357,11 @@ def _paramiko_auth_autokey(self, username: str, password: str | None) -> None: file = os.path.expanduser(file) if os.path.isfile(file): keyfiles.append((cls, file)) - self._paramiko_auth_key(username, keyfiles, password) + return self._paramiko_auth_key(username, keyfiles, password) def _get_auth_methods( self, allowed_types: list[str] - ) -> list[Callable[[str, str | None], None]]: + ) -> list[Callable[[str, str | None], list[str]]]: """ Returns the methods for the authentication types to try, in order: the preferred_authentications that the server allows, or all allowed @@ -395,13 +399,12 @@ def _get_auth_methods( def _paramiko_auth(self, username: str, password: str | None) -> None: """ - Authenticates the user. First tries the "none" method, and returns if - the server accepts it; else the server tells the authentication types - that it allows. The authentication banner of the server is passed to - the received data. Then tries the methods of _get_auth_methods() until - one succeeds. Before a method, reconnects if the transport is no - longer active, or if the driver sets reconnect_between_auth_methods - and a request reached the server since the last connect. + Authenticates the user with the "none" method, or else with the + methods of _get_auth_methods() until one succeeds, and after a partial + success with the other methods that the server allows next. Before a + method, reconnects if the transport is closed, or if the driver sets + reconnect_between_auth_methods and a request reached the server since + the last connect, but never after a partial success. :raises LoginFailure: If no method succeeds; lists the error of each. """ @@ -427,13 +430,19 @@ def _paramiko_auth(self, username: str, password: str | None) -> None: # Finally try all supported login methods. errors = [] attempted = True # auth_none() above was an attempt - for method in auth_methods: + partial: list[str] = [] # The names of the methods that succeeded partially. + while auth_methods: + method = auth_methods.pop(0) + # The stubs lack Transport.active. + active = cast(Any, self._transport).active + if partial and not active: + errors.append('Connection closed after a partial authentication') + break # Some devices (e.g. Cisco WS-C3750 with IOS) do not accept further # login attempts after failing one. So in this hack, we re-connect # after each attempt that reached the server, auth_none() included. reconnect = self.get_driver().reconnect_between_auth_methods and attempted - # The stubs lack Transport.active. - if reconnect or not cast(Any, self._transport).active: + if not partial and (reconnect or not active): if reconnect: self._dbg(1, 'Forced reconnect by driver.') else: @@ -449,8 +458,7 @@ def _paramiko_auth(self, username: str, password: str | None) -> None: # without keys, the public key methods send none. handler = cast(paramiko.Transport, self._transport).auth_handler try: - method(username, password) - return + allowed = method(username, password) except BadHostKeyException as e: msg = 'Bad host key with %s(): %s' % (method.__name__, str(e)) self._dbg(1, msg) @@ -463,6 +471,25 @@ def _paramiko_auth(self, username: str, password: str | None) -> None: msg = 'SSHException with %s(): %s' % (method.__name__, str(e)) self._dbg(1, msg) errors.append(msg) + else: + if cast(paramiko.Transport, self._transport).is_authenticated(): + return + if not allowed: + errors.append('%s() returned without authenticating' % method.__name__) + else: + # The server accepted the method, but requires another one. + msg = 'Partial authentication with %s(), allowed next: %s' + self._dbg(1, msg % (method.__name__, allowed)) + partial.append(method.__name__) + try: + next_methods = self._get_auth_methods(allowed) + except BadAuthenticationType as e: + errors.append('After a partial authentication: %s' % e) + break + auth_methods = [m for m in next_methods if m.__name__ not in partial] + if not auth_methods: + msg = 'No method to try after a partial authentication; allowed: %s' + errors.append(msg % allowed) attempted = ( attempted or cast(paramiko.Transport, self._transport).auth_handler is not handler ) @@ -508,6 +535,9 @@ def _protocol_authenticate_by_key(self, user: str, key: PrivateKey) -> None: """ Authenticates with the user name and the key file of the given key, decrypted with the password of the key, and opens the shell channel. + + :raises LoginFailure: If the server requires another authentication + method after the key, such as a password. """ # Allow multiple key files. key_file: str | list[str] | None = key.get_filename() @@ -520,7 +550,10 @@ def _protocol_authenticate_by_key(self, user: str, key: PrivateKey) -> None: keys = [] for file in key_file: keys.append((keymap[key.get_type()], file)) - self._paramiko_auth_key(user, keys, key.get_password()) + allowed = self._paramiko_auth_key(user, keys, key.get_password()) + if allowed: + msg = 'Login failed: after the key, the server requires one of: %s' + raise LoginFailure(msg % ', '.join(allowed)) # Open the channel. self._paramiko_channel() diff --git a/tests/Exscript/protocols/SSH2Test.py b/tests/Exscript/protocols/SSH2Test.py index 67a883cd..c383cc0d 100644 --- a/tests/Exscript/protocols/SSH2Test.py +++ b/tests/Exscript/protocols/SSH2Test.py @@ -10,6 +10,7 @@ from Exscript import Account, PrivateKey from Exscript.protocols import SSH2, Netconf +from Exscript.protocols.exception import LoginFailure from Exscript.protocols.ssh2 import _prefer_key_type from Exscript.servers import SSHd, sshd @@ -264,6 +265,104 @@ def testUnknownPrompt(self): self.assertEqual(interface.answers, [['', 'S3cret']]) self.assertEqual(stdout.getvalue(), 'Verification code: \nPassword: \n') + def testWithoutPassword(self): + conn = SSH2() + with self.assertRaises(paramiko.AuthenticationException): + conn._paramiko_auth_interactive('admin', None) + + +class _KeyThenPassword(paramiko.ServerInterface): + """Requires the client key and then the password, like OpenSSH's AuthenticationMethods publickey,password.""" + + client_key: paramiko.PKey + next_types = 'password' # the types allowed after the key + + def __init__(self): + self.key_accepted = False + + def get_allowed_auths(self, username): + return self.next_types if self.key_accepted else 'publickey' + + def check_auth_publickey(self, username, key): + if key != self.client_key: + return paramiko.AUTH_FAILED + self.key_accepted = True + return paramiko.AUTH_PARTIALLY_SUCCESSFUL + + def check_auth_password(self, username, password): + if self.key_accepted and password == 'S3cret': + return paramiko.AUTH_SUCCESSFUL + return paramiko.AUTH_FAILED + + +class PartialAuthenticationTest(unittest.TestCase): + """A server may require several authentication methods, e.g. a key and a password.""" + + client_key = paramiko.ECDSAKey.generate() + + def connect(self, next_types='password', **kwargs): + attributes = {'client_key': self.client_key, 'next_types': next_types} + interface = type('KeyThenPassword', (_KeyThenPassword,), attributes) + self.server = KeyServer(self, paramiko.RSAKey(filename=keyfile), interface=interface) + conn = SSH2(timeout=1, connect_rate_limit=0, verify_fingerprint=False, **kwargs) + self.addCleanup(conn.close, force=True) + conn.connect(HOST, self.server.port) + return conn + + def login(self, password='S3cret', next_types='password', **kwargs): + conn = self.connect(next_types, **kwargs) + home = tempfile.TemporaryDirectory() # no key files + self.addCleanup(home.cleanup) + with ( + mock.patch.dict(os.environ, {'HOME': home.name, 'USERPROFILE': home.name}), + mock.patch('paramiko.Agent') as agent, + ): + agent.return_value.get_keys.return_value = [self.client_key] + conn._paramiko_auth('admin', password) + return conn + + def testAgentKeyThenPassword(self): + conn = self.login() + self.assertTrue(conn._transport.is_authenticated()) + self.assertEqual(len(self.server._transports), 1) + + def testNoReconnectAfterPartialSuccess(self): + # ios reconnects after failed requests, but a reconnect would lose the key. + conn = self.login(driver='ios') + self.assertTrue(conn._transport.is_authenticated()) + self.assertEqual(len(self.server._transports), 2) # one reconnect, after auth_none() + + def testWrongPassword(self): + with self.assertRaisesRegex(LoginFailure, r'_paramiko_auth_password\(\) failed'): + self.login(password='wrong') + + def testUnsupportedNextMethod(self): + with self.assertRaisesRegex(LoginFailure, 'After a partial authentication.*hostbased'): + self.login(next_types='hostbased') + + def testMethodThatDoesNotAuthenticate(self): + # E.g. an override in a subclass that returns without an exception. + def _no_auth(username, password): + return [] + + conn = self.connect() + with ( + mock.patch.object(conn, '_get_auth_methods', return_value=[_no_auth]), + self.assertRaisesRegex(LoginFailure, r'_no_auth\(\) returned without authenticating'), + ): + conn._paramiko_auth('admin', 'S3cret') + + def testKeyOfTheAccount(self): + # The key of an account is tried without the password of the account. + tmpdir = tempfile.TemporaryDirectory() + self.addCleanup(tmpdir.cleanup) + filename = os.path.join(tmpdir.name, 'id_ecdsa') + self.client_key.write_private_key_file(filename) + account = Account('admin', 'S3cret', key=PrivateKey.from_file(filename), needs_lock=False) + conn = self.connect() + with self.assertRaisesRegex(LoginFailure, 'requires one of: password'): + conn.protocol_authenticate(account) + class ConnectTest(unittest.TestCase): def testTriesAllAddresses(self): @@ -325,6 +424,7 @@ def suite(): loader.loadTestsFromTestCase(ReconnectTest), loader.loadTestsFromTestCase(FingerprintTest), loader.loadTestsFromTestCase(KeyboardInteractiveTest), + loader.loadTestsFromTestCase(PartialAuthenticationTest), loader.loadTestsFromTestCase(ConnectTest), loader.loadTestsFromTestCase(TerminalSizeTest), ] From b2d33ee235cd7c373cc898bc77e4955a9ccd459d Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 10 Nov 2026 16:40:15 +0100 Subject: [PATCH 082/250] Test the lexer and the tokens of parselib Also add the example database of hypothesis to .gitignore. --- .gitignore | 1 + tests/Exscript/LexerTest.py | 298 +++++++++++++++++++++++++++++++++++- tests/Exscript/TokenTest.py | 125 +++++++++++++++ 3 files changed, 422 insertions(+), 2 deletions(-) create mode 100644 tests/Exscript/TokenTest.py diff --git a/.gitignore b/.gitignore index 1f536cbb..1a48fb71 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,4 @@ coverage.xml docs/_build docs/_autosummary docs/_generated +.hypothesis diff --git a/tests/Exscript/LexerTest.py b/tests/Exscript/LexerTest.py index 4fbadd7b..985e05c7 100644 --- a/tests/Exscript/LexerTest.py +++ b/tests/Exscript/LexerTest.py @@ -1,9 +1,205 @@ +import contextlib +import io +import os +import re +import tempfile import unittest +from types import SimpleNamespace -from Exscript.parselib import Lexer +from hypothesis import given, settings +from hypothesis import strategies as st + +from Exscript.parselib import Lexer, Token +from Exscript.parselib.exception import ( + CompileError, + CompileWarning, + ExecuteError, + LexerException, + LexerWarning, +) + +GRAMMAR = [ + ('number', re.compile(r'\d+')), + ('word', re.compile(r'[a-z]+')), + ('whitespace', re.compile(r'[ \t]+')), + ('newline', re.compile(r'\n')), +] + + +class Tokens: + """ + A parser built on the lexer: reads the tokens of the input with the + given grammar. + """ + + def __init__(self, lexer, grammar=GRAMMAR): + lexer.set_grammar(grammar) + self.tokens = [] + while not lexer.current_is('EOF'): + self.tokens.append(lexer.token()) + next(lexer) + lexer.restore_grammar() + + def dump(self): + print(self.tokens) + + +def lexer_for(text, filename=None): + """ + Returns a lexer that is positioned at the start of the given text and + uses GRAMMAR. + """ + lexer = Lexer(lambda lexer: lexer).parse(text, filename) + lexer.set_grammar(GRAMMAR) + return lexer class LexerTest(unittest.TestCase): + CORRELATE = Lexer + + def testConstructor(self): + lexer = Lexer(Tokens, [('any', re.compile('.'))]) + self.assertEqual(lexer.parse('ab').tokens, [('any', 'a'), ('any', 'b')]) + + def testParse(self): + lexer = Lexer(Tokens) + expected = [ + ('word', 'ab'), + ('whitespace', ' '), + ('number', '12'), + ('newline', '\n'), + ('word', 'cd'), + ] + self.assertEqual(lexer.parse('ab 12\ncd').tokens, expected) + # The lexer starts again for each input. + self.assertEqual(lexer.parse('x').tokens, [('word', 'x')]) + self.assertEqual(lexer.parse('').tokens, []) + + def testParseFile(self): + tmpdir = tempfile.TemporaryDirectory() + self.addCleanup(tmpdir.cleanup) + filename = os.path.join(tmpdir.name, 'words.txt') + with open(filename, 'w', encoding='utf-8') as fp: + fp.write('gr\u00fc\u00dfe\n') + any_char = [('any', re.compile('.', re.S))] + self.assertEqual(Lexer(Tokens, any_char).parse_file(filename).tokens[2], ('any', '\u00fc')) + with self.assertRaises(CompileError) as cm: + Lexer(Tokens).parse_file(filename) + self.assertTrue(str(cm.exception).endswith(' in %s:1' % filename)) + + def testDebug(self): + out = io.StringIO() + with contextlib.redirect_stdout(out): + Lexer(Tokens, debug=4).parse('ab') + Lexer(Tokens, debug=5).parse('cd') + self.assertEqual(out.getvalue(), "[('word', 'cd')]\n") + + def testSetGrammar(self): + lexer = lexer_for('ab12') + self.assertEqual(lexer.token(), ('word', 'ab')) + lexer.set_grammar([('char', re.compile('.'))]) + self.assertEqual(lexer.token(), ('char', 'a')) + lexer.restore_grammar() + self.assertEqual(lexer.token(), ('word', 'ab')) + + def testRestoreGrammar(self): + self.testSetGrammar() + + def testMatch(self): + lexer = lexer_for('ab #! cd\nef', 'words.txt') + lexer.match() + self.assertEqual(lexer.token_buffer, ('word', 'ab')) + lexer.forward(3) + with self.assertRaises(CompileError) as cm: + lexer.match() + message = "\nab #! cd\n ^\nInvalid syntax: '#! cd\\nef' in words.txt:1" + self.assertEqual(str(cm.exception), message) + lexer.forward(8) + lexer.match() + self.assertEqual(lexer.token_buffer, ('EOF', '')) + + def testMatchShowsWholeWords(self): + words = ' '.join(['word'] * 20) + with self.assertRaises(CompileError) as cm: + Lexer(Tokens).parse('#' + words) + context = re.search(r"Invalid syntax: '(.*)'", str(cm.exception)).group(1) + self.assertEqual(context, '#' + ' '.join(['word'] * 10)) + + def testGetCurrentLineNumber(self): + lexer = lexer_for('ab\ncd ef\n') + self.assertEqual(lexer.get_current_line_number(), 1) + lexer.forward(5) + self.assertEqual(lexer.get_current_line_number(), 2) + lexer.forward(4) + self.assertEqual(lexer.get_current_line_number(), 3) + + def testGetCurrentLine(self): + lexer = lexer_for('ab\ncd ef\n') + self.assertEqual(lexer.get_current_line(), 'ab') + lexer.forward(5) + self.assertEqual(lexer.get_current_line(), 'cd ef') + lexer.forward(4) + self.assertEqual(lexer.get_current_line(), '') + + def testError(self): + lexer = lexer_for('ab\ncd ef gh\n', 'f.txt') + with self.assertRaises(LexerException) as cm: + lexer.error('Boom') + self.assertEqual(str(cm.exception), '\nBoom') + self.assertRaises(ValueError, lexer.error, 'Boom', exc_cls=ValueError) + with self.assertRaises(ValueError) as cm: + lexer.error('Boom', SimpleNamespace(start=6, end=8), ValueError) + self.assertEqual(str(cm.exception), '\ncd ef gh\n ^^\nBoom in f.txt:2') + with self.assertWarns(CompileWarning) as warning: + lexer.error('Careful', exc_cls=CompileWarning) + self.assertEqual(str(warning.warning), '\nCareful') + + def testErrorOnSeveralLines(self): + lexer = lexer_for('ab\n\n cd ef\n gh\n', 'f.txt') + with self.assertRaises(LexerException) as cm: + lexer.error('Boom', SimpleNamespace(start=0, end=17)) + self.assertEqual(str(cm.exception), '\nab\n\n cd ef\n gh\n^^^^^^^\nBoom in f.txt:1-4') + with self.assertRaises(LexerException) as cm: + lexer.error('Boom', SimpleNamespace(start=6, end=16)) + self.assertEqual(str(cm.exception), '\n cd ef\n gh\n ^^^^^\nBoom in f.txt:3-4') + + def testErrorAtTheEnd(self): + lexer = lexer_for('ab\ncd\n') + with self.assertRaises(LexerException) as cm: + lexer.error('Boom', SimpleNamespace(start=3, end=6)) + self.assertEqual(str(cm.exception), '\ncd\n^^^\nBoom in None:2') + with self.assertRaises(LexerException) as cm: + lexer.error('Boom', SimpleNamespace(start=6, end=7)) + self.assertEqual(str(cm.exception), '\n\n^\nBoom in None:3') + + def testSyntaxError(self): + lexer = lexer_for('ab cd') + self.assertRaises(CompileError, lexer.syntax_error, 'Boom') + with self.assertRaises(CompileError) as cm: + lexer.syntax_error('Boom', SimpleNamespace(start=3, end=5)) + self.assertEqual(str(cm.exception), '\nab cd\n ^^\nBoom in None:1') + + def testSyntaxWarning(self): + lexer = lexer_for('ab cd') + with self.assertWarns(CompileWarning) as warning: + lexer.syntax_warning('Careful', SimpleNamespace(start=0, end=2)) + self.assertEqual(str(warning.warning), '\nab cd\n^^\nCareful in None:1') + with self.assertWarns(CompileWarning) as warning: + lexer.syntax_warning('Careful') + self.assertEqual(str(warning.warning), '\nCareful') + + def testRuntimeError(self): + lexer = lexer_for('ab cd') + with self.assertRaises(ExecuteError) as cm: + lexer.runtime_error('Boom', SimpleNamespace(start=3, end=4)) + self.assertEqual(str(cm.exception), '\nab cd\n ^\nBoom in None:1') + + def testExceptions(self): + for cls in CompileError, ExecuteError: + self.assertTrue(issubclass(cls, LexerException)) + self.assertTrue(issubclass(CompileWarning, LexerWarning)) + self.assertTrue(issubclass(LexerWarning, UserWarning)) + def testForward(self): # Public API for parsers built on parselib. lexer = Lexer(object) @@ -11,9 +207,107 @@ def testForward(self): lexer.forward() self.assertEqual((lexer.last_char, lexer.current_char), (3, 4)) + def testNext(self): + lexer = lexer_for('ab cd') + next(lexer) # no token was read yet + self.assertEqual(lexer.current_char, 0) + self.assertEqual(lexer.token(), ('word', 'ab')) + next(lexer) + self.assertEqual((lexer.last_char, lexer.current_char), (0, 2)) + self.assertEqual(lexer.token(), ('whitespace', ' ')) + lexer.forward(3) + self.assertEqual(lexer.token(), ('EOF', '')) + with self.assertRaises(StopIteration) as cm: + next(lexer) + self.assertEqual(str(cm.exception), 'Unexpected EOF while iterating tokens.') + + def testNextIf(self): + lexer = lexer_for('ab 12') + self.assertFalse(lexer.next_if('number')) + self.assertFalse(lexer.next_if('word', 'cd')) + self.assertEqual(lexer.current_char, 0) + self.assertTrue(lexer.next_if('word', 'ab')) + self.assertTrue(lexer.next_if(['number', 'whitespace'])) + self.assertTrue(lexer.next_if('number')) + self.assertEqual(lexer.current_char, 5) + + def testSkip(self): + lexer = lexer_for('12 34 ab 56') + lexer.skip(['number', 'whitespace']) + self.assertEqual(lexer.token(), ('word', 'ab')) + lexer.skip('number') + self.assertEqual(lexer.token(), ('word', 'ab')) + lexer.skip('word', 'ab') + lexer.skip('whitespace') + self.assertEqual(lexer.token(), ('number', '56')) + + def testExpect(self): + lexer = lexer_for('ab 12') + sender = Token('Sender', lexer, None) + lexer.expect(sender, 'word') + lexer.expect(sender, ['number', 'whitespace']) + for types, token, message in ( + ('word', None, "Expected word but got number '12'"), + (['word', 'whitespace'], None, "Expected one of word, whitespace but got number '12'"), + ('number', '34', "Expected '34' but got number '12'"), + ): + with self.assertRaises(CompileError) as cm: + lexer.expect(sender, types, token) + self.assertEqual(str(cm.exception), '\nab 12\n ^^\n%s in None:1' % message) + self.assertEqual((sender.start, sender.end), (3, 5)) + lexer.expect(sender, 'number', '12') + self.assertEqual(lexer.token(), ('EOF', '')) + + def testCurrentIs(self): + lexer = lexer_for('ab 12') + self.assertTrue(lexer.current_is('word')) + self.assertTrue(lexer.current_is(['number', 'word'])) + self.assertTrue(lexer.current_is('word', 'ab')) + self.assertFalse(lexer.current_is('word', 'a')) + self.assertFalse(lexer.current_is('number')) + self.assertFalse(lexer.current_is(['number', 'whitespace'], 'ab')) + + def testToken(self): + lexer = lexer_for('ab 12') + self.assertEqual(lexer.token(), ('word', 'ab')) + lexer.forward(3) + self.assertEqual(lexer.token(), ('number', '12')) + + +class LexerPropertyTest(unittest.TestCase): + @settings(max_examples=200, deadline=None) + @given(st.text(alphabet='ab12 \t\n', max_size=60)) + def testTokensCoverTheInput(self, text): + tokens = Lexer(Tokens).parse(text).tokens + self.assertEqual(''.join(token for _, token in tokens), text) + # The patterns are greedy, so only a newline may follow a token of its type. + for (first, _), (second, _) in zip(tokens, tokens[1:]): + self.assertTrue(first != second or first == 'newline', tokens) + + # match() shows the input up to 50 characters behind an error, up to a + # blank or the end of the input; shorter inputs always have such an end. + @settings(max_examples=200, deadline=None) + @given(st.text(alphabet='ab1 \n#', max_size=40)) + def testInvalidSyntaxShowsThePosition(self, text): + if '#' not in text: + Lexer(Tokens).parse(text) + return + with self.assertRaises(CompileError) as cm: + Lexer(Tokens).parse(text) + position = text.index('#') + lineno = text.count('\n', 0, position) + 1 + column = position - (text.rfind('\n', 0, position) + 1) + line = text.split('\n')[lineno - 1] + message = str(cm.exception) + self.assertEqual(message.split('\n')[:3], ['', line, ' ' * column + '^']) + self.assertTrue(message.endswith(' in None:%d' % lineno), message) + def suite(): - return unittest.TestLoader().loadTestsFromTestCase(LexerTest) + loader = unittest.TestLoader() + return unittest.TestSuite( + [loader.loadTestsFromTestCase(LexerTest), loader.loadTestsFromTestCase(LexerPropertyTest)] + ) if __name__ == '__main__': diff --git a/tests/Exscript/TokenTest.py b/tests/Exscript/TokenTest.py new file mode 100644 index 00000000..081e1b82 --- /dev/null +++ b/tests/Exscript/TokenTest.py @@ -0,0 +1,125 @@ +import contextlib +import io +import unittest + +from Exscript.parselib import Lexer, Token + + +class Leaf(Token): + """ + A token that records the contexts in which it is evaluated. + """ + + def __init__(self, name, lexer, parent=None): + Token.__init__(self, name, lexer, None, parent) + self.contexts = [] + + def value(self, context): + self.contexts.append(context) + return self.name + + +class TokenTest(unittest.TestCase): + CORRELATE = Token + + def setUp(self): + # Eleven characters: two blanks before "cd", and a tab and a blank behind it. + self.lexer = Lexer(lambda lexer: lexer).parse('ab cd\t \nef') + + def tree(self): + """ + Returns the tree a(b(d), c). + """ + a, b, c, d = (Token(name, self.lexer, None) for name in 'abcd') + a.add(b) + a.add(c) + b.add(d) + return a + + def testConstructor(self): + self.lexer.forward(2) + token = Token('Name', self.lexer, 'parser', 'parent') + self.assertEqual((token.name, token.parser, token.parent), ('Name', 'parser', 'parent')) + self.assertIs(token.lexer, self.lexer) + self.assertEqual((token.start, token.end), (2, 3)) + self.assertEqual(token.get_children(), []) + + def testValue(self): + parent = Token('parent', self.lexer, None) + children = [Leaf(name, self.lexer, parent) for name in ('x', 'y')] + for child in children: + parent.add(child) + self.assertIsNone(parent.value('context')) + self.assertEqual([child.contexts for child in children], [['context'], ['context']]) + + def testMarkStart(self): + token = Token('T', self.lexer, None) + self.lexer.forward(4) + token.mark_start() + self.assertEqual((token.start, token.end), (4, 5)) # the end moves behind the start + token.mark_start(1) + self.assertEqual((token.start, token.end), (1, 5)) + token.mark_start(-3) + self.assertEqual((token.start, token.end), (0, 5)) + token.mark_start(100) + self.assertEqual((token.start, token.end), (11, 11)) # the end of the input + + def testMarkEnd(self): + token = Token('T', self.lexer, None) + token.mark_end(4) + self.assertEqual(token.end, 4) + token.mark_end(distance=2) + self.assertEqual(token.end, 2) + self.lexer.forward(6) + token.mark_end() + self.assertEqual(token.end, 6) + token.mark_end(100) + self.assertEqual(token.end, 11) + token.mark_start(3) + token.mark_end(1) + self.assertEqual(token.end, 3) # never before the start + with self.assertRaises(ValueError) as cm: + token.mark_end(1, distance=1) + self.assertEqual(str(cm.exception), 'must not specify both arguments "char" and "distance"') + + def testMarkEndRstrip(self): + token = Token('T', self.lexer, None) + token.mark_end(9, rstrip=True) + self.assertEqual(token.end, 6) + token.mark_end(4, rstrip=True) + self.assertEqual(token.end, 2) + token.mark_end(6, rstrip='dc') + self.assertEqual(token.end, 4) + token.mark_end(9, rstrip=' \n') + self.assertEqual(token.end, 7) + + def testAdd(self): + parent = Token('parent', self.lexer, None) + first, second = Token('first', self.lexer, parent), Token('second', self.lexer, parent) + parent.add(first) + parent.add(second) + self.assertEqual(parent.get_children(), [first, second]) + + def testGetChildren(self): + self.testAdd() + + def testIter(self): + self.assertEqual([token.name for token in self.tree()], ['a', 'b', 'd', 'c']) + self.assertEqual([token.name for token in Token('single', self.lexer, None)], ['single']) + iterator = iter(self.tree()) + self.assertIs(iter(iterator), iterator) + self.assertEqual(next(iterator).name, 'a') + + def testDump(self): + out = io.StringIO() + with contextlib.redirect_stdout(out): + self.tree().dump() + self.assertEqual(out.getvalue(), 'a\n b\n d\n c\n') + + +def suite(): + return unittest.TestLoader().loadTestsFromTestCase(TokenTest) + + +if __name__ == '__main__': + unittest.TextTestRunner(verbosity=2).run(suite()) From fc13a1e12fb803c83d148d005da6ace27ae8dc00 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 10 Nov 2026 16:40:15 +0100 Subject: [PATCH 083/250] Test the interpreter of the template language --- tests/Exscript/interpreter/AssignTest.py | 103 ++++++ tests/Exscript/interpreter/CommandTest.py | 163 +++++++++ tests/Exscript/interpreter/ExpressionTest.py | 206 +++++++++++ tests/Exscript/interpreter/ExtractTest.py | 133 +++++++ tests/Exscript/interpreter/FailTest.py | 111 ++++++ tests/Exscript/interpreter/IfConditionTest.py | 103 ++++++ tests/Exscript/interpreter/LoopTest.py | 127 +++++++ .../interpreter/ParserPropertyTest.py | 178 +++++++++ tests/Exscript/interpreter/ParserTest.py | 343 ++++++++++++++++++ tests/Exscript/interpreter/RpcTest.py | 106 ++++++ tests/Exscript/interpreter/StringTest.py | 128 +++++++ tests/Exscript/interpreter/__init__.py | 0 tests/Exscript/interpreter/helpers.py | 85 +++++ tests/Exscript/util/templateTest.py | 177 ++++++++- 14 files changed, 1959 insertions(+), 4 deletions(-) create mode 100644 tests/Exscript/interpreter/AssignTest.py create mode 100644 tests/Exscript/interpreter/CommandTest.py create mode 100644 tests/Exscript/interpreter/ExpressionTest.py create mode 100644 tests/Exscript/interpreter/ExtractTest.py create mode 100644 tests/Exscript/interpreter/FailTest.py create mode 100644 tests/Exscript/interpreter/IfConditionTest.py create mode 100644 tests/Exscript/interpreter/LoopTest.py create mode 100644 tests/Exscript/interpreter/ParserPropertyTest.py create mode 100644 tests/Exscript/interpreter/ParserTest.py create mode 100644 tests/Exscript/interpreter/RpcTest.py create mode 100644 tests/Exscript/interpreter/StringTest.py create mode 100644 tests/Exscript/interpreter/__init__.py create mode 100644 tests/Exscript/interpreter/helpers.py diff --git a/tests/Exscript/interpreter/AssignTest.py b/tests/Exscript/interpreter/AssignTest.py new file mode 100644 index 00000000..4b94e0e9 --- /dev/null +++ b/tests/Exscript/interpreter/AssignTest.py @@ -0,0 +1,103 @@ +import unittest + +from Exscript.parselib.exception import CompileError + +from .helpers import TemplateTestCase + + +class AssignTest(TemplateTestCase): + """ + The assignment of variables. + """ + + def testAssign(self): + variables = self.evaluate('{x = 1}{x = x + 1}{y = "a"}{z = list.new("b", "c")}') + self.assertEqual(variables['x'], [2]) + self.assertEqual(variables['y'], ['a']) + self.assertEqual(variables['z'], ['b', 'c']) + + def testReassign(self): + variables = self.evaluate('{x = "hi"}{x = x . "!"}') + self.assertEqual(variables['x'], ['hi!']) + + def testAssignmentsInBlocksAreVisibleOutside(self): + self.evaluate('{if 1}{x = "eth0"}{end}\nshow interface $x\n') + self.assertSent('show interface eth0') + + def testInternalVariables(self): + with self.assertRaises(CompileError) as cm: + self.evaluate('{__hostname__ = "r2"}') + message = "Assignment to internal variable '__hostname__' in undefined:1" + self.assertError(cm.exception, ['{__hostname__ = "r2"}'], ' ^^^^^^^^^^^^', message) + + def testWhitespaceAroundTheEqualsSign(self): + for text, marker, token in ( + ('{x=1}', ' ^', "assign '='"), + ('{x =1}', ' ^', "number '1'"), + ): + with self.subTest(text=text): + with self.assertRaises(CompileError) as cm: + self.evaluate(text) + message = 'Expected whitespace but got %s in undefined:1' % token + self.assertError(cm.exception, [text], marker, message) + + def testExpressionOnTheNextLine(self): + with self.assertRaises(CompileError) as cm: + self.evaluate('{x = \n1}') + message = "Expected expression but got newline '\\n' in undefined:1" + self.assertError(cm.exception, ['{x = '], ' ^', message) + + +class AppendTest(TemplateTestCase): + """ + The append statement. + """ + + def testAppend(self): + variables = self.evaluate('{append "a" to x}{append 1 to x}') + self.assertEqual(variables['x'], ['a', 1]) + + def testAppendToAVariable(self): + variables = self.evaluate('{test1 = "one"}{test2 = "two"}{append test2 to test1}') + self.assertEqual(variables['test1'], ['one', 'two']) + self.assertEqual(variables['test2'], ['two']) + + def testAppendAList(self): + variables = self.evaluate('{x = "a"}{append list.new("b", "c") to x}') + self.assertEqual(variables['x'], ['a', 'b', 'c']) + + def testAppendInALoop(self): + variables = self.evaluate('{loop from 0 to 3 as i}{append i * 2 to doubles}{end}') + self.assertEqual(variables['doubles'], [0, 2, 4]) + + def testSyntaxErrors(self): + for text, marker, message in ( + ('{append 1 x}', ' ' * 10 + '^', "Expected 'to' but got varname 'x'"), + ( + '{append 1 to}', + ' ' * 12 + '^', + "Expected whitespace but got close_curly_bracket '}'", + ), + ('{append 1 to "x"}', ' ' * 13 + '^', "Expected varname but got string_delimiter '\"'"), + ): + with self.subTest(text=text): + with self.assertRaises(CompileError) as cm: + self.evaluate(text) + self.assertError(cm.exception, [text], marker, message + ' in undefined:1') + + def testExpressionOnTheNextLine(self): + with self.assertRaises(CompileError) as cm: + self.evaluate('{append \n1 to x}') + message = "Expected expression but got newline '\\n' in undefined:1" + self.assertError(cm.exception, ['{append '], ' ' * 8 + '^', message) + + +def suite(): + loader = unittest.TestLoader() + return unittest.TestSuite( + [loader.loadTestsFromTestCase(AssignTest), loader.loadTestsFromTestCase(AppendTest)] + ) + + +if __name__ == '__main__': + unittest.TextTestRunner(verbosity=2).run(suite()) diff --git a/tests/Exscript/interpreter/CommandTest.py b/tests/Exscript/interpreter/CommandTest.py new file mode 100644 index 00000000..8df429ac --- /dev/null +++ b/tests/Exscript/interpreter/CommandTest.py @@ -0,0 +1,163 @@ +import unittest + +from Exscript.interpreter import Parser +from Exscript.parselib.exception import CompileError, ExecuteError +from Exscript.protocols.exception import InvalidCommandException +from Exscript.util import template + +from .helpers import TemplateTestCase + + +class CommandTest(TemplateTestCase): + """ + The text of a template outside of code blocks: each line is a command. + """ + + def testCommandsAreSentLineByLine(self): + variables = self.evaluate('show version\nls\n') + self.assertSent('show version', 'ls') + self.assertEqual(variables['__response__'], ['file1', 'file2']) + + def testBlankLinesAndIndentationAreNotSent(self): + self.evaluate('\n show version\n\t\n ls\n\n') + self.assertSent('show version', 'ls') + + def testCommentsAreNotSent(self): + text = ( + '# a comment\n' + 'ls\n' + ' # an indented comment\n' + 'show version # not a comment\n' + '{x = 1}# a comment after a code block\n' + ) + self.evaluate(text) + self.assertSent('ls', 'show version # not a comment') + + def testCommandBeforeACodeBlock(self): + variables = self.evaluate('show version{extract /^Version (\\S+)/ as version}\n') + self.assertSent('show version') + self.assertEqual(variables['version'], ['1.0']) + + def testResponse(self): + variables = self.evaluate('show version\n') + self.assertEqual(variables['__response__'], ['Version 1.0', 'Uptime 5 days']) + + def testResponseWithCommand(self): + variables = self.evaluate('show version\n', strip_command=False) + expected = ['show version', 'Version 1.0', 'Uptime 5 days'] + self.assertEqual(variables['__response__'], expected) + + def testVariables(self): + self.evaluate('{name = "eth0"}{n = 1 + 1}\nshow interface $name $n $__hostname__\n') + self.assertSent('show interface eth0 2 r1') + + def testVariablesOfTheCaller(self): + self.evaluate('show interface $name\nshow vlan $vlan\n', name='eth0', vlan=10) + self.assertSent('show interface eth0', 'show vlan 10') + + def testDollarSign(self): + self.evaluate('{name = "eth0"}\nshow \\$name $ $\n') + self.assertSent('show $name $ $') + + def testUndefinedVariable(self): + with self.assertRaises(ExecuteError) as cm: + self.evaluate('ls\nshow $nothing\n') + message = "Undefined variable 'nothing' in undefined:2" + self.assertError(cm.exception, ['show $nothing'], ' ^^^^^^^^', message) + self.assertSent() # the template is checked before it runs + + def testUndefinedVariableAfterACodeBlock(self): + with self.assertRaises(ExecuteError) as cm: + self.evaluate('{x = "a"}\n show $x $nothing\n') + message = "Undefined variable 'nothing' in undefined:2" + self.assertError(cm.exception, [' show $x $nothing'], ' ^^^^^^^^', message) + + def testInvalidVariableName(self): + with self.assertRaises(ExecuteError) as cm: + self.evaluate('show $1x\n') + message = "'1x' is not a variable name in undefined:1" + self.assertError(cm.exception, ['show $1x'], ' ^^^', message) + + def testEscapeSequences(self): + self.evaluate('show \\x41\\101\\u00e9\\U0001F600 \\t \\e \\a \\0 \\v \\f \\b\n') + self.assertSent('show AA\u00e9\U0001f600 \t \x1b \x07 \x00 \x0b \x0c \x08') + + def testNoEscaping(self): + text = 'show \\x41 \\101 \\u00e9 \\U0001F600 \\t \\e\n' + self.evaluate(text, no_escaping=True) + self.assertSent('show x41 101 u00e9 U0001F600 t e') + + def testEscapedSpecialCharacters(self): + self.evaluate('show \\{a\\} \\\\ \\$x \\# \\"\n') + self.assertSent('show {a} \\ $x # "') + + def testBackslashAtTheEndOfALine(self): + with self.assertRaises(CompileError) as cm: + self.evaluate('ls \\\nshow version\n') + message = "Invalid syntax: '\\\\\\nshow version\\n' in undefined:1" + self.assertError(cm.exception, ['ls \\'], ' ^', message) + + def testClosingBracketWithoutCodeBlock(self): + with self.assertRaises(CompileError) as cm: + self.evaluate('ls\n}\n') + message = "Unexpected close_curly_bracket '}' in undefined:2" + self.assertError(cm.exception, ['}'], '^', message) + self.assertSent() + + def testErrorInTheResponseStopsTheTemplate(self): + self.device.add_command('bad', '% Invalid input detected') + with self.assertRaises(InvalidCommandException): + self.evaluate('show version\nbad\nls\n') + self.assertSent('show version', 'bad') + + def testPasteDoesNotWaitForPrompts(self): + self.device.add_command('reload', 'Proceed with reload? [confirm]', prompt=False) + variables = template.paste(self.conn, 'reload\ny\n') + self.assertSent('reload', 'y') + self.assertEqual(variables['__response__'], ['']) + + def testPasteRejectsExtract(self): + with self.assertRaises(CompileError) as cm: + template.paste(self.conn, 'show version{extract /(.*)/ as x}\n') + message = "'extract' keyword does not work with --no-prompt in undefined:1" + lines = ['show version{extract /(.*)/ as x}'] + self.assertError(cm.exception, lines, ' ^^^^^^^', message) + self.assertSent() + + def testConnectionIsRequired(self): + program = Parser().parse('ls\n') + with self.assertRaises(ExecuteError) as cm: + program.execute() + message = "Undefined variable '__connection__' in None:1" + self.assertError(cm.exception, ['ls'], '^^', message) + + +class EnterTest(TemplateTestCase): + """ + The enter statement. + """ + + def testEnterSendsAnEmptyLine(self): + self.device.add_command(r'\r?$', 'empty line') + variables = self.evaluate('show version\n{enter}') + self.assertSent('show version', '') + self.assertEqual(variables['__response__'], ['empty line']) + + def testEnterInPaste(self): + template.paste(self.conn, '{enter}ls\n{enter}') + self.assertSent('', 'ls', '') + + def testEnterInABlock(self): + self.evaluate('{loop from 0 to 2\n enter\nend}') + self.assertSent('', '') + + +def suite(): + loader = unittest.TestLoader() + return unittest.TestSuite( + [loader.loadTestsFromTestCase(CommandTest), loader.loadTestsFromTestCase(EnterTest)] + ) + + +if __name__ == '__main__': + unittest.TextTestRunner(verbosity=2).run(suite()) diff --git a/tests/Exscript/interpreter/ExpressionTest.py b/tests/Exscript/interpreter/ExpressionTest.py new file mode 100644 index 00000000..ebec68d1 --- /dev/null +++ b/tests/Exscript/interpreter/ExpressionTest.py @@ -0,0 +1,206 @@ +import unittest + +from Exscript.parselib.exception import CompileError, ExecuteError + +from .helpers import TemplateTestCase + + +class ExpressionTest(TemplateTestCase): + """ + Terms, operators and the priorities of operators in expressions. + """ + + def assertValues(self, text, **expected): + variables = self.evaluate(text) + actual = {name: variables[name] for name in expected} + self.assertEqual(actual, expected) + + def testNumbers(self): + text = '{a = 42}{b = 0x1f}{c = 017}{d = 0o17}{e = true}{f = false}' + self.assertValues(text, a=[42], b=[31], c=[15], d=[15], e=[1], f=[0]) + + def testStringsAndVariables(self): + self.assertValues('{a = "text"}{b = a}', a=['text'], b=['text']) + + def testListVariable(self): + self.assertValues('{a = list.new("x", "y")}{b = a}', b=['x', 'y']) + + def testUndefinedVariable(self): + with self.assertRaises(NameError) as cm: + self.evaluate('ls\n{x = nothing}') + message = "Undefined variable 'nothing' in undefined:2" + self.assertError(cm.exception, ['{x = nothing}'], ' ^^^^^^^', message) + self.assertSent() + + def testExpectedTerm(self): + with self.assertRaises(CompileError) as cm: + self.evaluate('{x = )}') + message = "Expected term but got close_bracket ')' in undefined:1" + self.assertError(cm.exception, ['{x = )}'], ' ^', message) + + def testArithmetic(self): + text = '{a = 7 + 3}{b = 7 - 3}{c = 7 * 3}{d = 7 / 3}{e = 7 % 3}' + self.assertValues(text, a=[10], b=[4], c=[21], d=[2], e=[1]) + + def testArithmeticWithDigitStrings(self): + self.assertValues('{a = "5" + "6"}{b = "5" * 2}', a=[11], b=[10]) + + def testConcatenation(self): + self.assertValues('{a = "a" . "b"}{b = "a" . 1}{c = 1 . 2}', a=['ab'], b=['a1'], c=['12']) + + def testOperandIsNotANumber(self): + for text, marker in ( + ('{x = "a" * 2}', ' ^^^'), + ('{x = 1 + "b"}', ' ^^^'), + ('{x = "a" - 1}', ' ^^^'), + ('{x = 5 % "b"}', ' ^^^'), + ): + with self.subTest(text=text): + with self.assertRaises(ExecuteError) as cm: + self.evaluate(text) + op = text.split()[3] + message = 'Operand for %s is not a number in undefined:1' % op + self.assertError(cm.exception, [text], marker, message) + + def testEquality(self): + text = '{a = 1 is 1}{b = "a" is "b"}{c = 1 is not 2}{d = "a" is not "a"}' + self.assertValues(text, a=[True], b=[False], c=[True], d=[False]) + + def testNumericComparison(self): + text = '{a = 3 ge 3}{b = 3 gt 3}{c = 2 le 3}{d = 4 lt 3}{e = "10" gt "9"}' + self.assertValues(text, a=[True], b=[False], c=[True], d=[False], e=[True]) + + def testIn(self): + text = ( + '{l = list.new("a", "b")}' + '{a = "a" in l}{b = "c" in l}{c = "c" not in l}{d = "a" not in l}' + ) + self.assertValues(text, a=[True], b=[False], c=[True], d=[False]) + + def testMatches(self): + text = ( + '{a = "abc" matches /b/}{b = "abc" matches /^b/}{c = "ABC" matches /b/i}' + '{l = list.new("ab", "cd")}{d = l matches /^c/}{e = l matches /^x/}' + ) + self.assertValues(text, a=[1], b=[0], c=[1], d=[1], e=[0]) + + def testMatchesRequiresARegularExpression(self): + with self.assertRaises(ExecuteError) as cm: + self.evaluate('{x = "abc" matches "b"}') + message = 'Right hand operator is not a regular expression in undefined:1' + self.assertError(cm.exception, ['{x = "abc" matches "b"}'], ' ' * 19 + '^^^', message) + + def testLogicalOperators(self): + text = '{a = 1 and 0}{b = 0 or 1}{c = not 0}{d = not 1}{e = 1 and 1 and 0}' + self.assertValues(text, a=[0], b=[1], c=[True], d=[False], e=[0]) + + def testMultiplicationBeforeAddition(self): + text = '{a = 2 + 3 * 4}{b = 2 * 3 + 4}{c = 20 - 12 / 4}{d = 20 / 4 - 1}{e = 1 + 7 % 3}' + self.assertValues(text, a=[14], b=[10], c=[17], d=[4], e=[2]) + + def testAdditionBeforeConcatenation(self): + self.assertValues('{a = 1 + 2 . 3}{b = "a" . 1 + 2}', a=['33'], b=['a3']) + + def testConcatenationBeforeComparison(self): + text = '{a = "a" . "b" is "ab"}{b = 2 * 3 is 6}{c = 1 + 1 is not 3}' + self.assertValues(text, a=[True], b=[True], c=[True]) + + def testComparisonBeforeNot(self): + self.assertValues('{a = not 1 is 2}{b = not "a" in "a"}', a=[True], b=[False]) + + def testNotBeforeAndOr(self): + self.assertValues('{a = not 0 and 0}{b = not 1 or 1}', a=[0], b=[1]) + + def testComparisonBeforeAndOr(self): + text = '{a = 1 is 1 and 2 is 3}{b = 1 is 2 or 2 is 2}' + self.assertValues(text, a=[False], b=[True]) + + def testExpressionOnSeveralLines(self): + text = '{a = 1 is 1\n and 2\n is 2\n}' + self.assertValues(text, a=[True]) + + def testListOperands(self): + text = ( + '{l = list.new("a", "b")}{e = list.new()}' + '{a = l . "c"}{b = e . "c"}{c = e is ""}{d = "c" . e}' + ) + self.assertValues(text, a=['ac'], b=['c'], c=[True], d=['c']) + + def testRegularExpressionTerm(self): + variables = self.evaluate('{r = /a.c/i}') + self.assertEqual([pattern.pattern for pattern in variables['r']], ['a.c']) + + +class FunctionCallTest(TemplateTestCase): + """ + Calls of functions in expressions and as statements. + """ + + def testCall(self): + variables = self.evaluate( + '{a = string.toupper("a" . "b")}{b = list.length(list.new("x", 1))}' + ) + self.assertEqual(variables['a'], ['AB']) + self.assertEqual(variables['b'], [2]) + + def testCallAsStatement(self): + self.evaluate('{connection.sendline("ls")}') + self.assertSent('ls') + + def testFunctionOfTheCaller(self): + calls = [] + + def record(scope, *args): + calls.append((scope.get('__hostname__'), args)) + return ['result'] + + variables = self.evaluate( + '{x = record("a", 1, list.new("b", "c"))}{record()}', record=record + ) + self.assertEqual(variables['x'], ['result']) + self.assertEqual(calls, [(['r1'], (['a'], [1], ['b', 'c'])), (['r1'], ())]) + + def testUndefinedFunction(self): + with self.assertRaises(CompileError) as cm: + self.evaluate('ls\n{x = unknown.func(1)}') + message = "Undefined function 'unknown.func' in undefined:2" + self.assertError(cm.exception, ['{x = unknown.func(1)}'], ' ^^^^^^^^^^^^', message) + self.assertSent() + + def testErrorInFunction(self): + with self.assertRaises(ExecuteError) as cm: + self.evaluate('{x = list.get(list.new("a"), 3)}') + message = ( + "ValueError while executing function 'list.get': no such item in the list" + ' in undefined:1' + ) + self.assertError( + cm.exception, ['{x = list.get(list.new("a"), 3)}'], ' ^^^^^^^^', message + ) + + def testMissingComma(self): + with self.assertRaises(CompileError) as cm: + self.evaluate('{x = string.replace("abc", "b" "c")}') + message = "Expected comma or closing bracket but got string_delimiter '\"' in undefined:1" + lines = ['{x = string.replace("abc", "b" "c")}'] + self.assertError(cm.exception, lines, ' ' * 31 + '^', message) + + def testMissingClosingBracket(self): + with self.assertRaises(CompileError) as cm: + self.evaluate('{x = list.new("a"}') + message = "Expected comma or closing bracket but got close_curly_bracket '}' in undefined:1" + self.assertError(cm.exception, ['{x = list.new("a"}'], ' ' * 17 + '^', message) + + +def suite(): + loader = unittest.TestLoader() + return unittest.TestSuite( + [ + loader.loadTestsFromTestCase(ExpressionTest), + loader.loadTestsFromTestCase(FunctionCallTest), + ] + ) + + +if __name__ == '__main__': + unittest.TextTestRunner(verbosity=2).run(suite()) diff --git a/tests/Exscript/interpreter/ExtractTest.py b/tests/Exscript/interpreter/ExtractTest.py new file mode 100644 index 00000000..b539b640 --- /dev/null +++ b/tests/Exscript/interpreter/ExtractTest.py @@ -0,0 +1,133 @@ +import unittest + +from Exscript.parselib.exception import CompileError, ExecuteError +from Exscript.util import template + +from .helpers import TemplateTestCase + + +class ExtractTest(TemplateTestCase): + """ + The extract statement. + """ + + def testExtractAs(self): + variables = self.evaluate('show version{extract /^(\\w+) (\\S+)/ as names, values}\n') + self.assertSent('show version') + self.assertEqual(variables['names'], ['Version', 'Uptime']) + self.assertEqual(variables['values'], ['1.0', '5']) + + def testAsReplacesTheValues(self): + variables = self.evaluate('ls{extract /(file\\d)/ as files}{extract /(file1)/ as files}\n') + self.assertEqual(variables['files'], ['file1']) + + def testIntoAppendsTheValues(self): + text = '{files = "old"}\nls{extract /(file\\d)/ into files}{extract /(file1)/ into files}\n' + variables = self.evaluate(text) + self.assertEqual(variables['files'], ['old', 'file1', 'file2', 'file1']) + + def testSeveralExtracts(self): + text = ( + 'dir {\n' + ' extract /^[^d].*\\s(\\S+)$/ into files\n' + ' extract /^d.*\\s(\\S+)$/ into directories\n' + '}\n' + ) + self.device.add_command('dir', 'drwx 2 home\n-rw- 1 notes.txt\ndrwx 3 tmp') + variables = self.evaluate(text) + self.assertEqual(variables['files'], ['notes.txt']) + self.assertEqual(variables['directories'], ['home', 'tmp']) + + def testExtractFromAVariable(self): + text = '{extract /^(d.*)/ into directories from lines}' + variables = self.evaluate(text, lines=['drwx a', '-rw- b', 'drwx c']) + self.assertEqual(variables['directories'], ['drwx a', 'drwx c']) + + def testExtractFromAString(self): + variables = self.evaluate('{extract /(.)(.)/ as a, b from "xy"}') + self.assertEqual((variables['a'], variables['b']), (['x'], ['y'])) + + def testExtractFromAFunctionResult(self): + variables = self.evaluate('{extract /^(\\w)/ as first from string.split("ab,cd", ",")}') + self.assertEqual(variables['first'], ['a', 'c']) + + def testNoMatch(self): + variables = self.evaluate('ls{extract /(nomatch)/ as found}\n') + self.assertEqual(variables['found'], []) + + def testGroupThatDidNotMatch(self): + variables = self.evaluate('{extract /(x)?(y)/ as a, b from "y"}') + self.assertEqual((variables['a'], variables['b']), ([None], ['y'])) + + def testVariableInTheRegularExpression(self): + variables = self.evaluate('ls{extract /^($prefix\\d)$/ as files}\n', prefix='file') + self.assertEqual(variables['files'], ['file1', 'file2']) + + def testOnSeveralLines(self): + variables = self.evaluate('show version{\n extract /^Version (\\S+)/\n as version\n}\n') + self.assertEqual(variables['version'], ['1.0']) + + def testNumberOfGroups(self): + with self.assertRaises(CompileError) as cm: + self.evaluate('show version{extract /(\\w+)/ as a, b}\n') + message = 'Extraction to 2 variable(s), but regex has 1 capture group(s) in undefined:1' + lines = ['show version{extract /(\\w+)/ as a, b}'] + self.assertError(cm.exception, lines, ' ' * 21 + '^' * 15, message) + self.assertSent() + + def testNumberOfGroupsAtRuntime(self): + # The variable turns the parentheses into literals when the template runs. + text = '{v = "\\\\"}{extract /$v(a$v)/ as x from "(a)"}' + with self.assertRaises(ExecuteError) as cm: + self.evaluate(text) + message = ( + 'Extraction to variable #1 has no corresponding regex capture group' + ' (index out of range). in undefined:1' + ) + self.assertError(cm.exception, [text], ' ' * 11 + '^' * 33, message) + + def testSyntaxErrors(self): + for text, marker, message in ( + ( + '{extract /(a)/ as __a__}', + ' ' * 18 + '^' * 5, + "Extraction to internal variable '__a__'", + ), + ('{extract /(a)(b)/ as a, a}', ' ' * 24 + '^', "Duplicate variable name 'a'"), + ( + '{extract /(a)/ foo a}', + ' ' * 15 + '^^^', + "Expected 'as' or 'into' but got varname 'foo'", + ), + ( + '{extract/(a)/ as v}', + ' ' * 8 + '^', + "Expected whitespace but got regex_delimiter '/'", + ), + ( + '{extract /(a)/ as }', + ' ' * 18 + '^', + "Expected varname but got close_curly_bracket '}'", + ), + ( + '{extract /(a)/}', + ' ' * 14 + '^', + "Expected one of whitespace, newline but got close_curly_bracket '}'", + ), + ): + with self.subTest(text=text): + with self.assertRaises(CompileError) as cm: + self.evaluate(text) + self.assertError(cm.exception, [text], marker, message + ' in undefined:1') + + def testNotInPaste(self): + with self.assertRaises(CompileError): + template.paste(self.conn, '{extract /(a)/ as v from "a"}') + + +def suite(): + return unittest.TestLoader().loadTestsFromTestCase(ExtractTest) + + +if __name__ == '__main__': + unittest.TextTestRunner(verbosity=2).run(suite()) diff --git a/tests/Exscript/interpreter/FailTest.py b/tests/Exscript/interpreter/FailTest.py new file mode 100644 index 00000000..26ca93eb --- /dev/null +++ b/tests/Exscript/interpreter/FailTest.py @@ -0,0 +1,111 @@ +import unittest + +from Exscript.interpreter.exception import ExscriptException, FailException +from Exscript.parselib.exception import CompileError, ExecuteError +from Exscript.protocols.exception import InvalidCommandException + +from .helpers import TemplateTestCase + + +class FailTest(TemplateTestCase): + """ + The fail statement. + """ + + def testFail(self): + with self.assertRaises(FailException) as cm: + self.evaluate('show version\n{fail "Error: Failed!"}\nls\n') + self.assertEqual(str(cm.exception), 'Error: Failed!') + self.assertIsInstance(cm.exception, ExscriptException) + self.assertSent('show version') + + def testFailIf(self): + text = '{fail "No POS interface found!" if "POS" not in interfaces}\nls\n' + self.evaluate(text, interfaces=['POS', 'Ethernet0']) + self.assertSent('ls') + with self.assertRaises(FailException) as cm: + self.evaluate(text, interfaces=['Ethernet0']) + self.assertEqual(str(cm.exception), 'No POS interface found!') + self.assertSent('ls') + + def testMessageWithVariables(self): + with self.assertRaises(FailException) as cm: + self.evaluate('{fail "$x on $__hostname__" if x is "bad"}', x='bad') + self.assertEqual(str(cm.exception), 'bad on r1') + + def testInABlock(self): + with self.assertRaises(FailException) as cm: + self.evaluate('{loop from 0 to 5 as i}\nls $i\n{fail "stop at " . i if i is 2}{end}') + self.assertEqual(str(cm.exception), 'stop at 2') + self.assertSent('ls 0', 'ls 1', 'ls 2') + + def testSyntaxErrors(self): + for text, line, marker, message in ( + ('{fail}', '{fail}', ' ^', "Expected whitespace but got close_curly_bracket '}'"), + ('{fail\n"x"}', '{fail', ' ^', "Expected whitespace but got newline '\\n'"), + ('{fail \n"x"}', '{fail ', ' ^', "Expected expression but got newline '\\n'"), + ( + '{fail "x" if}', + '{fail "x" if}', + ' ' * 12 + '^', + "Expected whitespace but got close_curly_bracket '}'", + ), + ): + with self.subTest(text=text): + with self.assertRaises(CompileError) as cm: + self.evaluate(text) + self.assertError(cm.exception, [line], marker, message + ' in undefined:1') + + +class TryTest(TemplateTestCase): + """ + The try statement, which ignores errors of the device in its block. + """ + + def setUp(self): + super().setUp() + self.device.add_command('bad', '% Invalid input detected') + + def testTry(self): + self.evaluate('{try}\nbad\nshow version\n{end}\nls\n') + self.assertSent('bad', 'ls') + + def testOnOneLine(self): + self.evaluate('{try}bad{end}\nls\n') + self.assertSent('bad', 'ls') + + def testWithoutError(self): + variables = self.evaluate('{try}\nshow version\nls\n{end}\n') + self.assertSent('show version', 'ls') + self.assertEqual(variables['__response__'], ['file1', 'file2']) + + def testNested(self): + self.evaluate('{try}{try}bad{end}\nshow version\nbad\nls\n{end}\nls\n') + self.assertSent('bad', 'show version', 'bad', 'ls') + + def testInACodeBlock(self): + variables = self.evaluate('{\n try\n x = 1\n end\n}') + self.assertEqual(variables['x'], [1]) + + def testOtherErrorsAreNotIgnored(self): + with self.assertRaises(FailException): + self.evaluate('{try}{fail "boom"}{end}\nls\n') + with self.assertRaises(ExecuteError): + self.evaluate('{try}{x = list.get(list.new(), 1)}{end}\nls\n') + self.assertSent() + + def testErrorsOutsideAreNotIgnored(self): + with self.assertRaises(InvalidCommandException): + self.evaluate('{try}\nls\n{end}\nbad\nshow version\n') + self.assertSent('ls', 'bad') + + +def suite(): + loader = unittest.TestLoader() + return unittest.TestSuite( + [loader.loadTestsFromTestCase(FailTest), loader.loadTestsFromTestCase(TryTest)] + ) + + +if __name__ == '__main__': + unittest.TextTestRunner(verbosity=2).run(suite()) diff --git a/tests/Exscript/interpreter/IfConditionTest.py b/tests/Exscript/interpreter/IfConditionTest.py new file mode 100644 index 00000000..3a6965bb --- /dev/null +++ b/tests/Exscript/interpreter/IfConditionTest.py @@ -0,0 +1,103 @@ +import unittest + +from Exscript.parselib.exception import CompileError + +from .helpers import TemplateTestCase + + +class IfConditionTest(TemplateTestCase): + """ + The if statement, with else and else if. + """ + + def testIf(self): + self.evaluate('{if 1}\nshow version\n{end}\n{if 0}\nls\n{end}\n') + self.assertSent('show version') + + def testElse(self): + text = '{if x is "a"}\nshow version\n{else}\nls\n{end}\n' + self.evaluate(text, x='b') + self.evaluate(text, x='a') + self.assertSent('ls', 'show version') + + def testElseIf(self): + text = ( + '{if x is 1}\n' + ' show one\n' + '{else if x is 2}\n' + ' show two\n' + '{else if x is 3}\n' + ' show three\n' + '{else}\n' + ' show other\n' + '{end}\n' + ) + for x in 1, 2, 3, 4: + self.evaluate(text, x=x) + self.assertSent('show one', 'show two', 'show three', 'show other') + + def testElseIfWithoutElse(self): + self.evaluate('{if 0}\nshow one\n{else if 0}\nshow two\n{end}\nls\n') + self.assertSent('ls') + + def testOnOneLine(self): + self.evaluate('{if 1}ls{end}\n{if 0}show version{else}show other{end}\n') + self.assertSent('ls', 'show other') + + def testInACodeBlock(self): + text = ( + '{\n' + ' x = 1\n' + ' if x is 1\n' + ' append "one" to result\n' + ' else\n' + ' append "other" to result\n' + ' end\n' + ' append "end" to result\n' + '}' + ) + variables = self.evaluate(text) + self.assertEqual(variables['result'], ['one', 'end']) + + def testNested(self): + text = '{loop from 0 to 4 as i}{if i gt 0}{if i lt 3}\nshow $i\n{end}{end}{end}' + self.evaluate(text) + self.assertSent('show 1', 'show 2') + + def testElseWithoutIf(self): + with self.assertRaises(CompileError) as cm: + self.evaluate('ls\n{else}\nls\n') + message = "Unexpected 'else' without a scope in undefined:2" + self.assertError(cm.exception, ['{else}'], ' ^^^^', message) + self.assertSent() + + def testEndWithoutIf(self): + with self.assertRaises(CompileError) as cm: + self.evaluate('{x = 1\nend}') + message = "Unexpected 'end' without a scope in undefined:2" + self.assertError(cm.exception, ['end}'], '^^^', message) + + def testMissingEnd(self): + with self.assertRaises(CompileError) as cm: + self.evaluate('{if 1}\nls\n') + self.assertError(cm.exception, [''], '^', "Unexpected EOF '' in undefined:3") + + def testMissingCondition(self): + with self.assertRaises(CompileError) as cm: + self.evaluate('{if}ls{end}\n') + message = "Expected whitespace but got close_curly_bracket '}' in undefined:1" + self.assertError(cm.exception, ['{if}ls{end}'], ' ^', message) + + def testTwoConditions(self): + with self.assertRaises(CompileError) as cm: + self.evaluate('{if 1 2}ls{end}\n') + message = "Unexpected number '2' in undefined:1" + self.assertError(cm.exception, ['{if 1 2}ls{end}'], ' ^', message) + + +def suite(): + return unittest.TestLoader().loadTestsFromTestCase(IfConditionTest) + + +if __name__ == '__main__': + unittest.TextTestRunner(verbosity=2).run(suite()) diff --git a/tests/Exscript/interpreter/LoopTest.py b/tests/Exscript/interpreter/LoopTest.py new file mode 100644 index 00000000..8b8f147d --- /dev/null +++ b/tests/Exscript/interpreter/LoopTest.py @@ -0,0 +1,127 @@ +import unittest + +from Exscript.parselib.exception import CompileError, ExecuteError + +from .helpers import TemplateTestCase + + +class LoopTest(TemplateTestCase): + """ + The loop statement: over lists, over a range of numbers, and with while + or until conditions. + """ + + def testLoopOverAList(self): + variables = self.evaluate( + '{loop interfaces as interface}\nshow $interface\n{end}', interfaces=['e0', 'e1'] + ) + self.assertSent('show e0', 'show e1') + self.assertEqual(variables['interface'], ['e1']) + + def testLoopOverAnEmptyList(self): + self.evaluate('{loop list.new() as x}\nshow $x\n{end}\nls\n') + self.assertSent('ls') + + def testLoopOverAFunctionResult(self): + self.evaluate('{loop string.split("a,b", ",") as x}\nshow $x\n{end}') + self.assertSent('show a', 'show b') + + def testLoopOverSeveralLists(self): + text = '{loop modes,\n directories\n as mode,\n directory}\nls $mode $directory\n{end}' + self.evaluate(text, modes=['drwx', '-rw'], directories=['a', 'b']) + self.assertSent('ls drwx a', 'ls -rw b') + + def testListsOfDifferentLengths(self): + text = '\n{loop a, b as i, j}\ncmd $i $j\n{end}\n' + with self.assertRaises(ExecuteError) as cm: + self.evaluate(text, a=['1', '2'], b=['x']) + lines = ['{loop a, b as i, j}', 'cmd $i $j', '{end}'] + message = 'All list variables must have the same length in undefined:2-4' + self.assertError(cm.exception, lines, '^' * 19, message) + self.assertSent() + + def testNumberOfListsAndVariables(self): + with self.assertRaises(CompileError) as cm: + self.evaluate('{loop a as i, j}\ncmd $i\n{end}\n', a=['1']) + message = 'Loop of 1 list(s), but 2 loop variable(s) defined in undefined:1' + self.assertError(cm.exception, ['{loop a as i, j}'], ' ' + '^' * 14, message) + + def testLoopFromTo(self): + variables = self.evaluate('{loop from 1 to 4}\nls $counter\n{end}') + self.assertSent('ls 1', 'ls 2', 'ls 3') + self.assertEqual(variables['counter'], [3]) + + def testLoopFromToAs(self): + self.evaluate('{loop from n * 2 to n * 3 as slot}\nshow diag $slot\n{end}', n=2) + self.assertSent('show diag 4', 'show diag 5') + + def testLoopFromToWithoutNumbers(self): + self.evaluate('{loop from 3 to 3}\nls\n{end}\n{loop from 3 to 1}\nls\n{end}') + self.assertSent() + + def testLoopWhile(self): + variables = self.evaluate('{n = 0}{loop while n lt 3}{n = n + 1}\nls $n\n{end}') + self.assertSent('ls 1', 'ls 2', 'ls 3') + self.assertEqual(variables['n'], [3]) + + def testLoopUntil(self): + variables = self.evaluate( + '{number = 0}\n{loop until number is 3}\n{number = number + 1}\nls\n{end}' + ) + self.assertSent('ls', 'ls', 'ls') + self.assertEqual(variables['number'], [3]) + + def testConditionIsCheckedFirst(self): + self.evaluate('{loop while 0}\nls\n{end}\n{loop until 1}\nls\n{end}\nshow version\n') + self.assertSent('show version') + + def testLoopOverAListWhile(self): + text = '{loop directories as directory while directory is not "stop"}\nls $directory\n{end}' + variables = self.evaluate(text, directories=['a', 'b', 'stop', 'c']) + self.assertSent('ls a', 'ls b') + self.assertEqual(variables['directory'], ['stop']) + + def testLoopOverAListUntil(self): + text = ( + '{loop directories as directory until directory is "my_subdir"}\nls $directory\n{end}' + ) + self.evaluate(text, directories=['a', 'my_subdir', 'b']) + self.assertSent('ls a') + + def testLoopFromToWhile(self): + self.evaluate('{loop from 0 to 10 as i while i lt 2}\nls $i\n{end}') + self.assertSent('ls 0', 'ls 1') + + def testLoopFromToUntil(self): + self.evaluate('{loop from 0 to 10 until counter is 2}\nls $counter\n{end}') + self.assertSent('ls 0', 'ls 1') + + def testNestedLoops(self): + text = '{loop from 0 to 2 as i}{loop from 0 to 2 as j}\nls $i$j\n{end}{end}' + self.evaluate(text) + self.assertSent('ls 00', 'ls 01', 'ls 10', 'ls 11') + + def testInACodeBlock(self): + variables = self.evaluate('{\n loop from 1 to 3 as i\n append i to numbers\n end\n}') + self.assertEqual(variables['numbers'], [1, 2]) + + def testSyntaxErrors(self): + for text, marker, message in ( + ('{loop}', ' ^', "Expected whitespace but got close_curly_bracket '}'"), + ('{loop a i}', ' ^', "Expected 'as' but got varname 'i'"), + ('{loop a}', ' ^', "Expected 'as' but got close_curly_bracket '}'"), + ('{loop a as "i"}', ' ' * 11 + '^', "Expected varname but got string_delimiter '\"'"), + ('{loop from 1 2}', ' ' * 13 + '^', "Expected 'to' but got number '2'"), + ): + with self.subTest(text=text): + with self.assertRaises(CompileError) as cm: + self.evaluate(text + '\nls\n{end}\n', a=['1']) + self.assertError(cm.exception, [text], marker, message + ' in undefined:1') + + +def suite(): + return unittest.TestLoader().loadTestsFromTestCase(LoopTest) + + +if __name__ == '__main__': + unittest.TextTestRunner(verbosity=2).run(suite()) diff --git a/tests/Exscript/interpreter/ParserPropertyTest.py b/tests/Exscript/interpreter/ParserPropertyTest.py new file mode 100644 index 00000000..e39f16e7 --- /dev/null +++ b/tests/Exscript/interpreter/ParserPropertyTest.py @@ -0,0 +1,178 @@ +import re +import unittest +import warnings + +from hypothesis import assume, given, settings +from hypothesis import strategies as st + +from Exscript.interpreter.exception import PermissionError +from Exscript.parselib.exception import CompileWarning, LexerException +from Exscript.util import template + +from .helpers import RecordingDevice, connect + +# The exceptions that the compilation of a template may raise. +COMPILE_ERRORS = (LexerException, NameError, PermissionError, CompileWarning) + +# Pieces of templates, to reach more paths of the parser than random text. +# The regular expressions of extract statements are always given: without one, +# the parser fails with re.error (reported). +FRAGMENTS = [ + '{', + '}', + '\n', + '\r', + ' ', + ' ', + '\t', + '"', + '/', + '\\', + '\\\n', + '$', + '#', + ' # ', + '=', + ',', + '(', + ')', + '.', + '+', + '-', + '*', + '%', + '?', + 'x', + 'y', + 'x = ', + 'ls', + '1', + '0x1f', + '017', + 'true', + '"a"', + '"$x"', + '"$nothing"', + '$x', + '$nothing', + '/(a)/', + '/a/i', + '\\$', + '\\n', + '\\0', + '\\e', + '\\x41', + '\\u00e9', + '\\U0010FFFF', + 'if ', + 'else', + 'end', + 'loop ', + 'from ', + ' to ', + ' as ', + 'into ', + 'extract /', + 'append ', + 'fail ', + 'try', + 'enter', + 'while ', + 'until ', + 'not ', + ' and ', + ' or ', + ' is ', + ' in ', + ' matches ', + ' ge ', + 'list.new(', + 'string.toupper(', + 'sys.exec(', + '__hostname__', + '', + '', + '', + '', + '\x00', + '\u00e9', +] + +# Escape sequences above U+10FFFF raise SyntaxError instead of CompileError (reported). +INVALID_ESCAPE = re.compile(r'\\U([0-9a-fA-F]{8})') + +# Text that the template language does not interpret: no code blocks, no +# escape sequences, no variables, no comments and no elements. +plain_text = st.text( + st.characters(categories=('L', 'M', 'N', 'P', 'S', 'Zs'), exclude_characters='{}\\$#<'), + max_size=20, +) +plain_lines = st.lists( + st.tuples(st.sampled_from(['', ' ', '\t', ' ']), plain_text).map(''.join), max_size=8 +) + + +class ParserPropertyTest(unittest.TestCase): + """ + Properties of the compilation of any text. + """ + + def setUp(self): + self.device = RecordingDevice() + self.conn = connect(self.device) + self.addCleanup(self.conn.close, force=True) + + def assertCompiles(self, text): + """ + Compiles the text, and checks that an error shows a position in it. + """ + assume(not any(int(code, 16) > 0x10FFFF for code in INVALID_ESCAPE.findall(text))) + try: + with warnings.catch_warnings(): + warnings.simplefilter('error', CompileWarning) + template.test(text, x='1', y=['a', 'b']) + except COMPILE_ERRORS as exc: + self.assertPosition(str(exc), text) + + def assertPosition(self, message, text): + """ + Checks that the message shows lines of the text with a marker in them. + """ + match = re.search(r' in undefined:(\d+)(?:-(\d+))?\Z', message) + self.assertIsNotNone(match, message) + first = int(match.group(1)) + last = int(match.group(2) or first) + lines = text.split('\n') + self.assertTrue(1 <= first <= last <= len(lines), message) + shown = message.split('\n') + self.assertEqual(shown[0], '', message) + self.assertEqual(shown[1 : last - first + 2], lines[first - 1 : last]) + marker = shown[last - first + 2] + self.assertRegex(marker, r'^ *\^+$') + self.assertLessEqual(len(marker), max(len(line) for line in lines[first - 1 : last]) + 1) + + @settings(max_examples=150, deadline=None) + @given(st.text(max_size=80)) + def testAnyText(self, text): + self.assertCompiles(text) + + @settings(max_examples=200, deadline=None) + @given(st.lists(st.sampled_from(FRAGMENTS), max_size=30).map(''.join)) + def testPiecesOfTemplates(self, text): + self.assertCompiles(text) + + @settings(max_examples=100, deadline=None) + @given(plain_lines) + def testPlainTextIsSentAsCommands(self, lines): + del self.device.received[:] + template.paste(self.conn, '\n'.join(lines) + '\n') + expected = [line.lstrip(' \t') + '\r' for line in lines if line.strip()] + self.assertEqual(self.device.received, expected) + + +def suite(): + return unittest.TestLoader().loadTestsFromTestCase(ParserPropertyTest) + + +if __name__ == '__main__': + unittest.TextTestRunner(verbosity=2).run(suite()) diff --git a/tests/Exscript/interpreter/ParserTest.py b/tests/Exscript/interpreter/ParserTest.py new file mode 100644 index 00000000..7b7ea8bd --- /dev/null +++ b/tests/Exscript/interpreter/ParserTest.py @@ -0,0 +1,343 @@ +import contextlib +import io +import os +import tempfile +import unittest +from unittest import mock + +from Exscript import stdlib +from Exscript.interpreter import Parser +from Exscript.interpreter.exception import PermissionError +from Exscript.interpreter.program import Program +from Exscript.interpreter.scope import Scope +from Exscript.parselib import Lexer +from Exscript.parselib.exception import CompileError, ExecuteError +from Exscript.util import template + +from .helpers import split_error + + +class ParserTest(unittest.TestCase): + """ + Compiling templates with the Parser. + """ + + CORRELATE = Parser + + def testConstructor(self): + # Unlike Exscript.util.template, the parser allows all functions by default. + parser = Parser() + parser.define_object(**stdlib.functions) + parser.parse('{sys.env("HOME")}') + parser = Parser(secure=True) + parser.define_object(**stdlib.functions) + self.assertRaises(PermissionError, parser.parse, '{sys.env("HOME")}') + parser.parse('{list.new()}') + + parser = Parser(no_prompt=True) + self.assertRaises(CompileError, parser.parse, '{extract /(a)/ as a from "a"}') + + def testDefine(self): + def function(scope): + return [True] + + parser = Parser() + parser.define(a='x', b=1, c=['y', 'z'], d=('t',), e=function) + self.assertEqual( + parser.variables, {'a': ['x'], 'b': [1], 'c': ['y', 'z'], 'd': ('t',), 'e': function} + ) + variables = parser.parse('{x = a . b . c . e()}').execute() + self.assertEqual(variables['x'], ['x1yTrue']) + + def testDefineObject(self): + parser = Parser() + parser.define_object(a='x', b=1) + self.assertEqual(parser.variables, {'a': 'x', 'b': 1}) + + def testParse(self): + parser = Parser() + parser.define(items=['a']) + program = parser.parse('{x = items}{append "b" to x}', 'my.exscript') + self.assertIsInstance(program, Program) + self.assertEqual(program.execute()['x'], ['a', 'b']) + self.assertEqual(parser.parse('{y = items}').execute()['y'], ['a']) + self.assertEqual(parser.variables, {'items': ['a']}) + + with self.assertRaises(CompileError) as cm: + parser.parse('\n{x = }', 'my.exscript') + message = "Expected term but got close_curly_bracket '}' in my.exscript:2" + self.assertEqual(split_error(cm.exception), (['{x = }'], ' ^', message)) + + def testParseFile(self): + tmpdir = tempfile.TemporaryDirectory() + self.addCleanup(tmpdir.cleanup) + filename = os.path.join(tmpdir.name, 'test.exscript') + with open(filename, 'w', encoding='utf-8') as fp: + fp.write('{x = "Gr\u00fc\u00dfe"}\n{y = }\n') + with self.assertRaises(CompileError) as cm: + Parser().parse_file(filename) + message = "Expected term but got close_curly_bracket '}' in %s:2" % filename + self.assertEqual(split_error(cm.exception), (['{y = }'], ' ^', message)) + + with open(filename, 'w', encoding='utf-8') as fp: + fp.write('{x = "Gr\u00fc\u00dfe"}\n') + variables = Parser().parse_file(filename).execute() + self.assertEqual(variables['x'], ['Gr\u00fc\u00dfe']) + + +class ProgramTest(unittest.TestCase): + """ + Running compiled templates. + """ + + def setUp(self): + parser = Parser() + parser.define(x='a') + self.program = parser.parse('{y = x . "!"}') + + def testExecute(self): + self.assertEqual(self.program.execute()['y'], ['a!']) + variables = self.program.execute(variables={'x': ['b'], 'z': ['c']}) + self.assertEqual((variables['y'], variables['z']), (['b!'], ['c'])) + self.assertEqual(self.program.execute()['y'], ['a!']) + + def testInit(self): + self.program.init(x='b', y=['ignored'], **{'f.g': 'ignored', '_h': 'ignored'}) + variables = self.program.execute() + self.assertEqual(variables['y'], ['b!']) + self.assertNotIn('f.g', variables) + self.assertNotIn('_h', variables) + self.program.init(x=['c', 'd']) + self.assertEqual(self.program.execute()['y'], ['c!']) + + def testVariableThatIsUndefinedAtRuntime(self): + with self.assertRaises(ExecuteError) as cm: + self.program.execute(variables={'x': None}) + expected = (['{y = x . "!"}'], ' ^', "Undefined variable 'x' in None:1") + self.assertEqual(split_error(cm.exception), expected) + + def testFunctionThatIsUndefinedAtRuntime(self): + parser = Parser() + parser.define_object(**stdlib.functions) + program = parser.parse('{x = list.new()}') + with self.assertRaises(ExecuteError) as cm: + program.execute(variables={'list.new': None}) + expected = ( + ['{x = list.new()}'], + ' ^^^^^^^^', + "Undefined function 'list.new' in None:1", + ) + self.assertEqual(split_error(cm.exception), expected) + + +class ScopeTest(unittest.TestCase): + """ + The scopes that hold the variables; functions of the template language + receive them as their first argument. + """ + + CORRELATE = Scope + + def setUp(self): + lexer = Lexer(object) + self.root = Scope('Root', lexer, None, variables={'a': [1]}) + self.child = Scope('Child', lexer, None, self.root) + + def testConstructor(self): + lexer = Lexer(object) + with self.assertRaises(TypeError) as cm: + Scope('Root', lexer, None, variables={'x': 'a'}) + self.assertEqual( + str(cm.exception), "Root: variable x must be a list, but got " + ) + Scope('Root', lexer, None, variables={'f.g': 'a', '_h': 1, 'i': []}) + + def testDefine(self): + self.child.define(b='x', c=['y'], **{'f.g': 'function', '_h': 'private'}) + self.assertEqual( + self.root.variables, + {'a': [1], 'b': ['x'], 'c': ['y'], 'f.g': 'function', '_h': 'private'}, + ) + self.assertEqual(self.child.variables, {}) + + def testDefineObject(self): + self.child.define_object(b='x') + self.assertEqual(self.child.variables, {'b': 'x'}) + self.assertEqual(self.root.variables, {'a': [1]}) + + def testGet(self): + self.child.define_object(b='x') + self.assertEqual(self.child.get('a'), [1]) + self.assertEqual(self.child.get('b'), 'x') + self.assertIsNone(self.root.get('b')) + self.assertEqual(self.root.get('b', 'default'), 'default') + + def testIsDefined(self): + self.child.define_object(b='x') + self.assertTrue(self.child.is_defined('a')) + self.assertTrue(self.child.is_defined('b')) + self.assertFalse(self.root.is_defined('b')) + + def testGetVars(self): + self.child.define_object(a=[2], b='x') + self.assertEqual(self.child.get_vars(), {'a': [2], 'b': 'x'}) + self.assertEqual(self.root.get_vars(), {'a': [1]}) + + def testCopyPublicVars(self): + self.child.define_object(b=['x'], _c='private') + copied = self.child.copy_public_vars() + self.assertEqual(copied, {'a': [1], 'b': ['x']}) + copied['b'].append('y') + self.assertEqual(self.child.get('b'), ['x']) + + def testExitRequest(self): + self.assertFalse(self.child.exit_requested) + self.child.exit_request() + self.assertTrue(self.child.exit_requested) + + def testValue(self): + self.assertEqual(self.root.value(self.root), 1) + program = Parser().parse('{x = 1}{y = "a"}') + self.assertEqual(program.value(program), ['a']) + + def testDump(self): + out = io.StringIO() + with contextlib.redirect_stdout(out): + self.root.add(self.child) + self.root.dump() + self.assertEqual(out.getvalue(), 'Root start\n Child start\n Child end\nRoot end\n') + + def testDumpVars(self): + out = io.StringIO() + with contextlib.redirect_stdout(out): + self.child.dump_vars() + self.assertEqual(out.getvalue(), "Scope: {'a': [1]}\n") + + +class CodeTest(unittest.TestCase): + """ + Syntax errors in code blocks. + """ + + def assertCompileError(self, text, lines, marker, message): + with self.assertRaises(CompileError) as cm: + template.test(text) + self.assertEqual(split_error(cm.exception), (lines, marker, message)) + + def testUnexpectedToken(self): + self.assertCompileError( + '{x = 1 ,}', ['{x = 1 ,}'], ' ^', "Unexpected comma ',' in undefined:1" + ) + self.assertCompileError('{as}', ['{as}'], ' ^^', "Unexpected keyword 'as' in undefined:1") + + def testUnterminatedCodeBlock(self): + self.assertCompileError('{x = 1\n', [''], '^', "Unexpected EOF '' in undefined:2") + + def testComments(self): + text = '{# a comment\n x = 1\n# a comment\n # another comment\n y = 2 # not a comment\n}' + with self.assertRaises(CompileError): + Parser().parse(text) + variables = Parser().parse(text.replace(' # not a comment', '')).execute() + self.assertEqual((variables['x'], variables['y']), ([1], [2])) + + +class DumpTest(unittest.TestCase): + """ + The dump of the parse tree, as with exscript --parser-verbose 5. + """ + + def dump(self, text, **kwargs): + out = io.StringIO() + with mock.patch.object(template, 'debug', 5), contextlib.redirect_stdout(out): + template.test(text, **kwargs) + return out.getvalue() + + def testDump(self): + expected = ( + 'Program start\n' + ' Template start\n' + " Execute 'show $x'\n" + ' Code start\n' + ' Assign y start\n' + ' Expression start\n' + ' ExpressionNode + start\n' + ' Term\n' + ' Number 1\n' + ' Operator +\n' + ' ExpressionNode None start\n' + ' Term\n' + ' Variable x\n' + ' Operator None\n' + ' ExpressionNode None end\n' + ' ExpressionNode + end\n' + ' Expression end\n' + ' Assign y end\n' + ' Code end\n' + ' Template end\n' + 'Program end\n' + ) + self.assertEqual(self.dump('show $x\n{y = 1 + x}', x='1'), expected) + + def testDumpOfAllStatements(self): + text = ( + '{\n' + ' append "a" to l\n' + ' if not 1 is 2\n' + ' sys.message("yes")\n' + ' else\n' + ' fail "no" if 1 matches /1/i\n' + ' end\n' + ' loop l as i\n' + ' enter\n' + ' end\n' + ' loop from 0 to 2 while 1\n' + ' end\n' + ' loop until 1\n' + ' end\n' + ' try\n' + ' end\n' + ' extract /(a)/ as a from l\n' + ' if 0\n' + ' end\n' + ' fail "always"\n' + '}\n' + '\n' + ) + dump = self.dump(text) + for line in ( + 'Append to l', + 'IfCondition start', + 'ExpressionNode not start', + 'FunctionCall sys.message start', + 'Fail start', + "String 'always'", + "Regex '1'", + 'Loop ', + 'While start', + 'Until start', + 'Enter', + "Execute ''", + 'Try start', + "Extract /(a)/ as a from ", + "String 'yes'", + 'RPCBody start', + "Execute ''", + ): + self.assertIn(line, dump) + + +def suite(): + loader = unittest.TestLoader() + return unittest.TestSuite( + [ + loader.loadTestsFromTestCase(ParserTest), + loader.loadTestsFromTestCase(ProgramTest), + loader.loadTestsFromTestCase(ScopeTest), + loader.loadTestsFromTestCase(CodeTest), + loader.loadTestsFromTestCase(DumpTest), + ] + ) + + +if __name__ == '__main__': + unittest.TextTestRunner(verbosity=2).run(suite()) diff --git a/tests/Exscript/interpreter/RpcTest.py b/tests/Exscript/interpreter/RpcTest.py new file mode 100644 index 00000000..32e303b1 --- /dev/null +++ b/tests/Exscript/interpreter/RpcTest.py @@ -0,0 +1,106 @@ +import unittest + +from Exscript.parselib.exception import CompileError, CompileWarning, ExecuteError + +from .helpers import TemplateTestCase, split_error + + +class RpcTest(TemplateTestCase): + """ + NETCONF RPCs in templates: the body of each element is a command. + """ + + def testRpc(self): + self.evaluate('\n \n \n \n\n') + self.assertSent('\n \n') + + def testRpcOnOneLine(self): + self.evaluate('\nls\n') + self.assertSent('', 'ls') + + def testCommandBeforeAnRpc(self): + self.evaluate('ls\n') + self.assertSent('ls', '') + + def testRpcInABlock(self): + self.evaluate('{if 1}{end}\n{if 0}{end}\n') + self.assertSent('') + + def testResponse(self): + self.device.add_command('', 'abc') + variables = self.evaluate('{extract /([^<]*)<\\/data>/ as data}\n') + self.assertEqual(variables['data'], ['abc']) + + def testCommentsAreNotSent(self): + self.evaluate('\n # a comment\n \n # another comment\n \n\n') + self.assertSent('\n') + + def testVariables(self): + text = '<$source/>\n' + self.evaluate(text, source='candidate') + self.assertSent('') + + def testEmptyRpc(self): + self.evaluate('\n\n\n') + self.assertSent() + + def testUndefinedVariable(self): + with self.assertRaises(ExecuteError) as cm: + self.evaluate('\n $nothing\n\n') + message = "Undefined variable 'nothing' in undefined:2" + self.assertError(cm.exception, [' $nothing'], ' ^^^^^^^^', message) + + def testInvalidXml(self): + with self.assertRaises(CompileError) as cm: + self.evaluate('ls\n\n \n\n') + lines, marker, message = split_error(cm.exception) + self.assertEqual((lines, marker), ([''], ' ^')) + self.assertTrue( + message.startswith( + 'XML validation of the RPC command has failed. The error was caused by:\n' + 'XMLSyntaxError: ' + ), + message, + ) + self.assertTrue(message.endswith(' in undefined:4'), message) + self.assertSent() + + def testInvalidXmlWithVariables(self): + # The values of the variables might make the XML valid. + with self.assertWarns(CompileWarning) as cm: + self.evaluate('\n <$tag>\n\n', tag='get/') + lines, marker, message = split_error(cm.warning) + self.assertEqual((lines, marker), ([''], ' ^')) + self.assertIn('This could be a real syntax error or might depend on the final', message) + self.assertSent('') + + def testCharacterThatIsNotUtf8(self): + with self.assertRaises(CompileError) as cm: + self.evaluate('\n \ud800\n\n') + lines, marker, message = split_error(cm.exception) + self.assertEqual((lines, marker), ([' \ud800'], ' ^')) + self.assertIn('UnicodeEncodeError', message) + + def testSyntaxErrors(self): + for text, lines, marker, message in ( + ('\n \n', [''], '^', "Expected '' but got EOF in undefined:3"), + ( + '\n', + [''], + ' ^^^^^', + "Unexpected open_rpc_tag '' in undefined:1", + ), + ('\n', [''], '^^^^^^', "Unexpected close_rpc_tag '' in undefined:1"), + ): + with self.subTest(text=text): + with self.assertRaises(CompileError) as cm: + self.evaluate(text) + self.assertError(cm.exception, lines, marker, message) + + +def suite(): + return unittest.TestLoader().loadTestsFromTestCase(RpcTest) + + +if __name__ == '__main__': + unittest.TextTestRunner(verbosity=2).run(suite()) diff --git a/tests/Exscript/interpreter/StringTest.py b/tests/Exscript/interpreter/StringTest.py new file mode 100644 index 00000000..78c92f35 --- /dev/null +++ b/tests/Exscript/interpreter/StringTest.py @@ -0,0 +1,128 @@ +import unittest + +from Exscript.parselib.exception import CompileError, ExecuteError + +from .helpers import TemplateTestCase, split_error + + +class StringTest(TemplateTestCase): + """ + Strings in double quotes: escape sequences and variables. + """ + + def assertString(self, text, expected, **kwargs): + variables = self.evaluate('{x = "%s"}' % text, **kwargs) + self.assertEqual(variables['x'], [expected]) + + def testEscapeSequences(self): + text = 'a\\tb\\nc\\x41\\101\\u00e9\\U0001F600\\e\\0\\a\\v\\f\\b\\r' + self.assertString(text, 'a\tb\ncAA\u00e9\U0001f600\x1b\x00\x07\x0b\x0c\x08\r') + + def testEscapedCharacters(self): + self.assertString('\\"q\\" \\\\ \\/ \\q \\{\\}', '"q" \\ / q {}') + + def testNoEscaping(self): + text = '\\x41\\101\\u00e9\\e|\\t|\\n|\\r' + self.assertString(text, 'x41101u00e9e|\t|\n|\r', no_escaping=True) + + def testVariables(self): + variables = self.evaluate( + '{a = "1"}{b = list.new("x", "y")}{n = 2 * 3}{e = list.new()}' + '{x = "$a-$b-$n-$__hostname__-[$e]"}' + ) + self.assertEqual(variables['x'], ['1-x\ny-6-r1-[]']) + + def testVariableNames(self): + variables = self.evaluate('{a = "1"}{a_b = "2"}{x = "$a.b $a_b $a_b2"}', a_b2='3') + self.assertEqual(variables['x'], ['1.b 2 3']) + + def testDollarSign(self): + variables = self.evaluate('{a = "1"}{x = "\\$a $ a$"}') + self.assertEqual(variables['x'], ['$a $ a$']) + + def testUndefinedVariable(self): + with self.assertRaises(ExecuteError) as cm: + self.evaluate('ls\n{x = "a $nothing b"}') + message = "Undefined variable 'nothing' in undefined:2" + self.assertError(cm.exception, ['{x = "a $nothing b"}'], ' ^^^^^^^^', message) + self.assertSent() + + def testInvalidVariableName(self): + with self.assertRaises(ExecuteError) as cm: + self.evaluate('{x = "$1abc"}') + message = "'1abc' is not a variable name in undefined:1" + self.assertError(cm.exception, ['{x = "$1abc"}'], ' ^^^^^', message) + + def testLineBreak(self): + with self.assertRaises(CompileError) as cm: + self.evaluate('{x = "a\nb"}') + message = "Expected string but got newline '\\n' in undefined:1" + self.assertError(cm.exception, ['{x = "a'], ' ^', message) + + def testBackslashBeforeLineBreak(self): + with self.assertRaises(CompileError) as cm: + self.evaluate('{x = "a\\\nb"}') + message = "Invalid syntax: '\\\\\\nb\"}' in undefined:1" + self.assertError(cm.exception, ['{x = "a\\'], ' ^', message) + + def testUnterminatedString(self): + with self.assertRaises(CompileError) as cm: + self.evaluate('{x = "abc}') + message = "Expected string but got EOF '' in undefined:1" + self.assertError(cm.exception, ['{x = "abc}'], ' ' * 10 + '^', message) + + +class RegexTest(TemplateTestCase): + """ + Regular expressions between slashes. + """ + + def assertMatches(self, text, expected, **kwargs): + variables = self.evaluate('{x = %s}' % text, **kwargs) + self.assertEqual(variables['x'], [expected]) + + def testModifier(self): + self.assertMatches('"ABC" matches /b/', 0) + self.assertMatches('"ABC" matches /b/i', 1) + self.assertMatches('"ABC" matches /b/i and 1', 1) + + def testEscapes(self): + self.assertMatches('"a/b" matches /^a\\/b$/', 1) + self.assertMatches('"a1 b" matches /^\\w\\d\\s\\x62/', 1) + self.assertMatches('"a$" matches /^a[$]$/', 1) + + def testVariables(self): + self.assertMatches('"abc" matches /^a$p$/', 1, p='bc') + self.assertMatches('"abc" matches /^a$p$/', 0, p='b') + self.assertMatches('"a.c" matches /^a$p$/', 1, p='[.]c') + + def testInvalidModifier(self): + with self.assertRaises(CompileError) as cm: + self.evaluate('{x = /abc/x}') + message = "Invalid regular expression modifier 'x' in undefined:1" + self.assertError(cm.exception, ['{x = /abc/x}'], ' ' * 10 + '^', message) + + def testInvalidRegularExpression(self): + with self.assertRaises(CompileError) as cm: + self.evaluate('ls\n{x = /ab(c/}') + lines, marker, message = split_error(cm.exception) + self.assertEqual((lines, marker), (['{x = /ab(c/}'], ' ^^^^^^')) + self.assertRegex(message, r"^Invalid regular expression 'ab\(c': .+ in undefined:2$") + self.assertSent() + + def testLineBreak(self): + with self.assertRaises(CompileError) as cm: + self.evaluate('{x = /a\nb/}') + message = "Expected regex but got newline '\\n' in undefined:1" + self.assertError(cm.exception, ['{x = /a'], ' ^', message) + + +def suite(): + loader = unittest.TestLoader() + return unittest.TestSuite( + [loader.loadTestsFromTestCase(StringTest), loader.loadTestsFromTestCase(RegexTest)] + ) + + +if __name__ == '__main__': + unittest.TextTestRunner(verbosity=2).run(suite()) diff --git a/tests/Exscript/interpreter/__init__.py b/tests/Exscript/interpreter/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/Exscript/interpreter/helpers.py b/tests/Exscript/interpreter/helpers.py new file mode 100644 index 00000000..9f694582 --- /dev/null +++ b/tests/Exscript/interpreter/helpers.py @@ -0,0 +1,85 @@ +""" +Helpers for the tests of the template language: a virtual device that records +the commands that it receives, and a test case that runs templates on it. +""" + +import re +import unittest + +from Exscript.emulators import VirtualDevice +from Exscript.protocols import Dummy +from Exscript.util import template + + +class RecordingDevice(VirtualDevice): + """ + A virtual device that needs no login, answers unknown commands with the + prompt only, and records the commands that it receives. + """ + + def __init__(self, hostname='r1', **kwargs): + kwargs.setdefault('login_type', VirtualDevice.LOGIN_TYPE_NONE) + kwargs.setdefault('strict', False) + super().__init__(hostname, **kwargs) + self.received = [] + + def do(self, command): + self.received.append(command) + return super().do(command) + + +def connect(device, timeout=5): + """ + Returns a Dummy connection to the given device that has read the banner + and the first prompt. + """ + conn = Dummy(device=device, timeout=timeout) + conn.connect(device.hostname) + conn.expect_prompt() + return conn + + +def split_error(exc): + """ + Splits the message of an error that the lexer raised with a position + into the lines of the template that it shows, the line that marks the + position with carets, and the message below that line. + """ + lines = str(exc).split('\n') + for i, line in enumerate(lines): + if re.fullmatch(r' *\^+', line): + return lines[1:i], line, '\n'.join(lines[i + 1 :]) + raise AssertionError('The message shows no position: %r' % str(exc)) + + +class TemplateTestCase(unittest.TestCase): + """ + Runs templates on a RecordingDevice that knows the commands + "show version" and "ls". + """ + + def setUp(self): + self.device = RecordingDevice() + self.device.add_command('show version', 'Version 1.0\nUptime 5 days') + self.device.add_command('ls', 'file1\nfile2') + self.conn = connect(self.device) + self.addCleanup(self.conn.close, force=True) + + def evaluate(self, text, **kwargs): + """ + Runs the template with template.eval() and returns its variables. + """ + return template.eval(self.conn, text, **kwargs) + + def assertSent(self, *commands): + """ + Checks that the device received the given commands, each with a CR. + """ + self.assertEqual(self.device.received, [command + '\r' for command in commands]) + + def assertError(self, exc, lines, marker, message): + """ + Checks the lines of the template, the position marker and the message + that the given error shows. + """ + self.assertEqual(split_error(exc), (lines, marker, message)) diff --git a/tests/Exscript/util/templateTest.py b/tests/Exscript/util/templateTest.py index 7752c2b5..e05d841d 100644 --- a/tests/Exscript/util/templateTest.py +++ b/tests/Exscript/util/templateTest.py @@ -1,11 +1,47 @@ import os +import tempfile import unittest -from Exscript.emulators import VirtualDevice +from hypothesis import given, settings +from hypothesis import strategies as st +from interpreter.helpers import RecordingDevice, connect, split_error + +from Exscript import stdlib from Exscript.interpreter.exception import PermissionError -from Exscript.protocols import Dummy +from Exscript.parselib.exception import CompileError from Exscript.util import template +# The functions that run local commands or access local files. +INSECURE = { + 'file.chmod', + 'file.clear', + 'file.mkdir', + 'file.read', + 'file.rm', + 'file.write', + 'sys.env', + 'sys.exec', +} + +# Statements with a function call (%s), also where the call never runs. +CALLS = ( + '{%s}', + '{x = %s}', + '{if 0}{%s}{end}', + '{if %s}{end}', + '{loop from 0 to 0}{%s}{end}', + '{loop %s as item}{end}', + '{loop while %s}{end}', + '{fail "x" if %s}', + '{x = string.toupper(%s)}', + '{try}{%s}{end}', + '{append %s to y}', + '{extract /(a)/ as z from %s}', + 'ls\n{%s}', +) + +statements = st.tuples(st.sampled_from(sorted(stdlib.functions)), st.sampled_from(CALLS)) + def shout(scope, string): """A function that the caller passes to the template.""" @@ -16,9 +52,84 @@ class templateTest(unittest.TestCase): CORRELATE = template def setUp(self): - self.conn = Dummy(device=VirtualDevice('r1', strict=False), connect_rate_limit=0) - self.conn.connect('r1') + self.device = RecordingDevice() + self.device.add_command('ls', 'file1\nfile2') + self.conn = connect(self.device) self.addCleanup(self.conn.close, force=True) + tmpdir = tempfile.TemporaryDirectory() + self.addCleanup(tmpdir.cleanup) + self.tmpdir = tmpdir.name + + def write(self, text): + """ + Writes the given template to a file and returns its name. + """ + filename = os.path.join(self.tmpdir, 'test.exscript') + with open(filename, 'w', encoding='utf-8') as fp: + fp.write(text) + return filename + + def testTest(self): + self.assertIsNone(template.test('ls\n{x = y}', y='a')) + self.assertRaises(NameError, template.test, '{x = y}') + with self.assertRaises(CompileError) as cm: + template.test('ls\n{x = }') + message = "Expected term but got close_curly_bracket '}' in undefined:2" + self.assertEqual(split_error(cm.exception), (['{x = }'], ' ^', message)) + self.assertEqual(self.device.received, []) + + def testTestFile(self): + filename = self.write('ls\n{x = y . "\u00e9"}\n') + self.assertIsNone(template.test_file(filename, y='a')) + filename = self.write('ls\n{x = }\n') + with self.assertRaises(CompileError) as cm: + template.test_file(filename) + self.assertTrue(str(cm.exception).endswith(' in %s:2' % filename)) + self.assertEqual(self.device.received, []) + + def testEval(self): + variables = template.eval( + self.conn, 'ls\n{x = y}\n{u = __username__}{f = __filename__}', y='a' + ) + self.assertEqual(self.device.received, ['ls\r']) + self.assertEqual(variables['__response__'], ['file1', 'file2']) + self.assertEqual(variables['__hostname__'], ['r1']) + self.assertEqual( + (variables['x'], variables['u'], variables['f']), (['a'], ['undefined'], ['undefined']) + ) + self.assertIs(variables['__connection__'], self.conn) + + def testEvalWithOptions(self): + variables = template.eval(self.conn, 'ls\n', strip_command=False) + self.assertEqual(variables['__response__'], ['ls', 'file1', 'file2']) + template.eval(self.conn, 'ls \\x41\n', no_escaping=True) + self.assertEqual(self.device.received, ['ls\r', 'ls x41\r']) + + def testEvalBuiltinVariables(self): + # The caller may override the built-in variables. + variables = template.eval(self.conn, '{h = __hostname__}', __hostname__='other') + self.assertEqual((variables['h'], variables['__hostname__']), (['other'], ['other'])) + + def testEvalFile(self): + filename = self.write('ls\n{f = __filename__}\n') + variables = template.eval_file(self.conn, filename) + self.assertEqual(self.device.received, ['ls\r']) + self.assertEqual(variables['f'], [filename]) + variables = template.eval_file(self.conn, filename, strip_command=False) + self.assertEqual(variables['__response__'], ['ls', 'file1', 'file2']) + + def testPaste(self): + variables = template.paste(self.conn, 'ls\n{x = y}\nls \\x41\n', y='a') + self.assertEqual(self.device.received, ['ls\r', 'ls A\r']) + self.assertEqual((variables['__response__'], variables['x']), ([''], ['a'])) + template.paste(self.conn, 'ls \\x41\n', no_escaping=True) + self.assertEqual(self.device.received[2:], ['ls x41\r']) + + def testPasteFile(self): + filename = self.write('ls\n{f = __filename__}\n') + variables = template.paste_file(self.conn, filename) + self.assertEqual(self.device.received, ['ls\r']) + self.assertEqual((variables['__response__'], variables['f']), ([''], [filename])) def testSecureModeIsTheDefault(self): for source in '{sys.exec("true")}', '{sys.env("HOME")}', '{file.rm("x")}': @@ -27,8 +138,58 @@ def testSecureModeIsTheDefault(self): self.assertRaises(PermissionError, template.paste, self.conn, source) template.test(source, secure=False) + def testInsecureFunctionsAreForbidden(self): + for name in sorted(INSECURE): + text = 'ls\n{%s()}\n' % name + filename = self.write(text) + for function, args in ( + (template.test, (text,)), + (template.eval, (self.conn, text)), + (template.paste, (self.conn, text)), + (template.test_file, (filename,)), + (template.eval_file, (self.conn, filename)), + (template.paste_file, (self.conn, filename)), + ): + with self.subTest(name=name, function=function.__name__): + with self.assertRaises(PermissionError) as cm: + function(*args) + lines, marker, message = split_error(cm.exception) + self.assertEqual((lines, marker), (['{%s()}' % name], ' ' + '^' * len(name))) + origin = filename if function.__name__.endswith('_file') else 'undefined' + expected = 'Use of insecure function %r is not permitted in %s:2' % ( + name, + origin, + ) + self.assertEqual(message, expected) + self.assertEqual(self.device.received, []) + + def testInsecureFunctionsAreForbiddenEverywhere(self): + for call in CALLS: + text = call % 'file.read("x")' + with self.subTest(text=text): + self.assertRaises(PermissionError, template.test, text) + template.test(text, secure=False) + def testSecureFunctionsAreAllowed(self): template.test('{sys.message("hello")}{file.exists("x")}') + for name in sorted(set(stdlib.functions) - INSECURE): + with self.subTest(name=name): + template.test('{%s()}' % name) + + @settings(max_examples=100, deadline=None) + @given(st.lists(statements, min_size=1, max_size=4)) + def testOnlySecureFunctionsInSecureMode(self, calls): + text = '\n'.join(call % (name + '()') for name, call in calls) + '\n' + template.test(text, secure=False) + insecure = [name for name, _ in calls if name in INSECURE] + if not insecure: + template.test(text) + return + with self.assertRaises(PermissionError) as cm: + template.test(text) + self.assertIn( + 'Use of insecure function %r is not permitted' % insecure[0], str(cm.exception) + ) def testFunctionsOfTheCallerAreAllowed(self): variables = template.eval(self.conn, '{x = shout("hi")}', shout=shout) @@ -38,3 +199,11 @@ def testFunctionsOfTheCallerAreAllowed(self): def testExecDoesNotUseAShell(self): variables = template.eval(self.conn, '{sys.exec("echo a;echo b")}', secure=False) self.assertEqual(variables['__response__'], ['a;echo b\n']) + + +def suite(): + return unittest.TestLoader().loadTestsFromTestCase(templateTest) + + +if __name__ == '__main__': + unittest.TextTestRunner(verbosity=2).run(suite()) From 54851d8b99210de5d92affee7af2459ba5025582 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 10 Nov 2026 16:40:15 +0100 Subject: [PATCH 084/250] Test the standard library of the template language --- tests/Exscript/stdlib/__init__.py | 0 tests/Exscript/stdlib/connectionTest.py | 209 ++++++++++++++++++++++++ tests/Exscript/stdlib/cryptTest.py | 33 ++++ tests/Exscript/stdlib/fileTest.py | 108 ++++++++++++ tests/Exscript/stdlib/functionsTest.py | 52 ++++++ tests/Exscript/stdlib/ipv4Test.py | 81 +++++++++ tests/Exscript/stdlib/listTest.py | 63 +++++++ tests/Exscript/stdlib/mysysTest.py | 71 ++++++++ tests/Exscript/stdlib/stringTest.py | 57 +++++++ tests/Exscript/stdlib/utilTest.py | 33 ++++ 10 files changed, 707 insertions(+) create mode 100644 tests/Exscript/stdlib/__init__.py create mode 100644 tests/Exscript/stdlib/connectionTest.py create mode 100644 tests/Exscript/stdlib/cryptTest.py create mode 100644 tests/Exscript/stdlib/fileTest.py create mode 100644 tests/Exscript/stdlib/functionsTest.py create mode 100644 tests/Exscript/stdlib/ipv4Test.py create mode 100644 tests/Exscript/stdlib/listTest.py create mode 100644 tests/Exscript/stdlib/mysysTest.py create mode 100644 tests/Exscript/stdlib/stringTest.py create mode 100644 tests/Exscript/stdlib/utilTest.py diff --git a/tests/Exscript/stdlib/__init__.py b/tests/Exscript/stdlib/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/Exscript/stdlib/connectionTest.py b/tests/Exscript/stdlib/connectionTest.py new file mode 100644 index 00000000..51794ca5 --- /dev/null +++ b/tests/Exscript/stdlib/connectionTest.py @@ -0,0 +1,209 @@ +import unittest +from unittest import mock + +from interpreter.helpers import RecordingDevice, TemplateTestCase, split_error + +import Exscript.stdlib.connection +from Exscript import Account +from Exscript.emulators import VirtualDevice +from Exscript.parselib.exception import ExecuteError +from Exscript.protocols import Dummy +from Exscript.protocols.exception import InvalidCommandException + + +class connectionTest(TemplateTestCase): + """ + The functions of the template language that use the connection. + """ + + CORRELATE = Exscript.stdlib.connection + + def setUp(self): + super().setUp() + self.device.add_command('bad', '% Invalid input detected') + self.device.add_command('reload', 'Proceed with reload? [confirm]', prompt=False) + + def useLoginDevice(self, login=True): + """ + Connects to a device that asks for a user name and a password, logs + in as admin with the password "secret" unless login is False, and + forgets the commands of the login. + """ + device = RecordingDevice(login_type=VirtualDevice.LOGIN_TYPE_BOTH) + device.add_command('enable', 'Password: ', prompt=False) + device.add_command('relogin', lambda command: device.init(), prompt=False) + self.device = device + self.conn = Dummy(device=device, timeout=5) + self.conn.connect('r1') + self.addCleanup(self.conn.close, force=True) + if login: + self.conn.login(Account('admin', 'secret')) + del device.received[:] + + def testAuthenticate(self): + self.useLoginDevice() + variables = self.evaluate( + '{connection.sendline("relogin")}{connection.authenticate()}\nls\n' + ) + self.assertSent('relogin', 'admin', 'secret', 'ls') + self.assertEqual(variables['__username__'], ['admin']) + + def testAuthenticateWithoutAccount(self): + self.useLoginDevice(login=False) + with self.assertRaises(ExecuteError) as cm: + self.evaluate('{connection.authenticate()}') + message = ( + "TypeError while executing function 'connection.authenticate':" + ' An account is required in undefined:1' + ) + self.assertEqual(split_error(cm.exception)[2], message) + + def testAuthenticateUser(self): + self.useLoginDevice(login=False) + self.evaluate('{connection.authenticate_user("admin", "secret")}\nls\n') + self.assertSent('admin', 'secret', 'ls') + self.assertTrue(self.conn.is_app_authenticated()) + + def testAuthenticateUserAgain(self): + self.useLoginDevice() + self.evaluate('{connection.sendline("relogin")}{connection.authenticate_user()}\nls\n') + self.assertSent('relogin', 'admin', 'secret', 'ls') + + def testAuthorize(self): + self.useLoginDevice() + text = ( + '{connection.sendline("enable")}{connection.authorize()}\n' + '{connection.sendline("enable")}{connection.authorize("en")}\n' + 'ls\n' + ) + self.evaluate(text) + self.assertSent('enable', 'secret', 'enable', 'en', 'ls') + self.assertTrue(self.conn.is_app_authorized()) + + def testAutoAuthorize(self): + # The IOS driver starts the authorization with "enable". + self.useLoginDevice() + self.conn.set_driver('ios') + self.evaluate('{connection.auto_authorize()}\n{connection.auto_authorize("en")}\nls\n') + self.assertSent('enable', 'secret', 'enable', 'en', 'ls') + self.assertTrue(self.conn.is_app_authorized()) + + def testAutoAuthorizeWithoutCommand(self): + # The generic driver has no command to start the authorization. + self.useLoginDevice() + self.evaluate('{connection.sendline("enable")}{connection.auto_authorize("en")}\nls\n') + self.assertSent('enable', 'en', 'ls') + + def testAutoinit(self): + self.evaluate('{connection.autoinit()}') + self.assertSent() + self.conn.set_driver('ios') + self.evaluate('{connection.autoinit()}') + self.assertSent('term len 0', 'term width 0') + + def testClose(self): + with mock.patch.object(self.conn, 'close', wraps=self.conn.close) as close: + variables = self.evaluate('{connection.close()}') + close.assert_called_once_with(force=False) + self.assertFalse(self.conn.connected) + self.assertEqual(variables['__response__'], ['']) + + self.conn.connect('r1') + with mock.patch.object(self.conn, 'close', wraps=self.conn.close) as close: + self.evaluate('{connection.close(true)}') + close.assert_called_once_with(force=True) + self.assertFalse(self.conn.connected) + + def testExec_(self): + variables = self.evaluate('{connection.exec("ls\\r")}') + self.assertSent('ls') + self.assertEqual(variables['__response__'], ['file1', 'file2']) + variables = self.evaluate('{connection.exec(list.new("show version\\r", "ls\\r"))}') + self.assertSent('ls', 'show version', 'ls') + self.assertEqual( + variables['__response__'], ['Version 1.0', 'Uptime 5 days', 'file1', 'file2'] + ) + + def testExecline(self): + variables = self.evaluate('{connection.execline("ls")}') + self.assertSent('ls') + self.assertEqual(variables['__response__'], ['file1', 'file2']) + variables = self.evaluate( + '{connection.execline(commands)}', commands=['show version', 'ls'] + ) + self.assertSent('ls', 'show version', 'ls') + self.assertEqual( + variables['__response__'], ['Version 1.0', 'Uptime 5 days', 'file1', 'file2'] + ) + + def testExpect(self): + variables = self.evaluate( + '{connection.sendline("reload")}{connection.expect(/\\[confirm\\]/)}' + ) + self.assertSent('reload') + self.assertEqual(variables['__response__'], ['Proceed with reload? ']) + + def testGetProtocol(self): + self.assertEqual(self.evaluate('{x = connection.get_protocol()}')['x'], ['dummy']) + + def testGuessOs(self): + self.assertEqual(self.evaluate('{x = connection.guess_os()}')['x'], ['unknown']) + self.device.set_prompt('user@r1:~$ ') + self.assertEqual(self.evaluate('ls\n{x = connection.guess_os()}')['x'], ['shell']) + + def testSend(self): + variables = self.evaluate( + '{connection.send("reload\\r")}{connection.send(list.new("y", "\\r"))}' + ) + self.assertEqual(self.device.received, ['reload\r', 'y', '\r']) + self.assertEqual(variables['__response__'], ['']) + + def testSendline(self): + self.evaluate('{connection.sendline("reload")}{connection.sendline(list.new("y", "n"))}') + self.assertSent('reload', 'y', 'n') + + def testSetError(self): + with self.assertRaises(InvalidCommandException): + self.evaluate('{connection.set_error(/file2/)}ls\n') + self.evaluate('{connection.set_error(/nothing/)}bad\n') + with self.assertRaises(InvalidCommandException): + self.evaluate('{connection.set_error()}bad\n') + self.assertSent('ls', 'bad', 'bad') + + def testAddError(self): + with self.assertRaises(InvalidCommandException): + self.evaluate('{connection.add_error(/file2/)}ls\n') + with self.assertRaises(InvalidCommandException): + self.evaluate('{connection.add_error(/nothing/)}bad\n') + self.assertSent('ls', 'bad') + + def testSetPrompt(self): + variables = self.evaluate('{connection.set_prompt(/\\[confirm\\]/)}reload\n') + self.assertEqual(variables['__response__'], ['Proceed with reload? ']) + variables = self.evaluate('{connection.set_prompt()}y\nls\n') + self.assertSent('reload', 'y', 'ls') + self.assertEqual(variables['__response__'], ['file1', 'file2']) + + def testAddPrompt(self): + self.evaluate('{connection.add_prompt(/\\[confirm\\]/)}reload\ny\nls\n') + self.assertSent('reload', 'y', 'ls') + + def testSetTimeout(self): + self.evaluate('{connection.set_timeout(7)}') + self.assertEqual(self.conn.get_timeout(), 7) + self.evaluate('{connection.set_timeout("8")}') + self.assertEqual(self.conn.get_timeout(), 8) + + def testWaitFor(self): + text = '{connection.sendline("reload")}{connection.wait_for(/\\[confirm\\]/)}{connection.wait_for(/\\[confirm\\]/)}' + variables = self.evaluate(text) + self.assertSent('reload') + self.assertEqual(variables['__response__'], ['Proceed with reload? ']) + + +def suite(): + return unittest.TestLoader().loadTestsFromTestCase(connectionTest) + + +if __name__ == '__main__': + unittest.TextTestRunner(verbosity=2).run(suite()) diff --git a/tests/Exscript/stdlib/cryptTest.py b/tests/Exscript/stdlib/cryptTest.py new file mode 100644 index 00000000..4f08cd59 --- /dev/null +++ b/tests/Exscript/stdlib/cryptTest.py @@ -0,0 +1,33 @@ +import unittest + +from interpreter.helpers import TemplateTestCase + +import Exscript.stdlib.crypt + + +class cryptTest(TemplateTestCase): + """ + The cryptographic functions of the template language. + """ + + CORRELATE = Exscript.stdlib.crypt + + def testOtp(self): + variables = self.evaluate( + '{a = crypt.otp("password", "abc123", 9999)}' + '{b = crypt.otp("password", "abc123", "9999")}' + '{c = crypt.otp("password", "abc123", list.new(9999, 9998))}' + ) + self.assertEqual(variables['a'], ['ACTA AMMO CAR WEB BIN YAP']) + self.assertEqual(variables['b'], ['ACTA AMMO CAR WEB BIN YAP']) + self.assertEqual( + variables['c'], ['ACTA AMMO CAR WEB BIN YAP', 'LESS CLUE LISA MEAT MAGI USER'] + ) + + +def suite(): + return unittest.TestLoader().loadTestsFromTestCase(cryptTest) + + +if __name__ == '__main__': + unittest.TextTestRunner(verbosity=2).run(suite()) diff --git a/tests/Exscript/stdlib/fileTest.py b/tests/Exscript/stdlib/fileTest.py new file mode 100644 index 00000000..87b681eb --- /dev/null +++ b/tests/Exscript/stdlib/fileTest.py @@ -0,0 +1,108 @@ +import os +import stat +import tempfile +import unittest + +from interpreter.helpers import TemplateTestCase, split_error + +import Exscript.stdlib.file +from Exscript.parselib.exception import ExecuteError + + +class fileTest(TemplateTestCase): + """ + The file functions of the template language. Except for file.exists(), + they work only with secure=False. + """ + + CORRELATE = Exscript.stdlib.file + + def setUp(self): + super().setUp() + tmpdir = tempfile.TemporaryDirectory() + self.addCleanup(tmpdir.cleanup) + self.tmpdir = tmpdir.name + + def path(self, *names): + return os.path.join(self.tmpdir, *names) + + def run_insecure(self, text, **kwargs): + return self.evaluate(text, secure=False, **kwargs) + + def write(self, name, text): + with open(self.path(name), 'w', encoding='utf-8') as fp: + fp.write(text) + + def read(self, name): + with open(self.path(name), encoding='utf-8') as fp: + return fp.read() + + def testChmod(self): + self.write('a', 'a') + self.write('b', 'b') + paths = [self.path('a'), self.path('b')] + self.addCleanup(lambda: [os.chmod(path, stat.S_IWRITE | stat.S_IREAD) for path in paths]) + self.run_insecure('{file.chmod(paths, 0400)}', paths=paths) + for path in paths: + self.assertEqual(os.stat(path).st_mode & 0o222, 0) # read-only, also on Windows + + def testClear(self): + self.write('a', 'old content\n') + self.run_insecure('{file.clear(a)}{file.clear(b)}', a=self.path('a'), b=self.path('b')) + self.assertEqual((self.read('a'), self.read('b')), ('', '')) + + def testExists(self): + self.write('a', '') + text = '{x = file.exists(a)}{y = file.exists(b)}{z = file.exists(both)}' + paths = {'a': self.path('a'), 'b': self.path('b')} + variables = self.evaluate(text, both=[paths['a'], paths['b']], **paths) + self.assertEqual( + (variables['x'], variables['y'], variables['z']), ([True], [False], [True, False]) + ) + + def testMkdir(self): + dirs = [self.path('a', 'b'), self.path('c')] + self.run_insecure('{file.mkdir(dirs)}{file.mkdir(d, 0700)}', dirs=dirs, d=self.path('d')) + for path in dirs + [self.path('d')]: + self.assertTrue(os.path.isdir(path), path) + if os.name != 'nt': + self.assertEqual(os.stat(self.path('d')).st_mode & 0o777, 0o700) + + def testRead(self): + self.write('a', 'line1\nline2\n') + variables = self.run_insecure('{x = file.read(a)}', a=self.path('a')) + self.assertEqual([line.rstrip('\n') for line in variables['x']], ['line1', 'line2']) + self.assertEqual(variables['__response__'], variables['x']) + + def testReadAMissingFile(self): + with self.assertRaises(ExecuteError) as cm: + self.run_insecure('{x = file.read(a)}', a=self.path('missing')) + self.assertIn( + "FileNotFoundError while executing function 'file.read'", split_error(cm.exception)[2] + ) + + def testRm(self): + for name in 'abc': + self.write(name, name) + self.run_insecure( + '{file.rm(a)}{file.rm(bc)}', a=self.path('a'), bc=[self.path('b'), self.path('c')] + ) + self.assertEqual(os.listdir(self.tmpdir), []) + + def testWrite(self): + path = self.path('a') + self.run_insecure( + '{file.write(path, "first")}{file.write(path, list.new("second ", "third"))}', + path=path, + ) + self.assertEqual(self.read('a'), 'first\nsecond\nthird\n') + self.run_insecure('{file.write(path, lines, "w")}', path=path, lines=['new', 'lines']) + self.assertEqual(self.read('a'), 'new\nlines\n') + + +def suite(): + return unittest.TestLoader().loadTestsFromTestCase(fileTest) + + +if __name__ == '__main__': + unittest.TextTestRunner(verbosity=2).run(suite()) diff --git a/tests/Exscript/stdlib/functionsTest.py b/tests/Exscript/stdlib/functionsTest.py new file mode 100644 index 00000000..e293a7b5 --- /dev/null +++ b/tests/Exscript/stdlib/functionsTest.py @@ -0,0 +1,52 @@ +import unittest + +from Exscript import stdlib +from Exscript.stdlib import connection, crypt, file, ipv4, mysys, string +from Exscript.stdlib import list as list_functions + +# The functions that run local commands or access local files. +INSECURE = { + 'file.chmod', + 'file.clear', + 'file.mkdir', + 'file.read', + 'file.rm', + 'file.write', + 'sys.env', + 'sys.exec', +} + + +class functionsTest(unittest.TestCase): + """ + The table of the functions of the template language. + """ + + def testNames(self): + modules = { + 'connection': connection, + 'crypt': crypt, + 'file': file, + 'ipv4': ipv4, + 'list': list_functions, + 'string': string, + 'sys': mysys, + } + renamed = {'connection.exec': 'exec_', 'sys.exec': 'execute'} + for name, function in stdlib.functions.items(): + with self.subTest(name=name): + prefix, attribute = name.split('.') + attribute = renamed.get(name, attribute) + self.assertIs(function, getattr(modules[prefix], attribute)) + + def testInsecureFunctions(self): + insecure = {name for name, f in stdlib.functions.items() if not hasattr(f, '_is_secure')} + self.assertEqual(insecure, INSECURE) + + +def suite(): + return unittest.TestLoader().loadTestsFromTestCase(functionsTest) + + +if __name__ == '__main__': + unittest.TextTestRunner(verbosity=2).run(suite()) diff --git a/tests/Exscript/stdlib/ipv4Test.py b/tests/Exscript/stdlib/ipv4Test.py new file mode 100644 index 00000000..90424878 --- /dev/null +++ b/tests/Exscript/stdlib/ipv4Test.py @@ -0,0 +1,81 @@ +import unittest + +from interpreter.helpers import TemplateTestCase, split_error + +import Exscript.stdlib.ipv4 +from Exscript.parselib.exception import ExecuteError + + +class ipv4Test(TemplateTestCase): + """ + The IPv4 functions of the template language. + """ + + CORRELATE = Exscript.stdlib.ipv4 + + def assertResults(self, *calls): + for call, expected in calls: + with self.subTest(call=call): + self.assertEqual(self.evaluate('{x = %s}' % call)['x'], expected) + + def testInNetwork(self): + self.assertResults( + ('ipv4.in_network("10.0.0.0/8", "10.1.2.3")', [True]), + ('ipv4.in_network("10.0.0.0/8", "11.0.0.1")', [False]), + ('ipv4.in_network(list.new("10.0.0.0/8", "192.168.0.0/16"), "192.168.5.5")', [True]), + # Without a prefix length, /24 applies, or the given default. + ('ipv4.in_network("192.168.1.0", "192.168.1.200")', [True]), + ('ipv4.in_network("192.168.1.0", "192.168.2.1")', [False]), + ('ipv4.in_network("192.168.0.0", "192.168.2.1", 16)', [True]), + ) + + def testMask(self): + self.assertResults(('ipv4.mask("192.168.1.77", "255.255.255.0")', ['192.168.1.0'])) + variables = self.evaluate( + '{x = ipv4.mask(ips, "255.255.0.0")}', ips=['10.1.2.3', '10.2.3.4'] + ) + self.assertEqual(variables['x'], ['10.1.0.0', '10.2.0.0']) + + def testMask2Pfxlen(self): + variables = self.evaluate( + '{x = ipv4.mask2pfxlen(masks)}', + masks=['255.255.255.0', '255.255.0.0', '255.255.255.252'], + ) + self.assertEqual(variables['x'], [24, 16, 30]) + + def testPfxlen2Mask(self): + self.assertResults( + ('ipv4.pfxlen2mask(30)', ['255.255.255.252']), + ('ipv4.pfxlen2mask("16")', ['255.255.0.0']), + ) + + def testNetwork(self): + self.assertResults(('ipv4.network("10.1.2.3/8")', ['10.0.0.0'])) + + def testBroadcast(self): + self.assertResults(('ipv4.broadcast("10.1.2.3/8")', ['10.255.255.255'])) + + def testPfxmask(self): + variables = self.evaluate('{x = ipv4.pfxmask(ips, 16)}', ips=['10.1.2.3', '192.168.7.1']) + self.assertEqual(variables['x'], ['10.1.0.0', '192.168.0.0']) + + def testRemoteIp(self): + variables = self.evaluate( + '{x = ipv4.remote_ip(ips)}', ips=['10.0.0.1', '10.0.0.2', '10.0.0.6'] + ) + self.assertEqual(variables['x'], ['10.0.0.2', '10.0.0.1', '10.0.0.5']) + + def testInvalidAddress(self): + with self.assertRaises(ExecuteError) as cm: + self.evaluate('{x = ipv4.mask2pfxlen("abc")}') + lines, marker, message = split_error(cm.exception) + self.assertEqual((lines, marker), (['{x = ipv4.mask2pfxlen("abc")}'], ' ' + '^' * 16)) + self.assertIn(" while executing function 'ipv4.mask2pfxlen': ", message) + + +def suite(): + return unittest.TestLoader().loadTestsFromTestCase(ipv4Test) + + +if __name__ == '__main__': + unittest.TextTestRunner(verbosity=2).run(suite()) diff --git a/tests/Exscript/stdlib/listTest.py b/tests/Exscript/stdlib/listTest.py new file mode 100644 index 00000000..4ac521a7 --- /dev/null +++ b/tests/Exscript/stdlib/listTest.py @@ -0,0 +1,63 @@ +import unittest + +from interpreter.helpers import TemplateTestCase, split_error + +import Exscript.stdlib.list +from Exscript.parselib.exception import ExecuteError + + +class listTest(TemplateTestCase): + """ + The list functions of the template language. + """ + + CORRELATE = Exscript.stdlib.list + + def testNew(self): + variables = self.evaluate('{a = list.new()}{b = list.new("x", 1, list.new("y", "z"))}') + self.assertEqual(variables['a'], []) + self.assertEqual(variables['b'], ['x', 1, 'y']) + + def testLength(self): + variables = self.evaluate( + '{a = list.length(l)}{b = list.length(list.new())}{c = list.length("x")}', l=['x', 'y'] + ) + self.assertEqual((variables['a'], variables['b'], variables['c']), ([2], [0], [1])) + + def testGet(self): + variables = self.evaluate('{a = list.get(l, 1)}{b = list.get(l, "0")}', l=['x', 'y']) + self.assertEqual((variables['a'], variables['b']), (['y'], ['x'])) + + def testGetErrors(self): + for argument, error in ( + ('"x"', 'index is not an integer'), + ('2', 'no such item in the list'), + ('list.new()', 'index variable is required'), + ): + with self.subTest(argument=argument): + with self.assertRaises(ExecuteError) as cm: + self.evaluate('{a = list.get(l, %s)}' % argument, l=['x', 'y']) + message = ( + "ValueError while executing function 'list.get': %s in undefined:1" % error + ) + self.assertEqual(split_error(cm.exception)[2], message) + + def testSort(self): + variables = self.evaluate('{a = list.sort(l)}', l=['b', 'c', 'a']) + self.assertEqual((variables['a'], variables['l']), (['a', 'b', 'c'], ['b', 'c', 'a'])) + + def testReverse(self): + variables = self.evaluate('{a = list.reverse(l)}', l=['a', 'b', 'c']) + self.assertEqual((variables['a'], variables['l']), (['c', 'b', 'a'], ['a', 'b', 'c'])) + + def testUnique(self): + variables = self.evaluate('{a = list.unique(l)}', l=['b', 'a', 'b', 'c', 'a']) + self.assertEqual(variables['a'], ['b', 'a', 'c']) + + +def suite(): + return unittest.TestLoader().loadTestsFromTestCase(listTest) + + +if __name__ == '__main__': + unittest.TextTestRunner(verbosity=2).run(suite()) diff --git a/tests/Exscript/stdlib/mysysTest.py b/tests/Exscript/stdlib/mysysTest.py new file mode 100644 index 00000000..652f2967 --- /dev/null +++ b/tests/Exscript/stdlib/mysysTest.py @@ -0,0 +1,71 @@ +import contextlib +import io +import os +import shlex +import subprocess +import sys +import unittest +from unittest import mock + +from interpreter.helpers import TemplateTestCase + +import Exscript.stdlib.mysys + + +def command_line(*args): + """ + Returns a command line that runs Python with the given arguments, quoted + for sys.exec() on this platform. + """ + if os.name == 'nt': + return subprocess.list2cmdline([sys.executable, *args]) + return shlex.join([sys.executable, *args]) + + +class mysysTest(TemplateTestCase): + """ + The system functions of the template language, sys.*. + """ + + CORRELATE = Exscript.stdlib.mysys + + def testEnv(self): + with mock.patch.dict(os.environ, {'EXSCRIPT_TEST_VALUE': 'a value'}): + os.environ.pop('EXSCRIPT_TEST_UNSET', None) + text = '{a = sys.env("EXSCRIPT_TEST_VALUE")}{b = sys.env("EXSCRIPT_TEST_UNSET")}' + variables = self.evaluate(text, secure=False) + self.assertEqual((variables['a'], variables['b']), (['a value'], [''])) + + def testExecute(self): + code = 'import sys; print("out", flush=True); print("err", file=sys.stderr, flush=True)' + variables = self.evaluate( + '{sys.exec(command)}', command=command_line('-c', code), secure=False + ) + self.assertEqual(variables['__response__'], ['out\n', 'err\n']) + self.assertSent() + + def testExecuteWithoutInput(self): + code = 'import sys; print(repr(sys.stdin.read()))' + variables = self.evaluate( + '{sys.exec(command)}', command=command_line('-c', code), secure=False + ) + self.assertEqual(variables['__response__'], ["''\n"]) + + def testMessage(self): + out = io.StringIO() + with contextlib.redirect_stdout(out): + self.evaluate('{sys.message("hello $__hostname__")}{sys.message(list.new("a", "b"))}') + self.assertEqual(out.getvalue(), 'hello r1\nab\n') + + def testWait(self): + with mock.patch.object(Exscript.stdlib.mysys.time, 'sleep') as sleep: + self.evaluate('{sys.wait(2)}{sys.wait("3")}') + self.assertEqual(sleep.call_args_list, [mock.call(2), mock.call(3)]) + + +def suite(): + return unittest.TestLoader().loadTestsFromTestCase(mysysTest) + + +if __name__ == '__main__': + unittest.TextTestRunner(verbosity=2).run(suite()) diff --git a/tests/Exscript/stdlib/stringTest.py b/tests/Exscript/stdlib/stringTest.py new file mode 100644 index 00000000..bc6d86d7 --- /dev/null +++ b/tests/Exscript/stdlib/stringTest.py @@ -0,0 +1,57 @@ +import unittest + +from interpreter.helpers import TemplateTestCase + +import Exscript.stdlib.string + + +class stringTest(TemplateTestCase): + """ + The string functions of the template language. + """ + + CORRELATE = Exscript.stdlib.string + + def testReplace(self): + variables = self.evaluate( + '{a = string.replace("ssuccessss", "ss", "s")}{b = string.replace(l, "-", "+")}', + l=['a-b', 'c-d-e'], + ) + self.assertEqual((variables['a'], variables['b']), (['success'], ['a+b', 'c+d+e'])) + + def testSplit(self): + variables = self.evaluate( + '{a = string.split("10,234,65,8", ",")}{b = string.split(l, ",")}', l=['a,b', 'c'] + ) + self.assertEqual( + (variables['a'], variables['b']), (['10', '234', '65', '8'], ['a', 'b', 'c']) + ) + + def testStrip(self): + variables = self.evaluate( + '{a = string.strip(" \\t a b \\n")}{b = string.strip("xxaxx", "x")}{c = string.strip(l)}', + l=[' a ', 'b '], + ) + self.assertEqual( + (variables['a'], variables['b'], variables['c']), (['a b'], ['a'], ['a', 'b']) + ) + + def testTolower(self): + variables = self.evaluate( + '{a = string.tolower("Low3R")}{b = string.tolower(l)}', l=['A', 'Bc'] + ) + self.assertEqual((variables['a'], variables['b']), (['low3r'], ['a', 'bc'])) + + def testToupper(self): + variables = self.evaluate( + '{a = string.toupper("Upp3r")}{b = string.toupper(l)}', l=['a', 'bC'] + ) + self.assertEqual((variables['a'], variables['b']), (['UPP3R'], ['A', 'BC'])) + + +def suite(): + return unittest.TestLoader().loadTestsFromTestCase(stringTest) + + +if __name__ == '__main__': + unittest.TextTestRunner(verbosity=2).run(suite()) diff --git a/tests/Exscript/stdlib/utilTest.py b/tests/Exscript/stdlib/utilTest.py new file mode 100644 index 00000000..a9fb9519 --- /dev/null +++ b/tests/Exscript/stdlib/utilTest.py @@ -0,0 +1,33 @@ +import unittest + +import Exscript.stdlib.util +from Exscript.stdlib.util import secure_function + + +def concat(scope, *args, separator=''): + """Joins the arguments.""" + return [separator.join(args)] + + +class utilTest(unittest.TestCase): + """ + The decorator that marks functions of the template language as secure. + """ + + CORRELATE = Exscript.stdlib.util + + def testSecureFunction(self): + wrapper = secure_function(concat) + self.assertTrue(wrapper._is_secure) + self.assertFalse(hasattr(concat, '_is_secure')) + self.assertEqual(wrapper('scope', 'a', 'b', separator='-'), ['a-b']) + self.assertEqual((wrapper.__name__, wrapper.__doc__), ('concat', 'Joins the arguments.')) + self.assertIs(wrapper.__wrapped__, concat) + + +def suite(): + return unittest.TestLoader().loadTestsFromTestCase(utilTest) + + +if __name__ == '__main__': + unittest.TextTestRunner(verbosity=2).run(suite()) From 2a4bd64a1d6ae0dedf415e4cfd9502f0fac8c1fb Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 10 Nov 2026 16:45:30 +0100 Subject: [PATCH 085/250] Close the SMTP session of util.mail.send() send() never ended the session, so each mail left a connection open until the garbage collector closed it. Build the message first, then send it in a with block, which sends QUIT and closes the connection, also on errors. --- CHANGES.md | 3 +++ src/Exscript/util/mail.py | 4 ++-- tests/Exscript/util/mailTest.py | 20 +++++++++++++++++++- 3 files changed, 24 insertions(+), 3 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 3749b620..4ade5010 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -386,6 +386,9 @@ The names below refer to functions callable from templates, not protocol methods exist and asks again, as documented. It passed its check and history one position too early to `prompt()`, so it never checked the name, and failed with a `TypeError` when a history was given. +- **`Exscript.util.mail.send()`:** Ends the SMTP session with `QUIT` and closes the + connection, also if sending fails; before, it left the connection open. It connects + only after the mail and its attachments are ready. - **Pseudo Protocol on Windows:** `pseudo://` URLs accept absolute Windows paths such as `pseudo://C:\devices\router.py`; the drive letter was taken for a port number. - **`Exscript.util.tty.get_terminal_size()`:** Uses `shutil.get_terminal_size()`: the diff --git a/src/Exscript/util/mail.py b/src/Exscript/util/mail.py index 8923009e..3c8822ba 100644 --- a/src/Exscript/util/mail.py +++ b/src/Exscript/util/mail.py @@ -529,7 +529,6 @@ def send(mail: Mail, server: str = 'localhost') -> None: """ sender = mail.get_sender() rcpt = mail.get_recipients() - session = smtplib.SMTP(server) message = MIMEMultipart() message['Subject'] = mail.get_subject() message['From'] = mail.get_sender() @@ -546,4 +545,5 @@ def send(mail: Mail, server: str = 'localhost') -> None: for filename in mail.get_attachments(): message.attach(_get_mime_object(filename)) - session.sendmail(sender, rcpt, message.as_string()) + with smtplib.SMTP(server) as session: # sends QUIT and closes the connection + session.sendmail(sender, rcpt, message.as_string()) diff --git a/tests/Exscript/util/mailTest.py b/tests/Exscript/util/mailTest.py index 81f5668f..f7fec11b 100644 --- a/tests/Exscript/util/mailTest.py +++ b/tests/Exscript/util/mailTest.py @@ -1,7 +1,9 @@ +import email import os import tempfile import unittest from getpass import getuser +from unittest import mock import Exscript.util.mail from Exscript.util.mail import Mail @@ -174,7 +176,23 @@ def testFromTemplate(self): self.checkResult(mail) def testSend(self): - pass # no easy way to test without spamming. + mail = Mail(sender='me@example.com', to='you@example.com', subject='Report', body='Grüße') + with mock.patch('smtplib.SMTP') as smtp: + Exscript.util.mail.send(mail, 'mail.example.com') + smtp.assert_called_once_with('mail.example.com') + sender, rcpt, text = smtp.return_value.__enter__.return_value.sendmail.call_args.args + self.assertEqual((sender, rcpt), ('me@example.com', ['you@example.com'])) + message = email.message_from_string(text) + self.assertEqual(message['Subject'], 'Report') + self.assertEqual(message.get_payload()[0].get_payload(decode=True), 'Grüße'.encode()) + smtp.return_value.__exit__.assert_called_once() # QUIT, and the connection is closed + + def testSendDoesNotConnectIfTheMailIsIncomplete(self): + mail = Mail(to='you@example.com') + mail.add_attachment(os.path.join(tempfile.gettempdir(), 'missing-attachment')) + with mock.patch('smtplib.SMTP') as smtp: + self.assertRaises(OSError, Exscript.util.mail.send, mail) + smtp.assert_not_called() def suite(): From d46925fad155d45038e9448a97a17a2dca089952 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 10 Nov 2026 16:47:21 +0100 Subject: [PATCH 086/250] Document the authorization password in host addresses Host addresses, also in the files of --hosts and --csv-hosts, accept a second password for authorization, e.g. the enable password of Cisco devices (upstream issue #186). Warn that passwords in a command line are visible in the process list and the shell history. --- docs/cli_tutorial.rst | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/docs/cli_tutorial.rst b/docs/cli_tutorial.rst index 90611789..b100fe58 100644 --- a/docs/cli_tutorial.rst +++ b/docs/cli_tutorial.rst @@ -65,13 +65,22 @@ Reading login information Depending on how you would like to provide the login information, there are a few options. The first is by including it in the hostname:: - exscript -c 2 test.exscript ssh://localhost - ssh://user:password@otherhost + exscript -c 2 test.exscript ssh://localhost ssh://user:password@otherhost In this case, Exscript still prompts the user for his login details, but the entered information is only used on hosts that do not have a user/password combination included in the hostname. +A second password after the first one, as in +``ssh://user:password:password2@otherhost``, is the authorization +password, e.g. the enable password of a Cisco device: the password that +``--authorization`` asks for. The host files of ``--hosts`` and +``--csv-hosts`` accept the same addresses. + +.. warning:: + Passwords in a command line can be read from the process list and the + shell history. Prefer the prompt or an account pool (see below). + If you do not want for Exscript to prompt for login details, the ``-i`` switch tells Exscript to not ask for a user and password. You need to make sure that all hosts have a user and password in the @@ -84,7 +93,7 @@ If you do not wish to hard code the host names or login details into the command, you may also list the hosts in an external file and load it using the ``--hosts`` option as follows:: - exscript -c 2 —hosts myhosts.txt test.exscript + exscript -c 2 --hosts myhosts.txt test.exscript Note that *hosts.txt* is a file containing a list of hostnames, e.g.:: @@ -124,7 +133,7 @@ user/password combination specified in their URL. :: - exscript -c 2 —hosts myhosts.txt —account-pool accounts.cfg test.exscript + exscript -c 2 --hosts myhosts.txt --account-pool accounts.cfg test.exscript Note that ``accounts.cfg`` is a config file with a defined syntax as seen in the API documentation for @@ -139,7 +148,7 @@ Logging Exscript has built-in support for logging - just pass the ``--logdir`` or ``-l`` option with a path to the directory in which logs are stored:: - exscript -l /tmp/logs -c 2 —hosts myhosts.txt —account-pool accounts.cfg test.exscript + exscript -l /tmp/logs -c 2 --hosts myhosts.txt --account-pool accounts.cfg test.exscript Exscript creates one logfile per device. In the case that an error happened on the remote device, it creates an additional file that From 45ccfa4031f0a27ebaaea1a6e2c936f0b51514da Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 10 Nov 2026 16:48:38 +0100 Subject: [PATCH 087/250] Answer frequent questions in the documentation Add a FAQ page for questions that upstream issues ask repeatedly: why send() returns nothing (#226, #234), how to answer a question of the host such as the sudo password (#216), and what replaces the telnetlib module that Python 3.13 removed (#242). --- docs/faq.rst | 54 ++++++++++++++++++++++++++++++++++++++++++++++++++ docs/index.rst | 1 + 2 files changed, 55 insertions(+) create mode 100644 docs/faq.rst diff --git a/docs/faq.rst b/docs/faq.rst new file mode 100644 index 00000000..250e7ab7 --- /dev/null +++ b/docs/faq.rst @@ -0,0 +1,54 @@ +Frequently Asked Questions +========================== + +Why does ``send()`` return nothing? +----------------------------------- + +``send()`` only sends the data; it does not wait for an answer. To run a +command and read its output, use ``execute()``, which sends the command, +waits for the prompt and keeps the output in ``conn.response``:: + + conn.execute('show version') + print(conn.response) + +To wait for something other than the prompt, follow ``send()`` with +``expect()``, which waits for one or more regular expressions:: + + conn.send('reload\r') + conn.expect(r'\[confirm\]') + conn.send('\r') + +How do I answer a question of the host, such as the password of ``sudo``? +-------------------------------------------------------------------------- + +Send the command, wait for the question with ``expect()``, answer it, and +wait for the prompt:: + + conn.send('sudo -k systemctl restart sshd\r') + conn.expect(r'[Pp]assword.*:\s*$') + conn.send(account.get_password() + '\r') + conn.expect_prompt() + +``sudo -k`` asks for the password even if sudo remembers an earlier one, +so the question always comes. If the host shows no prompt that Exscript +recognizes, set one with ``set_prompt()`` (see :doc:`trouble`). + +Python removed ``telnetlib``. What do I use instead? +---------------------------------------------------- + +:class:`Exscript.protocols.Telnet` connects to any Telnet host and has the +same functions as the other protocol adapters, e.g. ``expect()``, +``waitfor()`` and ``execute()``:: + + from Exscript import Account + from Exscript.protocols import Telnet + + conn = Telnet() + conn.connect('router1') + conn.login(Account('admin', 'secret')) + conn.execute('show version') + conn.close() + +``Exscript.protocols.telnetlib`` is internal. For a drop-in replacement of +the module that Python 3.13 removed, use the package +`standard-telnetlib `_. diff --git a/docs/index.rst b/docs/index.rst index edd0fbfb..10b75a86 100755 --- a/docs/index.rst +++ b/docs/index.rst @@ -83,6 +83,7 @@ Contents security platforms trouble + faq migration api changelog From 967183ce54a18ac5bde1c900c60a764df94f2c34 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 10 Nov 2026 17:08:02 +0100 Subject: [PATCH 088/250] Mark the package as typed All modules apart from the queue and util.sigint pass mypy --strict, so ship the py.typed marker (PEP 561). Also raise the docstring floor of interrogate to 93 percent. --- CHANGES.md | 4 ++++ pyproject.toml | 2 +- src/Exscript/py.typed | 0 3 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 src/Exscript/py.typed diff --git a/CHANGES.md b/CHANGES.md index 4ade5010..1d403ecf 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -18,6 +18,10 @@ marked with **BREAKING**; a migration guide will accompany the release. - **BREAKING - Python 3.12 or Later:** Exscript runs on Python 3.12 to 3.15 and on the free-threaded build of 3.14, on Linux, macOS and Windows. Python 2.7 and Python 3 versions before 3.12 are no longer supported. +- **Type Annotations:** Exscript is annotated and ships the `py.typed` marker (PEP 561), + so type checkers such as mypy and pyright check the code that uses it. All modules + pass `mypy --strict`, except the queue (`Exscript.queue`, `Exscript.workqueue`) and + `Exscript.util.sigint`, which are annotated only in part. - **Dependencies:** `future` and `configparser`, backports for Python 2, are no longer needed. paramiko 3.4 to 5.x is supported (before: 1.17 to 4.x). New: ncclient and lxml 5.0 or later, for NETCONF. diff --git a/pyproject.toml b/pyproject.toml index 41927c24..9bb33656 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -161,7 +161,7 @@ disallow_subclassing_any = false # Docstring coverage (P8). Private functions count; other dunder methods than # __init__, nested functions and property setters do not. Raise fail-under as # docstrings are added; the goal is 100. -fail-under = 87 +fail-under = 93 ignore-magic = true ignore-nested-functions = true ignore-setters = true diff --git a/src/Exscript/py.typed b/src/Exscript/py.typed new file mode 100644 index 00000000..e69de29b From 5fe79ce8d01aaee458df6bdd55f3a6615276ce5e Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 10 Nov 2026 17:32:27 +0100 Subject: [PATCH 089/250] Test the sessions of the SSH and Telnet test servers --- tests/Exscript/servers/SSHdTest.py | 167 +++++++++++++++++++++++++- tests/Exscript/servers/ServerTest.py | 56 ++++++++- tests/Exscript/servers/TelnetdTest.py | 61 +++++++++- 3 files changed, 280 insertions(+), 4 deletions(-) diff --git a/tests/Exscript/servers/SSHdTest.py b/tests/Exscript/servers/SSHdTest.py index 3248c64c..acb5e111 100644 --- a/tests/Exscript/servers/SSHdTest.py +++ b/tests/Exscript/servers/SSHdTest.py @@ -1,10 +1,24 @@ +import logging +import os +import shutil +import socket +import tempfile +import threading import unittest +from unittest import mock +import paramiko + +from Exscript import Account, PrivateKey +from Exscript.emulators import VirtualDevice from Exscript.protocols import SSH2 -from Exscript.servers import SSHd +from Exscript.protocols.exception import ConnectionClosedException +from Exscript.servers import SSHd, sshd from .ServerTest import ServerTest +KEYFILE = os.path.join(os.path.dirname(__file__), os.pardir, 'protocols', 'id_rsa') + class SSHdTest(ServerTest): CORRELATE = SSHd @@ -16,8 +30,157 @@ def _create_client(self): return SSH2(verify_fingerprint=False) +class _ImpatientServer(sshd._ParamikoServer): + """Waits 0.1 seconds instead of 10 for the shell request.""" + + def __init__(self): + super().__init__() + event = self.event + + class Event: + set, is_set = event.set, event.is_set + + def wait(self, timeout=None): + return event.wait(0.1) + + self.event = Event() + + +class SSHdSessionTest(unittest.TestCase): + """How the server handles its clients.""" + + def setUp(self): + self.device = VirtualDevice('r1', echo=False, login_type=VirtualDevice.LOGIN_TYPE_NONE) + self.daemon = SSHd('127.0.0.1', 0, self.device, key=PrivateKey.from_file(KEYFILE)) + self.daemon.dbg = 1 + self.daemon.timeout = 0.05 # stops faster + + def start(self): + self.daemon.start() + self.addCleanup(self.daemon.join) + self.addCleanup(self.daemon.exit) + + def connect(self): + conn = SSH2(timeout=2, connect_rate_limit=0, verify_fingerprint=False) + self.addCleanup(conn.close, force=True) + conn.connect('127.0.0.1', self.daemon.port) + conn.protocol_authenticate(Account('user', 'password', needs_lock=False)) + conn.expect_prompt() + return conn + + def testDefaultKey(self): + home = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, home) + os.mkdir(os.path.join(home, '.ssh')) + shutil.copy(KEYFILE, os.path.join(home, '.ssh', 'id_rsa')) + with mock.patch.dict(os.environ, {'HOME': home, 'USERPROFILE': home}): + daemon = SSHd('127.0.0.1', 0, self.device) + self.assertEqual(daemon.host_key, paramiko.RSAKey(filename=KEYFILE)) + + def testOnlySessionChannels(self): + server = sshd._ParamikoServer() + self.assertEqual(server.check_channel_request('session', 1), paramiko.OPEN_SUCCEEDED) + self.assertEqual( + server.check_channel_request('direct-tcpip', 2), + paramiko.OPEN_FAILED_ADMINISTRATIVELY_PROHIBITED, + ) + + def testAnyKeyLogsIn(self): + home = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, home) + filename = os.path.join(home, 'id_rsa') + paramiko.RSAKey.generate(1024).write_private_key_file(filename) + self.device.add_command('ls', 'file') + self.start() + conn = SSH2(timeout=2, connect_rate_limit=0, verify_fingerprint=False) + self.addCleanup(conn.close, force=True) + conn.connect('127.0.0.1', self.daemon.port) + key = PrivateKey.from_file(filename) + conn.protocol_authenticate(Account('user', key=key, needs_lock=False)) + conn.expect_prompt() + conn.execute('ls') + self.assertEqual(conn.response, 'file') + + def testCommandWithoutOutput(self): + self.device.add_command('silent', '', prompt=False) + self.device.add_command('ls', 'file') + self.start() + conn = self.connect() + conn.send('silent\r') + conn.execute('ls') + self.assertEqual(conn.response, 'file') + + def testServerStopsDuringTheSession(self): + def stop(command): + self.daemon.running = False + return 'bye' + + self.device.add_command('stop', stop, prompt=False) + self.start() + conn = self.connect() + conn.send('stop\r') + conn.expect('bye') + self.assertRaises(ConnectionClosedException, conn.expect_prompt) + + def testCommandHandlerError(self): + def fail(command): + raise RuntimeError('handler failed') + + self.device.add_command('fail', fail) + self.start() + conn = self.connect() + with self.assertLogs('Exscript.servers.sshd', logging.DEBUG) as logs: + self.assertRaises(ConnectionClosedException, conn.execute, 'fail') + self.assertIn('RuntimeError in connection handler: handler failed', '\n'.join(logs.output)) + + def testClientWithoutShell(self): + server_sock, client_sock = socket.socketpair() + self.daemon.running = True + handler = threading.Thread(target=self.daemon._handle_connection, args=(server_sock,)) + client = paramiko.Transport(client_sock) + self.addCleanup(client.close) + with ( + mock.patch.object(sshd, '_ParamikoServer', _ImpatientServer), + self.assertLogs('Exscript.servers.sshd', logging.DEBUG) as logs, + ): + handler.start() + client.start_client(timeout=5) + client.auth_password('user', 'password') + channel = client.open_session(timeout=5) + channel.settimeout(5) + self.assertEqual(channel.recv(1), b'') # the server closed the connection + handler.join(5) + self.assertFalse(handler.is_alive()) + self.assertIn('Client never asked for a shell.', '\n'.join(logs.output)) + + def testClientDisappearsDuringTheHandshake(self): + server_sock, client_sock = socket.socketpair() + client_sock.close() + with self.assertLogs('Exscript.servers.sshd', logging.DEBUG) as logs: + self.daemon._handle_connection(server_sock) + self.assertIn('Client disappeared before establishing connection', '\n'.join(logs.output)) + self.assertEqual(server_sock.fileno(), -1) # closed + + def testClientDisappearsBeforeOpeningAChannel(self): + server_sock, client_sock = socket.socketpair() + handler = threading.Thread(target=self.daemon._handle_connection, args=(server_sock,)) + client = paramiko.Transport(client_sock) + self.addCleanup(client.close) + with self.assertLogs('Exscript.servers.sshd', logging.DEBUG) as logs: + handler.start() + client.start_client(timeout=5) + client.auth_password('user', 'password') + client.close() + handler.join(5) + self.assertFalse(handler.is_alive()) + self.assertIn('Client disappeared before requesting channel.', '\n'.join(logs.output)) + + def suite(): - return unittest.TestLoader().loadTestsFromTestCase(SSHdTest) + loader = unittest.TestLoader() + return unittest.TestSuite( + [loader.loadTestsFromTestCase(SSHdTest), loader.loadTestsFromTestCase(SSHdSessionTest)] + ) if __name__ == '__main__': diff --git a/tests/Exscript/servers/ServerTest.py b/tests/Exscript/servers/ServerTest.py index 483b3b72..32d07f4e 100644 --- a/tests/Exscript/servers/ServerTest.py +++ b/tests/Exscript/servers/ServerTest.py @@ -1,5 +1,9 @@ +import gc import re +import socket import unittest +import unittest.mock +import warnings from Exscript import Account from Exscript.emulators import VirtualDevice @@ -74,8 +78,58 @@ def testExit(self): self.testConstructor() +class ServerInternalsTest(unittest.TestCase): + """The parts of the base class that need no client.""" + + def setUp(self): + self.server = Server('127.0.0.1', 0, VirtualDevice('r1')) + self.addCleanup(self.server.to_parent.close) + self.addCleanup(self.server.to_child.close) + + def testPollChildProcess(self): + self.server.running = True + self.assertFalse(self.server._poll_child_process()) # no message + self.server.to_child.send('ping') + self.assertTrue(self.server._poll_child_process()) + self.assertTrue(self.server.running) + self.server.to_child.send('shutdown') + self.assertFalse(self.server._poll_child_process()) + self.assertFalse(self.server.running) + self.server.to_child.send('ping') + self.assertFalse(self.server._poll_child_process()) # not running + + def testPollChildProcessWithBrokenPipe(self): + self.server.running = True + self.server.to_child.send('ping') + with unittest.mock.patch.object(self.server.to_parent, 'recv', side_effect=OSError): + self.assertFalse(self.server._poll_child_process()) + self.assertFalse(self.server.running) + + def testHandleConnectionIsAbstract(self): + self.assertRaises(NotImplementedError, self.server._handle_connection, None) + + def testStartOnAPortInUse(self): + with socket.create_server(('127.0.0.1', 0)) as listener: + server = Server('127.0.0.1', listener.getsockname()[1], self.server.device) + self.addCleanup(server.to_parent.close) + self.addCleanup(server.to_child.close) + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter('always', ResourceWarning) + self.assertRaises(OSError, server.start) + gc.collect() + self.assertEqual([str(w.message) for w in caught], []) # the socket was closed + self.assertIsNone(server.socket) + self.assertFalse(server.is_alive()) + + def suite(): - return unittest.TestLoader().loadTestsFromTestCase(ServerTest) + loader = unittest.TestLoader() + return unittest.TestSuite( + [ + loader.loadTestsFromTestCase(ServerTest), + loader.loadTestsFromTestCase(ServerInternalsTest), + ] + ) if __name__ == '__main__': diff --git a/tests/Exscript/servers/TelnetdTest.py b/tests/Exscript/servers/TelnetdTest.py index 9becb8ee..088a42a9 100644 --- a/tests/Exscript/servers/TelnetdTest.py +++ b/tests/Exscript/servers/TelnetdTest.py @@ -1,6 +1,11 @@ +import logging +import socket import unittest +from Exscript import Account +from Exscript.emulators import VirtualDevice from Exscript.protocols import Telnet +from Exscript.protocols.exception import ConnectionClosedException from Exscript.servers import Telnetd from .ServerTest import ServerTest @@ -16,8 +21,62 @@ def _create_client(self): return Telnet() +class TelnetdSessionTest(unittest.TestCase): + """How the server handles its clients.""" + + def setUp(self): + self.device = VirtualDevice('r1', echo=False, login_type=VirtualDevice.LOGIN_TYPE_NONE) + self.daemon = Telnetd('127.0.0.1', 0, self.device) + self.daemon.dbg = 1 + self.daemon.timeout = 0.05 # stops faster + + def connect(self): + self.daemon.start() + self.addCleanup(self.daemon.join) + self.addCleanup(self.daemon.exit) + conn = Telnet(timeout=2, connect_rate_limit=0) + self.addCleanup(conn.close, force=True) + conn.connect('127.0.0.1', self.daemon.port) + conn.protocol_authenticate(Account('user', 'password', needs_lock=False)) + conn.expect_prompt() + return conn + + def testCommandWithoutOutput(self): + self.device.add_command('silent', '', prompt=False) + self.device.add_command('ls', 'file') + conn = self.connect() + conn.send('silent\r') + conn.execute('ls') + self.assertEqual(conn.response, 'file') + + def testCommandHandlerError(self): + def fail(command): + raise RuntimeError('handler failed') + + self.device.add_command('fail', fail) + conn = self.connect() + with self.assertLogs('Exscript.servers.telnetd', logging.DEBUG) as logs: + self.assertRaises(ConnectionClosedException, conn.execute, 'fail') + self.assertIn('RuntimeError in connection handler: handler failed', '\n'.join(logs.output)) + + def testClientDisappears(self): + server_sock, client_sock = socket.socketpair() + client_sock.close() + self.daemon.running = True + with self.assertLogs('Exscript.servers.telnetd', logging.DEBUG) as logs: + self.daemon._handle_connection(server_sock) # fails to send the banner + self.assertIn('Client disappeared: ', '\n'.join(logs.output)) + self.assertEqual(server_sock.fileno(), -1) # closed + + def suite(): - return unittest.TestLoader().loadTestsFromTestCase(TelnetdTest) + loader = unittest.TestLoader() + return unittest.TestSuite( + [ + loader.loadTestsFromTestCase(TelnetdTest), + loader.loadTestsFromTestCase(TelnetdSessionTest), + ] + ) if __name__ == '__main__': From c1ea4c2e69e14fbd83935bf4cf11bf40a02912b4 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 10 Nov 2026 17:32:27 +0100 Subject: [PATCH 090/250] Test more of interact, start, impl and collections in Exscript.util --- tests/Exscript/util/collectionsTest.py | 55 ++++++++++ tests/Exscript/util/implTest.py | 136 +++++++++++++++++++++++ tests/Exscript/util/interactTest.py | 142 +++++++++++++++++++++---- tests/Exscript/util/startTest.py | 48 ++++++++- 4 files changed, 361 insertions(+), 20 deletions(-) create mode 100644 tests/Exscript/util/collectionsTest.py create mode 100644 tests/Exscript/util/implTest.py diff --git a/tests/Exscript/util/collectionsTest.py b/tests/Exscript/util/collectionsTest.py new file mode 100644 index 00000000..d2c7613e --- /dev/null +++ b/tests/Exscript/util/collectionsTest.py @@ -0,0 +1,55 @@ +import copy +import unittest + +from Exscript.util.collections import OrderedDefaultDict + + +class OrderedDefaultDictTest(unittest.TestCase): + CORRELATE = OrderedDefaultDict + + def testDefaultFactory(self): + groups = OrderedDefaultDict(list) + for name, group in ('r3', 'core'), ('r1', 'edge'), ('r2', 'core'): + groups[group].append(name) + self.assertEqual(list(groups.items()), [('core', ['r3', 'r2']), ('edge', ['r1'])]) + self.assertIs(groups.default_factory, list) + + def testArgumentsArePassedToOrderedDict(self): + counts = OrderedDefaultDict(int, [('b', 2), ('a', 1)], c=3) + counts['d'] += 4 + self.assertEqual(list(counts.items()), [('b', 2), ('a', 1), ('c', 3), ('d', 4)]) + + def testWithoutDefaultFactory(self): + plain = OrderedDefaultDict(None, a=1) + self.assertIsNone(plain.default_factory) + self.assertEqual(plain['a'], 1) + self.assertRaises(KeyError, plain.__getitem__, 'missing') + self.assertNotIn('missing', plain) + self.assertEqual(OrderedDefaultDict(), {}) + + def testDefaultFactoryMustBeCallable(self): + self.assertRaisesRegex(TypeError, 'must be callable', OrderedDefaultDict, []) + + def testCopy(self): + groups = OrderedDefaultDict(list, core=['r1']) + for duplicate in groups.copy(), copy.copy(groups): + self.assertIsInstance(duplicate, OrderedDefaultDict) + self.assertIs(duplicate.default_factory, list) + self.assertEqual(duplicate, groups) + self.assertIs(duplicate['core'], groups['core']) # a shallow copy + duplicate['edge'].append('r2') + self.assertNotIn('edge', groups) + + def testRepr(self): + groups = OrderedDefaultDict(list, core=['r1']) + self.assertEqual( + repr(groups), "OrderedDefaultDict(, OrderedDefaultDict({'core': ['r1']}))" + ) + + +def suite(): + return unittest.TestLoader().loadTestsFromTestCase(OrderedDefaultDictTest) + + +if __name__ == '__main__': + unittest.TextTestRunner(verbosity=2).run(suite()) diff --git a/tests/Exscript/util/implTest.py b/tests/Exscript/util/implTest.py new file mode 100644 index 00000000..f8f5c93d --- /dev/null +++ b/tests/Exscript/util/implTest.py @@ -0,0 +1,136 @@ +import io +import threading +import unittest +from unittest import mock + +import Exscript.util.impl +from Exscript.util.impl import Context, Decorator, debug, deprecated, deprecation, synchronized + + +class implTest(unittest.TestCase): + CORRELATE = Exscript.util.impl + + def testDeprecation(self): + def old_api(): + deprecation('old_api() is deprecated') + + with self.assertWarnsRegex(DeprecationWarning, 'old_api') as cm: + old_api() # the warning names the caller of the deprecated function + self.assertEqual(cm.filename, __file__) + + def testDeprecated(self): + @deprecated + def add(a, b=0): + """Adds two numbers.""" + return a + b + + self.assertEqual(add.__name__, 'add') + self.assertEqual(add.__doc__, 'Adds two numbers.') + with self.assertWarnsRegex(DeprecationWarning, r'Call to deprecated function add\.') as cm: + self.assertEqual(add(1, b=2), 3) + self.assertEqual(cm.filename, __file__) + + def tagged(): + pass + + tagged.label = 'kept' + self.assertEqual(deprecated(tagged).label, 'kept') + + def testSynchronized(self): + class Resource: + @synchronized + def use(self, probe): + """Uses the resource.""" + return probe() + + @synchronized + def nested(self): + return self.use(lambda: 'reentrant') + + def locked_for_other_threads(resource): + result = [] + + def try_lock(): + acquired = resource._sync_lock.acquire(block=False) + if acquired: + resource._sync_lock.release() + result.append(not acquired) + + other = threading.Thread(target=try_lock) + other.start() + other.join() + return result[0] + + resource = Resource() + self.assertEqual(Resource.use.__name__, 'use') + self.assertEqual(Resource.use.__doc__, 'Uses the resource.') + self.assertTrue(resource.use(lambda: locked_for_other_threads(resource))) + self.assertFalse(locked_for_other_threads(resource)) # released + self.assertEqual(resource.nested(), 'reentrant') + + # Each object has its lock, and an object may bring its own. + lock = resource._sync_lock + resource.use(lambda: None) + self.assertIs(resource._sync_lock, lock) + other = Resource() + other.use(lambda: None) + self.assertIsNot(other._sync_lock, lock) + own = Resource() + own._sync_lock = mock.MagicMock() + own.use(lambda: None) + own._sync_lock.__enter__.assert_called_once_with() + + def testDebug(self): + @debug + def divide(a, b): + return a / b + + with mock.patch('sys.stdout', new_callable=io.StringIO) as stdout: + self.assertEqual(divide(6, b=3), 2) + self.assertEqual(stdout.getvalue(), "Entering divide(6,) {'b': 3}\nLeaving divide(): 2.0\n") + + with mock.patch('sys.stdout', new_callable=io.StringIO) as stdout: + self.assertRaises(ZeroDivisionError, divide, 1, 0) + output = stdout.getvalue() + self.assertTrue(output.startswith('Entering divide(1, 0) {}\nTraceback caught:\n'), output) + self.assertIn('ZeroDivisionError', output) + self.assertNotIn('Leaving', output) + + +class DecoratorTest(unittest.TestCase): + CORRELATE = Decorator + + def testAttributes(self): + class Target: + name = 'target' + + target = Target() + wrapper = Decorator(target) + self.assertIs(wrapper.obj, target) + self.assertEqual(wrapper.name, 'target') + wrapper.name = 'changed' # writes through to the wrapped object + self.assertEqual(target.name, 'changed') + wrapper.obj = Target() # an attribute of the wrapper itself + self.assertIsNot(wrapper.obj, target) + self.assertRaises(AttributeError, getattr, wrapper, 'missing') + + def testContext(self): + account = mock.Mock() + with Context(account) as context: + self.assertIs(context.obj, account) + with context.context() as inner: # does not release + inner.get_name() + account.release.assert_not_called() + account.get_name.assert_called_once_with() + account.release.assert_called_once_with() + + +def suite(): + loader = unittest.TestLoader() + return unittest.TestSuite( + [loader.loadTestsFromTestCase(implTest), loader.loadTestsFromTestCase(DecoratorTest)] + ) + + +if __name__ == '__main__': + unittest.TextTestRunner(verbosity=2).run(suite()) diff --git a/tests/Exscript/util/interactTest.py b/tests/Exscript/util/interactTest.py index 8b91d67e..db6fb4bd 100644 --- a/tests/Exscript/util/interactTest.py +++ b/tests/Exscript/util/interactTest.py @@ -4,6 +4,7 @@ import unittest.mock import Exscript.util.interact +from Exscript import Account from Exscript.util.interact import InputHistory @@ -46,44 +47,149 @@ def testSet(self): self.assertEqual(self.history.get('bar', '...'), 'myvalue2') self.assertEqual(self.history.get('bar', None), 'myvalue2') + def testExistingFile(self): + with open(self.filename, 'w', encoding='utf-8') as fp: + fp.write('[script]\nhost = router1\n\n[other]\nhost = router2\n') + history = InputHistory(self.filename, 'script') + self.assertEqual(history.get('host'), 'router1') + history.set('user', 'admin') + self.assertEqual(InputHistory(self.filename, 'other').get('host'), 'router2') + self.assertEqual(InputHistory(self.filename, 'script').get('user'), 'admin') + class interactTest(unittest.TestCase): CORRELATE = Exscript.util.interact + def setUp(self): + tmpdir = tempfile.TemporaryDirectory() + self.addCleanup(tmpdir.cleanup) + self.tmpdir = tmpdir.name + self.history = InputHistory(os.path.join(tmpdir.name, 'history'), 'test') + + def typing(self, *answers): + """Lets input() return the given answers; returns the list of the prompts shown.""" + prompts = [] + answers = iter(answers) + + def fake_input(prompt=''): + prompts.append(prompt) + return next(answers) + + patcher = unittest.mock.patch('builtins.input', fake_input) + patcher.start() + self.addCleanup(patcher.stop) + return prompts + def testPrompt(self): - pass - # Can't really be tested, as it is interactive. + prompt = Exscript.util.interact.prompt + prompts = self.typing('', ' router2 ', '', ' x ') + self.assertEqual(prompt('host', 'Host', 'router1', history=self.history), 'router1') + self.assertEqual(self.history.get('host'), 'router1') + self.assertEqual(prompt('host', 'Host', 'router1', history=self.history), 'router2') + self.assertEqual(self.history.get('host'), 'router2') + + # With doverh=False, the input from the history is the default. + self.assertEqual( + prompt('host', 'Host', 'router1', doverh=False, history=self.history), 'router2' + ) + self.assertEqual(prompt('path', 'Path', 'x', strip=False, history=self.history), ' x ') + self.assertEqual( + prompts, ['Host [router1]: ', 'Host [router1]: ', 'Host [router2]: ', 'Path [x]: '] + ) + + def testPromptHistoryDefaultForEmptyHistory(self): + prompts = self.typing('') + value = Exscript.util.interact.prompt( + 'port', 'Port', '22', doverh=False, history=self.history + ) + self.assertEqual(value, '22') + self.assertEqual(prompts, ['Port [22]: ']) + + def testPromptCheck(self): + def check(value): + if len(value) < 4: + return ['Too short.', 'Please enter at least 4 characters.'] + return None + + prompts = self.typing('abc', 'abcd') + with unittest.mock.patch('builtins.print') as output: + value = Exscript.util.interact.prompt( + 'name', 'Name', 'abc', check=check, history=self.history + ) + self.assertEqual(value, 'abcd') + self.assertEqual(prompts, ['Name [abc]: ', 'Name [abc]: ']) + output.assert_called_once_with('Too short.\nPlease enter at least 4 characters.') + self.assertEqual(self.history.get('name'), 'abcd') # only the accepted input + + def testPromptWithoutHistory(self): + # The input is stored in ~/.exscript_history. + self.typing('router1') + with unittest.mock.patch.dict( + os.environ, {'HOME': self.tmpdir, 'USERPROFILE': self.tmpdir} + ): + value = Exscript.util.interact.prompt('host', 'Host', 'r') + self.assertEqual(value, 'router1') + with open(os.path.join(self.tmpdir, '.exscript_history'), encoding='utf-8') as fp: + self.assertIn('host = router1', fp.read()) def testGetFilename(self): # A name of a file that does not exist is rejected, and the prompt repeats. - tmpdir = tempfile.TemporaryDirectory() - self.addCleanup(tmpdir.cleanup) - existing = os.path.join(tmpdir.name, 'hosts.txt') + existing = os.path.join(self.tmpdir, 'hosts.txt') with open(existing, 'w', encoding='utf-8'): pass - history = InputHistory(os.path.join(tmpdir.name, 'history')) - missing = os.path.join(tmpdir.name, 'missing.txt') - inputs = iter([missing, ' ' + existing + ' ']) - with ( - unittest.mock.patch('builtins.input', lambda prompt: next(inputs)), - unittest.mock.patch('builtins.print') as output, - ): + history = InputHistory(os.path.join(self.tmpdir, 'history')) + missing = os.path.join(self.tmpdir, 'missing.txt') + self.typing(missing, ' ' + existing + ' ') + with unittest.mock.patch('builtins.print') as output: filename = Exscript.util.interact.get_filename('hosts', 'Host file', history=history) self.assertEqual(filename, existing) output.assert_called_once_with('File not found. Please enter a filename.') self.assertEqual(history.get('hosts'), existing) def testGetUser(self): - pass - # Can't really be tested, as it is interactive. + get_user = Exscript.util.interact.get_user + prompts = self.typing('', 'bob', '') + with unittest.mock.patch('getpass.getuser', return_value='alice'): + self.assertEqual(get_user(), 'alice') + self.assertEqual(get_user(), 'bob') + self.assertEqual(get_user('Login'), 'alice') + self.assertEqual( + prompts, + [ + 'Please enter your user name [alice]: ', + 'Please enter your user name [alice]: ', + 'Login [alice]: ', + ], + ) + + def testGetUserWithoutDefault(self): + prompts = self.typing('carol', 'dave') + with unittest.mock.patch('getpass.getuser', return_value=''): + self.assertEqual(Exscript.util.interact.get_user(), 'carol') + with unittest.mock.patch('getpass.getuser', side_effect=KeyError('no such user')): + self.assertEqual(Exscript.util.interact.get_user(), 'dave') + self.assertEqual(prompts, ['Please enter your user name: '] * 2) def testGetLogin(self): - pass - # Can't really be tested, as it is interactive. + prompts = self.typing('') + with ( + unittest.mock.patch('getpass.getuser', return_value='alice'), + unittest.mock.patch('getpass.getpass', return_value='S3cr3t') as getpass, + ): + self.assertEqual(Exscript.util.interact.get_login(), ('alice', 'S3cr3t')) + getpass.assert_called_once_with('Please enter your password: ') # not echoed + self.assertEqual(len(prompts), 1) def testReadLogin(self): - pass - # Can't really be tested, as it is interactive. + self.typing('bob') + with ( + unittest.mock.patch('getpass.getuser', return_value='alice'), + unittest.mock.patch('getpass.getpass', return_value='S3cr3t'), + ): + account = Exscript.util.interact.read_login() + self.assertIsInstance(account, Account) + self.assertEqual(account.get_name(), 'bob') + self.assertEqual(account.get_password(), 'S3cr3t') def suite(): diff --git a/tests/Exscript/util/startTest.py b/tests/Exscript/util/startTest.py index 2dd63b14..3848bb88 100644 --- a/tests/Exscript/util/startTest.py +++ b/tests/Exscript/util/startTest.py @@ -1,5 +1,6 @@ import unittest from multiprocessing import Value +from unittest import mock import Exscript import Exscript.util.start @@ -12,6 +13,16 @@ def count_calls(job, host, conn, data, **kwargs): data.value += 1 +def fail_once(job, host, conn, data, **kwargs): + data.value += 1 + if data.value == 1: + raise RuntimeError('first attempt failed') + + +def record_login_state(job, host, conn, data, **kwargs): + data.value = conn.is_app_authorized() * 10 + conn.is_app_authenticated() + + class startTest(unittest.TestCase): CORRELATE = Exscript.util.start @@ -46,16 +57,49 @@ def testRun(self): self.doTest(run) + def testRunAttempts(self): + from Exscript.util.decorator import bind + from Exscript.util.start import run + + run(self.account, 'dummy://localhost', bind(fail_once, self.data), verbose=0) + self.assertEqual(self.data.value, 1) # not retried + + self.data.value = 0 + run(self.account, 'dummy://localhost', bind(fail_once, self.data), attempts=2, verbose=0) + self.assertEqual(self.data.value, 2) + def testQuickrun(self): - pass # can't really be tested, as it is user interactive + from Exscript.util.start import quickrun + + with mock.patch.object(Exscript.util.start, 'read_login', return_value=self.account): + self.doTest(lambda account, *args, **kwargs: quickrun(*args, **kwargs)) def testStart(self): + from Exscript.util.decorator import bind from Exscript.util.start import start self.doTest(start) + start(self.account, 'dummy://localhost', bind(record_login_state, self.data), verbose=0) + self.assertEqual(self.data.value, 11) # authenticated and authorized + start( + self.account, + 'dummy://localhost', + bind(record_login_state, self.data), + only_authenticate=True, + verbose=0, + ) + self.assertEqual(self.data.value, 1) # authenticated only + def testQuickstart(self): - pass # can't really be tested, as it is user interactive + from Exscript.util.decorator import bind + from Exscript.util.start import quickstart + + with mock.patch.object(Exscript.util.start, 'read_login', return_value=self.account): + self.doTest(lambda account, *args, **kwargs: quickstart(*args, **kwargs)) + callback = bind(record_login_state, self.data) + quickstart('dummy://localhost', callback, only_authenticate=True, verbose=0) + self.assertEqual(self.data.value, 1) def suite(): From 336d1e738260ab01f811b09216245f77211f80be Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 10 Nov 2026 17:32:27 +0100 Subject: [PATCH 091/250] Test the protocol adapters more thoroughly --- tests/Exscript/protocols/ExpectTest.py | 311 +++++++++++++- tests/Exscript/protocols/NetconfTest.py | 479 +++++++++++++++++++++- tests/Exscript/protocols/OsGuesserTest.py | 15 + tests/Exscript/protocols/ProtocolsTest.py | 117 ++++++ tests/Exscript/protocols/SSH2Test.py | 381 ++++++++++++++++- tests/Exscript/protocols/TelnetTest.py | 55 ++- tests/Exscript/protocols/replay.py | 7 +- tests/Exscript/protocols/telnetlibTest.py | 32 ++ 8 files changed, 1389 insertions(+), 8 deletions(-) create mode 100644 tests/Exscript/protocols/ProtocolsTest.py diff --git a/tests/Exscript/protocols/ExpectTest.py b/tests/Exscript/protocols/ExpectTest.py index 67fe3cf3..1dda5570 100644 --- a/tests/Exscript/protocols/ExpectTest.py +++ b/tests/Exscript/protocols/ExpectTest.py @@ -1,6 +1,11 @@ +import io +import logging import os import re +import signal import socket +import sys +import tempfile import threading import time import unittest @@ -8,13 +13,15 @@ from Exscript import Account, PrivateKey from Exscript.emulators import VirtualDevice -from Exscript.protocols import SSH2, Dummy, Telnet +from Exscript.protocols import SSH2, Dummy, Telnet, drivers from Exscript.protocols.exception import ( ConnectionClosedException, + InvalidCommandException, LoginFailure, ProtocolException, TimeoutException, ) +from Exscript.protocols.osguesser import OsGuesser from Exscript.protocols.protocol import ( _SETTLE_TIME, Protocol, @@ -113,7 +120,7 @@ def testConnectionClosed(self): def testEndsWithAnchor(self): for pattern in r'#$', r'#\Z', r'(?:#|>)$', r'a\\$', r'a\\\Z': self.assertTrue(_ends_with_anchor(re.compile(pattern)), pattern) - for pattern in r'#', r'\$', r'a\\\$', r'\\Z', r'[$]', r'(a$|b)': + for pattern in r'#', r'\$', r'a\\\$', r'\\Z', r'aZ', r'[$]', r'(a$|b)': self.assertFalse(_ends_with_anchor(re.compile(pattern)), pattern) self.assertFalse(_ends_with_anchor(re.compile(r'#$', re.M))) # $ ends every line @@ -220,6 +227,240 @@ def testFailedLogoutStillCloses(self): self.assertFalse(conn.connected) +class DriverSwitchTest(unittest.TestCase): + """The OS guess may replace the driver while waiting; the wait goes on with the new driver.""" + + def testWaitforContinuesWithTheNewDriver(self): + conn = connect(['\r\nUser Access Verification\r\n\r\nUsername: '], debug=4) + with self.assertLogs('Exscript.protocols.protocol', logging.DEBUG) as logs: + conn.waitfor('Username: ') + self.assertEqual(conn.get_driver().name, 'ios') + self.assertIn('retrying to wait for a prompt', '\n'.join(logs.output)) + + def testSetDriverWhileConnected(self): + conn = connect(['show version\r\nrouter# ']) + conn.set_driver('ios') + conn.expect_prompt() + self.assertEqual(conn.response, 'show version') + + def testSetDriver(self): + conn = connect([]) + driver = drivers.Driver('custom') + conn.set_driver(driver) + self.assertIs(conn.get_driver(), driver) + self.assertRaises(TypeError, conn.set_driver, 'no-such-driver') + self.assertRaises(TypeError, conn.set_driver, 42) + self.assertIs(conn.get_driver(), driver) + + +class LoginTest(unittest.TestCase): + """App-level logins: S/Key, bailout, an account for the device, and a closed connection.""" + + account = Account('admin', 'secret', needs_lock=False) + + def connect(self, chunks, **kwargs): + return connect(chunks, driver='generic', **kwargs) # no OS guessing + + def testHostClosesTheConnection(self): + conn = self.connect(['User: ']) + self.assertRaises(ConnectionClosedException, conn.login, self.account) + self.assertEqual(conn.sent, ['admin\r']) + + def testSKey(self): + # The one-time password is sent instead of the password, and it is not logged. + challenge = 'otp-md4 9999 abc123 ext\r\nPassword: ' + conn = self.connect(['User: ', challenge, '\r\nrouter> '], debug=5) + challenges = [] + conn.otp_requested_event.connect(lambda account, seq, seed: challenges.append((seq, seed))) + with self.assertLogs('Exscript.protocols.protocol', logging.DEBUG) as logs: + conn.login(Account('admin', 'password', needs_lock=False)) + self.assertEqual(conn.sent, ['admin\r', 'ACTA AMMO CAR WEB BIN YAP\r']) + self.assertEqual(challenges, [(9999, 'abc123')]) + self.assertIn('Sending: ', '\n'.join(logs.output)) + self.assertNotIn('ACTA', '\n'.join(logs.output)) + + def testBailoutAfterThePassword(self): + conn = self.connect(['Password: ', '\r\nrouter> ']) + conn.app_authenticate(self.account, bailout=True) + self.assertTrue(conn.is_app_authenticated()) + self.assertEqual(conn.sent, ['secret\r']) + conn.expect_prompt() # left for the caller + + def testBailoutAfterTheOneTimePassword(self): + conn = self.connect(['otp-md4 9999 abc123 ext\r\nPassword: ', '\r\nrouter> ']) + conn.app_authenticate(Account('admin', 'password', needs_lock=False), bailout=True) + self.assertEqual(conn.sent, ['ACTA AMMO CAR WEB BIN YAP\r']) + conn.expect_prompt() + + def testAuthorizeWithoutPassword(self): + # The device shows its prompt at once, so the unknown password is not needed. + conn = self.connect(['\r\nrouter# ']) + conn.app_authorize(Account('admin', None, needs_lock=False)) + self.assertTrue(conn.is_app_authorized()) + self.assertEqual(conn.sent, []) + + def testAccountForTheDevice(self): + # The app account logs into the device; the other account is for the protocol. + conn = self.connect(['User: ', 'Password: ', '\r\nrouter> ']) + conn.login(self.account, app_account=Account('operator', 'op-secret', needs_lock=False)) + self.assertEqual(conn.sent, ['operator\r', 'op-secret\r']) + + def testProtocolAuthenticationHappensOnce(self): + conn = self.connect(['\r\nrouter> ']) + with mock.patch.object(conn, '_protocol_authenticate') as protocol_authenticate: + conn.protocol_authenticate(self.account) + conn.authenticate(self.account) + protocol_authenticate.assert_called_once_with('admin', 'secret') + self.assertTrue(conn.is_app_authenticated()) + + +class ResponseTest(unittest.TestCase): + """Error lines, timeouts, and prompts that stay in the buffer.""" + + def testLongErrorLineIsCut(self): + line = '% Invalid input detected at ' + 'x' * 300 + conn = connect(['bad\r\n' + line + '\r\nrouter# '], driver='ios') + with self.assertRaises(InvalidCommandException) as cm: + conn.execute('bad') + message = 'Error in the response from the device: ' + line[:200] + '...' + self.assertEqual(str(cm.exception), message) + + def testDriverWithoutErrorPatterns(self): + driver = drivers.Driver('lenient') + driver.error_re = [] + conn = connect(['bad\r\n% Invalid input\r\nrouter# '], driver=driver) + conn.execute('bad') + self.assertIn('% Invalid input', conn.response) + + def testTimeoutWithoutAnyData(self): + conn = connect([None]) + with self.assertRaises(TimeoutException) as cm: + conn.expect_prompt() + self.assertEqual( + str(cm.exception), "Timeout while waiting for a response/prompt. Buffer: ''" + ) + + def testExecuteWithoutConsumingThePrompt(self): + conn = connect(['show version\r\nrouter# '], driver='ios') + conn.execute('show version', consume=False) + self.assertEqual(conn.response, 'show version') + conn.expect_prompt() # the prompt is still in the buffer + self.assertEqual(conn.response, 'show version') + + def testPager(self): + # A script answers the pager of a device that pages its output. + conn = connect( + ['show run\r\nline 1\r\n --More-- ', '\r \rline 2\r\nrouter# '], driver='ios' + ) + conn.send('show run\r') + conn.expect('--More--') + self.assertIn('line 1', conn.response) + conn.send(' ') + conn.expect_prompt() + self.assertIn('line 2', conn.response) + self.assertEqual(conn.sent, ['show run\r', ' ']) + + +class ConnectionTest(unittest.TestCase): + """The connection as a context manager, its representation, and the logfile.""" + + def testContextManager(self): + conn = Scripted([]) + conn.close = mock.Mock(wraps=conn.close) + with conn as same: + self.assertIs(same, conn) + conn.connect('router', 23) + conn.close.assert_called_once_with(force=False) + self.assertEqual(conn.sent, ['exit\r']) # the logout of the driver + self.assertFalse(conn.connected) + + conn.close.reset_mock() + with self.assertRaises(RuntimeError), conn: + conn.connect('router', 23) + raise RuntimeError('the script failed') + conn.close.assert_called_once_with(force=True) + self.assertEqual(conn.sent, ['exit\r']) + + def testRepr(self): + self.assertRegex( + repr(Telnet()), r'^$' + ) + conn = connect([]) + self.assertRegex(repr(conn), r'^$') + + def testAlreadyConnected(self): + conn = connect([]) + self.assertRaisesRegex(ProtocolException, 'already connected to router:23', conn.connect) + + def testLogfileChangesWhileConnected(self): + tmpdir = tempfile.TemporaryDirectory() + self.addCleanup(tmpdir.cleanup) + first, second = (os.path.join(tmpdir.name, name) for name in ('first.log', 'second.log')) + conn = connect(['banner\n', 'more\n'], logfile=first) + conn.expect('banner') + conn.logfile = second + conn.expect('more') + conn.logfile = None + conn.close(force=True) + for filename, content in (first, 'banner\n'), (second, 'more\n'): + with open(filename, encoding='utf-8') as fp: + self.assertEqual(fp.read(), content) + + def testStreams(self): + conn = Scripted([]) + self.assertIsInstance(conn.os_guesser, OsGuesser) + self.assertIs(conn.stderr, sys.stderr) + stream = io.StringIO() + conn.stderr = stream + self.assertIs(conn.stderr, stream) + conn.stderr = None + self.assertIs(conn.stderr, sys.stderr) + + def testAdapterWithoutTransport(self): + # Subclasses implement the transport. + conn = Protocol(connect_rate_limit=0) + self.assertRaises(NotImplementedError, conn.connect, 'router', 23) + self.assertFalse(conn.connected) + + +class Shell(Scripted): + """Opens the interactive shell like the network adapters do.""" + + def _interact(self, key_handlers=None, handle_window_size=True): + return self._open_shell(None, key_handlers, handle_window_size) + + +class WindowsShellTest(unittest.TestCase): + """interact() without termios: a thread receives, and the typed characters are sent.""" + + def interact(self, stdin, send_data=None): + conn = Shell(['welcome\r\n']) + conn.connect('router', 23) + conn.send_data = send_data + handled = [] + with ( + mock.patch('Exscript.protocols.protocol._have_termios', False), + mock.patch('sys.stdin', stdin), + ): + conn.interact({'\x19': handled.append}) + for thread in threading.enumerate(): + if thread.name == 'channel.recv': + thread.join(5) + return conn, handled + + def testTypedCharacters(self): + conn, handled = self.interact(io.StringIO('ls\x19\r'), io.StringIO()) + self.assertEqual(conn.sent, ['l', 's', '\r']) + self.assertEqual(conn.send_data.getvalue(), 'ls\r') + self.assertEqual(handled, [conn]) + self.assertIn('welcome', conn.stdout.getvalue()) + + def testEndOfInput(self): + # ^Z or F6; the typed characters are sent without a send_data stream, too. + conn, _ = self.interact(mock.Mock(read=mock.Mock(side_effect=['a', EOFError]))) + self.assertEqual(conn.sent, ['a']) + + def start_daemon(test, daemon_cls, device, **kwargs): """Starts a test server for the device; it stops when the test ends.""" with socket.socket() as sock: @@ -310,3 +551,69 @@ def testSSH2(self): encoding='utf-8', ) self.interact(SSHd, conn, key=PrivateKey.from_file(KEYFILE)) + + +@unittest.skipUnless(hasattr(os, 'openpty'), 'interact() reads from a POSIX terminal') +class TerminalTest(unittest.TestCase): + """interact() on a POSIX terminal: key handlers, and the size of the local terminal.""" + + def interact(self, daemon_cls, conn, **kwargs): + import tty # POSIX only + + device = VirtualDevice(HOST, login_type=VirtualDevice.LOGIN_TYPE_NONE, strict=False) + device.add_command(r'say \S+', lambda command: command.split()[1] + '!') + daemon = daemon_cls(HOST, 0, device, **kwargs) + daemon.timeout = 0.05 + device.add_command('exit', daemon.exit_command) + daemon.start() + self.addCleanup(daemon.join) + self.addCleanup(daemon.exit) + self.addCleanup(conn.close, force=True) + conn.connect(HOST, daemon.port) + conn.protocol_authenticate(Account('user', 'password', needs_lock=False)) + conn.send_data = io.StringIO() + received = [] + conn.data_received_event.connect(received.append) + + master, slave = os.openpty() # interact() closes the slave + self.addCleanup(os.close, master) + raw = threading.Event() + setcbreak = tty.setcbreak + + def set_raw_mode(fd, *args): + setcbreak(fd, *args) + raw.set() + + def type_keys(): + raw.wait(5) # the switch to raw mode discards the keys typed before + os.write(master, b'\x19say hi\rexit\r') + + typist = threading.Thread(target=type_keys) + typist.start() + handled = [] + old_handler = signal.getsignal(signal.SIGWINCH) + with ( + mock.patch('sys.stdin', mock.Mock(encoding='utf-8', fileno=lambda: slave)), + mock.patch('tty.setcbreak', set_raw_mode), + mock.patch('Exscript.protocols.protocol.get_terminal_size', return_value=(40, 132)), + mock.patch.object(conn, '_set_terminal_size', wraps=conn._set_terminal_size) as resize, + ): + conn.interact({'\x19': handled.append}) + typist.join() + self.assertEqual(handled, [conn]) # ^Y was handled, not sent + self.assertIn('hi!', ''.join(received)) + self.assertEqual(conn.send_data.getvalue(), 'say hi\rexit\r') + resize.assert_called_once_with(40, 132) + self.assertEqual(signal.getsignal(signal.SIGWINCH), old_handler) + + def testTelnet(self): + self.interact(Telnetd, Telnet(timeout=1, connect_rate_limit=0)) + + def testSSH2(self): + conn = SSH2( + timeout=1, + connect_rate_limit=0, + verify_fingerprint=False, + preferred_authentications=['password'], + ) + self.interact(SSHd, conn, key=PrivateKey.from_file(KEYFILE)) diff --git a/tests/Exscript/protocols/NetconfTest.py b/tests/Exscript/protocols/NetconfTest.py index e61ee7c5..957cd643 100644 --- a/tests/Exscript/protocols/NetconfTest.py +++ b/tests/Exscript/protocols/NetconfTest.py @@ -1,6 +1,12 @@ -"""Tests for the parts of the NETCONF adapter that need no server.""" +""" +Tests for the NETCONF adapter: the parts that need no server, sessions with +a small NETCONF server, and the replay of a recorded netopeer2 session. +""" import inspect +import io +import os +import re import subprocess import sys import threading @@ -8,16 +14,48 @@ import ncclient.operations.rpc import ncclient.transport.session +import paramiko +from lxml import etree from ncclient.manager import make_device_handler from ncclient.operations.rpc import RPC, RPCReply from ncclient.transport.session import HelloHandler from ncclient.transport.ssh import SSHSession from ncclient.xml_ import NCElement, new_ele -from Exscript.protocols import netconf +from Exscript import Account +from Exscript.protocols import drivers, netconf +from Exscript.protocols.exception import ( + ConnectionClosedException, + LoginFailure, + ProtocolException, + RPCError, + TimeoutException, +) from Exscript.protocols.netconf import Netconf +from Exscript.util.xml import XMLValidationError + +from .replay import Recording, ReplayServer, decode_messages, encode_message +from .SSH2Test import KeyServer BASE = 'urn:ietf:params:netconf:base:1.0' +BASE_11 = 'urn:ietf:params:netconf:base:1.1' +CANDIDATE = 'urn:ietf:params:netconf:capability:candidate:1.0' +NS = 'urn:ietf:params:xml:ns:netconf:base:1.0' +EOM = b']]>]]>' +KEYFILE = os.path.join(os.path.dirname(__file__), 'id_rsa') +RECORDING = os.path.join( + os.path.dirname(__file__), os.pardir, os.pardir, 'recordings', 'netopeer2', + 'netopeer2-netconf.jsonl', +) # fmt: skip +GET_CONFIG = '' +LOCK = '' +FILTER = ( + '' + '' +) +CLOSE = object() +_message_id = re.compile(rb'message-id="([^"]*)"') class ReassembleChunksTest(unittest.TestCase): @@ -30,6 +68,7 @@ def testWithoutChunks(self): def testTrailingBytes(self): self.assertRaises(ValueError, Netconf.reassemble_chunks, '\n#2\nok\n##\nx') + self.assertRaises(ValueError, Netconf.reassemble_chunks, '\n#2\nokx') def testManyChunks(self): chunk = 'x' * 4096 @@ -83,3 +122,439 @@ def testImportLoadsNoLxml(self): [sys.executable, '-c', code], capture_output=True, text=True, check=True ) self.assertEqual(result.stdout.strip(), '[]') + + +class NetconfTest(unittest.TestCase): + """The adapter without a session.""" + + CORRELATE = Netconf + + def setUp(self): + self.conn = Netconf(timeout=1, connect_rate_limit=0) + + def testNoSessionInformationBeforeLogin(self): + self.assertIsNone(self.conn.get_client_capabilities()) + self.assertIsNone(self.conn.get_server_capabilities()) + self.assertIsNone(self.conn.get_session_id()) + self.assertIsNone(self.conn.get_response_element()) + + def testPrompts(self): + driver = self.conn.get_driver() + self.assertEqual(self.conn.get_prompt(), driver.netconf_prompt_re) + self.assertEqual(self.conn.get_error_prompt(), driver.netconf_error_re) + self.conn.set_prompt(r'') + self.conn.set_error_prompt(r'') + self.assertEqual([p.pattern for p in self.conn.get_prompt()], [r'']) + self.assertEqual([p.pattern for p in self.conn.get_error_prompt()], [r'']) + + def testNoShell(self): + self.conn.autoinit() # nothing to initialize + self.assertRaises(NotImplementedError, self.conn.interact) + + def testAppLevelAuthenticationIsNotNeeded(self): + self.conn.app_authenticate() + self.assertTrue(self.conn.is_app_authenticated()) + self.conn.auto_app_authorize() + self.assertTrue(self.conn.is_app_authorized()) + + +class NetconfServer: + """ + A NETCONF server on the netconf SSH subsystem, on 127.0.0.1 at a free port. + + :param reply: Called with each RPC (bytes) except close-session. Returns + the content of the rpc-reply (str; bytes are sent as they are), None + to send no reply, or CLOSE to close the session. close-session gets + , and then the server closes the session. + :param capabilities: The capabilities in the hello of the server. If both + hellos have base:1.1, the messages are framed in chunks (RFC 6242), + else with ]]>]]>. + :param chunk_size: Splits each message into chunks of this many bytes. + :param piece_size: Writes the data to the channel in pieces of this many bytes. + :param subsystem: Whether the netconf subsystem is available. + :param channels: Whether the server opens session channels. + """ + + def __init__( + self, + test, + reply=None, + *, + capabilities=(BASE, CANDIDATE), + chunk_size=None, + piece_size=None, + subsystem=True, + channels=True, + ): + self.reply = reply or (lambda rpc: '') + self.capabilities = capabilities + self.chunk_size = chunk_size + self.piece_size = piece_size + self.client_hello = None + self.hello_received = threading.Event() + self.rpcs = [] + self.channel = None # of the latest session + self.chunked = False + self._threads = [] + test.addCleanup(self._join) # after the server closed the transports + key = paramiko.RSAKey(filename=KEYFILE) + self.port = KeyServer(test, key, interface=self._interface(subsystem, channels)).port + + def _interface(self, subsystem, channels): + fake = self + + class Interface(paramiko.ServerInterface): + def get_allowed_auths(self, username): + return 'password' + + def check_auth_password(self, username, password): + return paramiko.AUTH_SUCCESSFUL + + def check_channel_request(self, kind, chanid): + if channels: + return paramiko.OPEN_SUCCEEDED + return paramiko.OPEN_FAILED_ADMINISTRATIVELY_PROHIBITED + + def check_channel_subsystem_request(self, channel, name): + if not subsystem or name != 'netconf': + return False + thread = threading.Thread(target=fake._serve, args=(channel,)) + fake._threads.append(thread) + thread.start() + return True + + return Interface + + def _join(self): + for thread in self._threads: + thread.join(5) + + def send(self, message): + """Sends a message (str or bytes), framed like the messages of the session.""" + if isinstance(message, str): + message = message.encode('utf-8') + sizes = [self.chunk_size] * (len(message) // self.chunk_size + 1) if self.chunk_size else [] + self._write(encode_message(message, self.chunked, sizes)) + + def _write(self, data): + size = self.piece_size or len(data) + for pos in range(0, len(data), size): + self.channel.sendall(data[pos : pos + size]) + + def _serve(self, channel): + self.channel = channel + capabilities = ''.join(f'{c}' for c in self.capabilities) + hello = ( + f'{capabilities}' + '4711' + ) + stream = b'' + + def read(chunked): + nonlocal stream + while not (messages := decode_messages(stream, chunked)[0]): + data = channel.recv(65536) + if not data: + return None + stream += data + message, sizes = messages[0] + stream = stream[len(encode_message(message, chunked, sizes)) :] + return message + + try: + self._write(hello.encode() + EOM) # hellos are always framed like this + self.client_hello = read(chunked=False) + if self.client_hello is None: + return + self.chunked = BASE_11 in self.capabilities and BASE_11.encode() in self.client_hello + self.hello_received.set() + while (rpc := read(self.chunked)) is not None: + self.rpcs.append(rpc) + closing = operation(rpc) == 'close-session' + content = '' if closing else self.reply(rpc) + if content is CLOSE: + break + if content is None: + continue + if isinstance(content, str): + content = content.encode('utf-8') + message_id = _message_id.search(rpc).group(1) + self.send( + b'%s' + % (NS.encode(), message_id, content) + ) + if closing: + break + except (OSError, EOFError): # paramiko raises EOFError when the client closed + pass + finally: + try: + channel.close() # sends a message, which fails if the client closed first + except (OSError, EOFError): + pass + + +def operation(rpc): + """Returns the name of the operation in the given RPC.""" + root = etree.fromstring(rpc.strip()) + return etree.QName(root[0]).localname + + +class SessionTest(unittest.TestCase): + """Sessions with a NETCONF 1.0 server.""" + + def login(self, reply=None, driver=None, timeout=1, **kwargs): + self.server = NetconfServer(self, reply, **kwargs) + conn = Netconf( + timeout=timeout, + connect_rate_limit=0, + verify_fingerprint=False, + preferred_authentications=['password'], + driver=driver, + ) + self.addCleanup(conn.close, force=True) + conn.connect('127.0.0.1', self.server.port) + conn.login(Account('admin', 'S3cr3t', needs_lock=False)) + return conn + + def testHello(self): + conn = self.login() + self.assertEqual(conn.get_server_capabilities(), [BASE, CANDIDATE]) + self.assertEqual(conn.get_session_id(), '4711') + self.assertIn(BASE_11, conn.get_client_capabilities()) + self.assertTrue(self.server.hello_received.wait(5)) + self.assertIn(BASE_11.encode(), self.server.client_hello) + self.assertTrue(conn.is_app_authorized()) + # The response of the login is the hello, which has no element. + self.assertIn('4711', conn.response) + self.assertIsNone(conn.get_response_element()) + + def testExecute(self): + conn = self.login(lambda rpc: 'r1') + conn.execute(GET_CONFIG) + self.assertEqual([operation(rpc) for rpc in self.server.rpcs], ['get-config']) + self.assertTrue(conn.response.startswith(']]>', conn.response) + element = conn.get_response_element() + self.assertEqual(element.tag, f'{{{NS}}}rpc-reply') + self.assertEqual(element.findtext(f'.//{{{NS}}}hostname'), 'r1') + + def testFraming11(self): + # Many small chunks, written in pieces that do not end with them. + content = 'r1rack 7' + conn = self.login( + lambda rpc: content, capabilities=(BASE, BASE_11), chunk_size=7, piece_size=5 + ) + conn.execute(GET_CONFIG) + self.assertTrue(self.server.chunked) + self.assertEqual([operation(rpc) for rpc in self.server.rpcs], ['get-config']) + self.assertNotIn('\n#', conn.response) + self.assertTrue(conn.response.endswith(content + ''), conn.response) + element = conn.get_response_element() + self.assertEqual(element.findtext(f'.//{{{NS}}}location'), 'rack 7') + + def testNotification(self): + conn = self.login() + conn.execute( + '' + ) + self.server.send(NOTIFICATION) + conn.expect('') + self.assertIn('link down', conn.response) + + def testSecondLoginKeepsTheSession(self): + conn = self.login() + session = conn._ncc_session + account = Account('admin', 'S3cr3t', needs_lock=False) + conn.login(account, app_account=Account('other', 'secret', needs_lock=False)) + self.assertIs(conn._ncc_session, session) + conn.execute(GET_CONFIG) + self.assertEqual(len(self.server.rpcs), 1) + + def testRPCErrors(self): + errors = ( + 'applicationinvalid-value' + 'errortoo-big' + '/interfaces/interface[name="eth0"]/mtu' + 'MTU out of range' + 'mtu' + 'protocoloperation-failed' + 'commit failed' + ) + conn = self.login(lambda rpc: errors) + with self.assertRaises(RPCError) as cm: + conn.execute('') + error = cm.exception + self.assertEqual(str(error), 'error: MTU out of range') + self.assertEqual(error.errlist[0], error) + self.assertEqual( + (error.type, error.tag, error.severity, error.message, error.app_tag, error.path), + ( + 'application', + 'invalid-value', + 'error', + 'MTU out of range', + 'too-big', + '/interfaces/interface[name="eth0"]/mtu', + ), + ) + self.assertIn('mtu', error.info) + self.assertEqual(etree.QName(error.element).localname, 'rpc-error') + other = error.errlist[1] + self.assertEqual( + (str(other), other.severity, other.tag), ('commit failed', '', 'operation-failed') + ) + self.assertIn('MTU out of range', conn.response) # the full reply + + def testWarningsAreNoErrors(self): + # Without , like servers send warnings (RFC 6241, section 4.4). + warning = ( + 'applicationoperation-failed' + 'warningdeprecated' + '' + ) + conn = self.login(lambda rpc: warning) + conn.execute('') + self.assertIn('deprecated', conn.response) + + def testInvalidRPCIsNotSent(self): + conn = self.login() + with self.assertRaises(XMLValidationError) as cm: + conn.execute('') + self.assertIn('XMLSyntaxError', str(cm.exception)) + self.assertIn('', cm.exception.xmlstring) + self.assertTrue(any('shifted by +1' in note for note in cm.exception.__notes__)) + with self.assertRaises(XMLValidationError) as cm: + conn.execute('\ud800') # not encodable + cause = cm.exception.__context__ + self.assertIsInstance(cause, UnicodeEncodeError) + self.assertEqual((cause.start, cause.end), (12, 13)) # the positions in the body + self.assertEqual(self.server.rpcs, []) + + def testTimeout(self): + conn = self.login(lambda rpc: None if operation(rpc) == 'get' else '', timeout=0.3) + self.assertRaises(TimeoutException, conn.execute, '') + self.assertIsNone(conn.get_response_element()) + self.assertRaises(TimeoutException, conn.expect, 'never') + + def testServerClosesTheSession(self): + conn = self.login(lambda rpc: CLOSE, timeout=0.3) + self.assertRaises(ProtocolException, conn.execute, '') + self.assertRaises(ConnectionClosedException, conn.expect_prompt) + conn.close() # the logout fails silently + self.assertFalse(conn.connected) + + def testCloseSession(self): + sent = [] + conn = self.login() + conn.data_sent_event.connect(sent.append) + conn.close() + self.assertEqual([operation(rpc) for rpc in self.server.rpcs], ['close-session']) + self.assertIn('close-session/>', ''.join(sent)) # pretty-printed + self.assertFalse(conn.connected) + + def testSubsystemRefused(self): + for options in {'subsystem': False}, {'channels': False}: + with self.assertRaisesRegex(LoginFailure, 'any subsystem: netconf'): + self.login(**options) + + def testHugeTree(self): + depth = 300 # lxml refuses documents nested deeper than 256 levels without huge_tree + content = '' + '' * depth + '' * depth + '
' + server = NetconfServer(self, lambda rpc: content) + conn = Netconf( + timeout=1, + connect_rate_limit=0, + verify_fingerprint=False, + preferred_authentications=['password'], + ) + self.addCleanup(conn.close, force=True) + self.assertFalse(conn.huge_tree) + conn.huge_tree = True + conn.connect('127.0.0.1', server.port) + conn.login(Account('admin', 'S3cr3t', needs_lock=False)) + self.assertTrue(conn._ncc_manager.huge_tree) + conn.execute(GET_CONFIG) + self.assertEqual(len(conn.get_response_element().xpath('//*[local-name()="a"]')), depth) + conn.huge_tree = False + self.assertFalse(conn._ncc_manager.huge_tree) + + def testDeviceHandlerThatTransformsTheReply(self): + # The Junos device handler of ncclient returns the reply as an NCElement. + driver = drivers.Driver('junos-netconf') + driver.netconf_device_params = {'name': 'junos'} + replies = iter(['23.4R1', LOCK_ERROR]) + conn = self.login(lambda rpc: next(replies), driver=driver) + conn.execute(GET_CONFIG) + self.assertIsInstance(conn._ncc_response, NCElement) + self.assertEqual(conn.get_response_element().findtext('.//version'), '23.4R1') + with self.assertRaisesRegex(RPCError, 'error: configuration database locked'): + conn.execute(LOCK) + + +NOTIFICATION = ( + '' + '2026-11-10T12:00:00Zlink down' +) +LOCK_ERROR = ( + 'protocollock-denied' + 'error' + 'configuration database locked' +) + + +class ReplayedSessionTest(unittest.TestCase): + """The NETCONF 1.1 session of netopeer2, replayed.""" + + def setUp(self): + server = ReplayServer(Recording(RECORDING)) + server.__enter__() + self.addCleanup(server.close) + self.stdout = io.StringIO() + self.conn = Netconf( + timeout=2, + connect_rate_limit=0, + verify_fingerprint=False, + preferred_authentications=['password'], + stdout=self.stdout, + ) + self.addCleanup(self.conn.close, force=True) + self.conn.connect('127.0.0.1', server.port) + self.conn.login(Account('admin', 'S3cr3t', needs_lock=False)) + self.server = server + + def testHello(self): + capabilities = self.conn.get_server_capabilities() + self.assertIn(BASE_11, capabilities) + self.assertIn('urn:ietf:params:netconf:capability:writable-running:1.0', capabilities) + self.assertEqual(self.conn.get_session_id(), '1') + self.assertEqual(self.conn.guess_os(), 'unknown') + + def testGetConfigAndFilter(self): + self.conn.execute(GET_CONFIG) + self.assertNotIn('\n##\n', self.conn.response) # the chunks are reassembled + element = self.conn.get_response_element() + keystore = '{urn:ietf:params:xml:ns:yang:ietf-keystore}keystore' + self.assertIsNotNone(element.find(f'{{{NS}}}data/{keystore}')) + self.assertEqual(self.conn.get_response(), self.conn.response.strip()) + + self.conn.execute(FILTER) + element = self.conn.get_response_element() + monitoring = '{urn:ietf:params:xml:ns:yang:ietf-netconf-monitoring}' + capabilities = element.findall(f'.//{monitoring}capability') + self.assertIn(BASE_11, [capability.text for capability in capabilities]) + + def testLockedTwice(self): + self.conn.execute(GET_CONFIG) + self.conn.execute(FILTER) + self.conn.execute(LOCK) + with self.assertRaises(RPCError) as cm: + self.conn.execute(LOCK) + messages = [error.message for error in cm.exception.errlist] + self.assertEqual(messages, ['Requested resource already locked', 'User callback failed.']) + self.assertEqual({error.tag for error in cm.exception.errlist}, {'operation-failed'}) + + def testCloseSession(self): + self.conn.close() + self.assertIn('close-session/>', self.stdout.getvalue()) + self.assertEqual(self.server.errors, []) diff --git a/tests/Exscript/protocols/OsGuesserTest.py b/tests/Exscript/protocols/OsGuesserTest.py index 179fa636..3be9ed03 100644 --- a/tests/Exscript/protocols/OsGuesserTest.py +++ b/tests/Exscript/protocols/OsGuesserTest.py @@ -103,6 +103,21 @@ def testCheckNetconfCapabilities(self): ) self.assertEqual(osg.get('os'), "junos") + def testCheckNetconfCapabilitiesAsText(self): + osg = OsGuesser() + osg.check_netconf_capabilities(None) + osg.check_netconf_capabilities([]) + self.assertEqual(osg.get('os'), 'unknown') + osg.check_netconf_capabilities( + 'urn:ietf:params:netconf:base:1.0\nhttp://xml.juniper.net/netconf/junos/1.0' + ) + self.assertEqual(osg.get('os'), 'junos') + + def testSetWithoutValue(self): + self.sa.set('os', 'ios', 50) + self.sa.set('os', None, 100) # ignored + self.assertEqual(self.sa.get('os', 50), 'ios') + def guess(head='', response='', protocol_info=''): """Returns the guess after the given data, fed one character per read (the worst case).""" diff --git a/tests/Exscript/protocols/ProtocolsTest.py b/tests/Exscript/protocols/ProtocolsTest.py new file mode 100644 index 00000000..e4c34af1 --- /dev/null +++ b/tests/Exscript/protocols/ProtocolsTest.py @@ -0,0 +1,117 @@ +"""Tests for the functions of the Exscript.protocols package.""" + +import os +import unittest + +import Exscript.protocols +from Exscript import Account, Host +from Exscript.emulators import VirtualDevice +from Exscript.protocols import ( + SSH2, + Dummy, + Netconf, + Telnet, + connect, + create_protocol, + get_protocol_from_name, + prepare, +) +from Exscript.servers import Telnetd + +PSEUDODEV = os.path.join(os.path.dirname(__file__), 'pseudodev.py') + + +class ProtocolsTest(unittest.TestCase): + CORRELATE = Exscript.protocols + + def startServer(self): + device = VirtualDevice('r1', echo=True) + device.add_command('show version', 'Version 1.0') + daemon = Telnetd('127.0.0.1', 0, device) + daemon.timeout = 0.05 + daemon.start() + self.addCleanup(daemon.join) + self.addCleanup(daemon.exit) + return daemon.port + + def testGetProtocolFromName(self): + classes = { + 'dummy': Dummy, + 'pseudo': Dummy, + 'telnet': Telnet, + 'ssh': SSH2, + 'ssh2': SSH2, + 'netconf': Netconf, + } + for name, cls in classes.items(): + self.assertIs(get_protocol_from_name(name), cls) + self.assertRaisesRegex( + ValueError, 'Unsupported protocol "rlogin"', get_protocol_from_name, 'rlogin' + ) + + def testCreateProtocol(self): + conn = create_protocol('telnet', timeout=7, driver='ios') + self.assertIsInstance(conn, Telnet) + self.assertEqual(conn.get_timeout(), 7) + self.assertEqual(conn.get_driver().name, 'ios') + self.assertFalse(conn.connected) + self.assertRaisesRegex( + ValueError, 'Unsupported protocol "rlogin"', create_protocol, 'rlogin' + ) + + def testPrepare(self): + conn = prepare('telnet://router1:2323', timeout=7) + self.assertIsInstance(conn, Telnet) + self.assertEqual(conn.get_timeout(), 7) + self.assertFalse(conn.connected) + self.assertIsInstance(prepare('router1'), SSH2) # the default protocol + self.assertIsInstance(prepare('router1', default_protocol='telnet'), Telnet) + self.assertIsInstance(prepare(Host('netconf://router1')), Netconf) + self.assertRaisesRegex(ValueError, 'Unsupported protocol "rlogin"', prepare, 'rlogin://r1') + + def testPseudoDevice(self): + # The address of a pseudo host is a file with the commands of the device. + host = Host('pseudo://' + PSEUDODEV) + host.set_account(Account('admin', 'secret', needs_lock=False)) + self.assertIsInstance(prepare(host), Dummy) + conn = connect(host) + self.addCleanup(conn.close, force=True) + self.assertTrue(conn.is_app_authenticated()) + conn.execute('show version') + self.assertIn('Cisco Internetwork Operating System Software', conn.response) + + def testConnectWithLogin(self): + port = self.startServer() + conn = connect(f'telnet://admin:secret@127.0.0.1:{port}', timeout=2, connect_rate_limit=0) + self.addCleanup(conn.close, force=True) + self.assertTrue(conn.connected) + self.assertTrue(conn.is_app_authenticated()) + conn.execute('show version') + self.assertIn('Version 1.0', conn.response) + + def testConnectWithoutLogin(self): + port = self.startServer() + host = Host(f'127.0.0.1:{port}', default_protocol='telnet') + conn = connect(host, timeout=2, connect_rate_limit=0) + self.addCleanup(conn.close, force=True) + self.assertIsInstance(conn, Telnet) + self.assertTrue(conn.connected) + self.assertFalse(conn.is_app_authenticated()) + conn.login(Account('admin', 'secret', needs_lock=False)) + self.assertTrue(conn.is_app_authorized()) + + def testConnectWithTheAccountOfTheHost(self): + port = self.startServer() + host = Host(f'telnet://127.0.0.1:{port}') + host.set_account(Account('admin', 'secret', needs_lock=False)) + conn = connect(host, timeout=2, connect_rate_limit=0) + self.addCleanup(conn.close, force=True) + self.assertTrue(conn.is_app_authenticated()) + + +def suite(): + return unittest.TestLoader().loadTestsFromTestCase(ProtocolsTest) + + +if __name__ == '__main__': + unittest.TextTestRunner(verbosity=2).run(suite()) diff --git a/tests/Exscript/protocols/SSH2Test.py b/tests/Exscript/protocols/SSH2Test.py index c383cc0d..934d776c 100644 --- a/tests/Exscript/protocols/SSH2Test.py +++ b/tests/Exscript/protocols/SSH2Test.py @@ -1,6 +1,10 @@ import io +import logging import os +import select +import shutil import socket +import stat import tempfile import threading import unittest @@ -9,8 +13,9 @@ import paramiko from Exscript import Account, PrivateKey +from Exscript.emulators import VirtualDevice from Exscript.protocols import SSH2, Netconf -from Exscript.protocols.exception import LoginFailure +from Exscript.protocols.exception import ConnectionClosedException, LoginFailure from Exscript.protocols.ssh2 import _prefer_key_type from Exscript.servers import SSHd, sshd @@ -71,9 +76,11 @@ def _accept(self): for host_key in self.keys: transport.add_server_key(host_key) self._transports.append(transport) + server = self.interface() + server.transport = transport # e.g. to drop the connection # With an event, the key exchange runs in the transport's thread, and the # transport stays open until the client or close() closes it. - transport.start_server(event=threading.Event(), server=self.interface()) + transport.start_server(event=threading.Event(), server=server) def close(self): self._closing.set() @@ -165,6 +172,44 @@ def testKnownKeyTypeIsPreferred(self): self.writeKnownHosts((self.name, self.server_key)) self.connect() + def testStandardPortUsesThePlainName(self): + # OpenSSH writes hosts on port 22 without brackets and port. + connect = socket.create_connection + + def redirect(address, *args, **kwargs): + return connect((address[0], self.server.port), *args, **kwargs) + + with mock.patch('socket.create_connection', redirect): + conn = SSH2(timeout=1, connect_rate_limit=0, known_hosts=self.known_hosts) + self.addCleanup(conn.close, force=True) + conn.host_key_policy = 'auto-add' + conn.connect(HOST, 22) + conn.close(force=True) + with open(self.known_hosts, encoding='utf-8') as fp: + self.assertEqual(fp.read().split()[0], HOST) + conn.host_key_policy = 'reject' + conn.connect(HOST, 22) # known now + + def testAutoAddToAnExistingFile(self): + line = f'192.0.2.1 {self.other_key.get_name()} {self.other_key.get_base64()}\n' + for content in '', line: + with open(self.known_hosts, 'w', encoding='utf-8') as fp: + fp.write(content) + conn = self.connect(host_key_policy='auto-add') + conn.close(force=True) + with open(self.known_hosts, encoding='utf-8') as fp: + lines = fp.read().splitlines() + expected = f'{self.name} ssh-rsa {self.server_key.get_base64()}' + self.assertEqual(lines, content.splitlines() + [expected]) + + def testAutoAddCreatesThePrivateDirectory(self): + directory = os.path.join(os.path.dirname(self.known_hosts), 'new', '.ssh') + self.known_hosts = os.path.join(directory, 'known_hosts') + self.connect(host_key_policy='auto-add') + self.assertEqual(len(paramiko.HostKeys(self.known_hosts).lookup(self.name)), 1) + if os.name == 'posix': + self.assertEqual(stat.S_IMODE(os.stat(directory).st_mode), 0o700) + def testPreferKeyType(self): options = mock.Mock(key_types=('ssh-ed25519', 'rsa-sha2-512', 'rsa-sha2-256')) _prefer_key_type(options, 'ssh-rsa') @@ -415,6 +460,336 @@ def testOptions(self): self.assertEqual(self.requestedSize(term_width=80, term_height=25), [(80, 25)]) +PASSWORD = 'S3cr3t-P@ss' + + +class _Auth(paramiko.ServerInterface): + """ + Accepts the authentication methods that the class attributes allow, and + records the requests. Use a subclass per test (see AuthenticationTest.serve). + """ + + allowed = 'password' + password = PASSWORD + keys = () # the accepted public keys + rounds = () # keyboard-interactive: the prompts of each round, (text, echo) + answers = [PASSWORD] # the expected keyboard-interactive answers + none = False # whether the "none" method succeeds + banner = None + channels = True + drop_after_failure = False # closes the connection after a failed keyboard-interactive + requests: list[tuple] = [] + + def get_allowed_auths(self, username): + return self.allowed + + def get_banner(self): + return (self.banner, 'en-US') if self.banner else (None, None) + + def check_auth_none(self, username): + self.requests.append(('none', username)) + return paramiko.AUTH_SUCCESSFUL if self.none else paramiko.AUTH_FAILED + + def check_auth_password(self, username, password): + self.requests.append(('password', username, password)) + return paramiko.AUTH_SUCCESSFUL if password == self.password else paramiko.AUTH_FAILED + + def check_auth_publickey(self, username, key): + if ('publickey', username, key.get_base64()) not in self.requests: + self.requests.append(('publickey', username, key.get_base64())) + return paramiko.AUTH_SUCCESSFUL if key in self.keys else paramiko.AUTH_FAILED + + def check_auth_interactive(self, username, submethods): + self.requests.append(('keyboard-interactive', username)) + self._rounds = list(self.rounds) + return self._query() + + def _query(self): + return paramiko.InteractiveQuery('', '', *self._rounds.pop(0)) + + def check_auth_interactive_response(self, responses): + self.requests.append(('answers', list(responses))) + if self._rounds: + return self._query() + if list(responses) == self.answers: + return paramiko.AUTH_SUCCESSFUL + if self.drop_after_failure: + self.transport.close() + return paramiko.AUTH_FAILED + + def check_channel_request(self, kind, chanid): + if self.channels: + return paramiko.OPEN_SUCCEEDED + return paramiko.OPEN_FAILED_ADMINISTRATIVELY_PROHIBITED + + def check_channel_pty_request( + self, channel, term, width, height, pixelwidth, pixelheight, modes + ): + return True + + def check_channel_shell_request(self, channel): + return True + + +class AuthenticationTest(unittest.TestCase): + """ + The SSH authentication methods. The tests use no SSH agent and no key + files of the user that runs them. + """ + + def setUp(self): + home = tempfile.TemporaryDirectory() + self.addCleanup(home.cleanup) + self.home = home.name + for patcher in ( + mock.patch.dict(os.environ, {'HOME': self.home, 'USERPROFILE': self.home}), + mock.patch('paramiko.Agent'), + ): + patcher.start() + self.addCleanup(patcher.stop) + paramiko.Agent.return_value.get_keys.return_value = [] + + def serve(self, **attributes): + """Starts a server with the given _Auth attributes; returns its list of requests.""" + self.interface = type('Auth', (_Auth,), {'requests': [], **attributes}) + self.server = KeyServer(self, paramiko.RSAKey(filename=keyfile), interface=self.interface) + return self.interface.requests + + def client(self, **kwargs): + conn = SSH2(timeout=1, connect_rate_limit=0, verify_fingerprint=False, **kwargs) + self.addCleanup(conn.close, force=True) + conn.connect(HOST, self.server.port) + return conn + + def login(self, account=None, **kwargs): + conn = self.client(**kwargs) + conn.protocol_authenticate(account or Account('admin', PASSWORD, needs_lock=False)) + return conn + + def testPassword(self): + requests = self.serve() + conn = self.login() + self.assertTrue(conn.is_protocol_authenticated()) + self.assertEqual(requests, [('none', 'admin'), ('password', 'admin', PASSWORD)]) + + def testWrongPassword(self): + requests = self.serve( + allowed='keyboard-interactive,password', + rounds=[[('Password: ', False)]], + password='other', + answers=['other'], + ) + with self.assertRaises(LoginFailure) as cm: + self.login() + message = str(cm.exception) + self.assertIn('_paramiko_auth_interactive() failed', message) + self.assertIn('_paramiko_auth_password() failed', message) + self.assertNotIn(PASSWORD, message) + self.assertEqual( + [r[0] for r in requests], ['none', 'keyboard-interactive', 'answers', 'password'] + ) + + def testPasswordIsNotLogged(self): + self.serve( + allowed='keyboard-interactive,password', + rounds=[[('Password: ', False)]], + answers=['other'], + ) + logfile = os.path.join(self.home, 'session.log') + with self.assertLogs(level=logging.DEBUG) as logs: + conn = self.login(debug=5, logfile=logfile) + conn.close(force=True) + self.assertTrue( + any('Authenticating with _paramiko_auth_password' in line for line in logs.output) + ) + self.assertEqual([line for line in logs.output if PASSWORD in line], []) + with open(logfile, encoding='utf-8') as fp: + self.assertNotIn(PASSWORD, fp.read()) + + def testKeyboardInteractiveWithAnEmptyRound(self): + # Some servers send a round without prompts first. + rounds = [[], [('Password: ', False)]] + requests = self.serve(allowed='keyboard-interactive', rounds=rounds) + self.login() + self.assertEqual(requests[-2:], [('answers', []), ('answers', [PASSWORD])]) + + def testKeyboardInteractiveSKey(self): + # An S/Key challenge gets the one-time password, not the password. + rounds = [[('otp-md4 9999 abc123 ext\nResponse: ', True)]] + answers = ['ACTA AMMO CAR WEB BIN YAP'] # otp('password', 'abc123', 9999) + requests = self.serve(allowed='keyboard-interactive', rounds=rounds, answers=answers) + conn = self.client() + challenges = [] + conn.otp_requested_event.connect(lambda *args: challenges.append(args)) + conn.protocol_authenticate(Account('admin', 'password', needs_lock=False)) + self.assertEqual(requests[-1], ('answers', answers)) + [(account, seq, seed)] = challenges + self.assertEqual((account.get_name(), seq, seed), ('admin', 9999, 'abc123')) + + def testKeyFile(self): + public = paramiko.RSAKey(filename=keyfile) + requests = self.serve(allowed='publickey', keys=(public,)) + key = PrivateKey.from_file(keyfile, password=None) + conn = self.login(Account('admin', key=key, needs_lock=False)) + self.assertTrue(conn.is_protocol_authenticated()) + self.assertEqual(requests, [('publickey', 'admin', public.get_base64())]) + + def testEncryptedKeyFile(self): + public = paramiko.RSAKey(filename=keyfile) + filename = os.path.join(self.home, 'id_rsa') + public.write_private_key_file(filename, password='phrase') # noqa: S106 + requests = self.serve(allowed='publickey', keys=(public,)) + key = PrivateKey.from_file(filename, password='phrase') # noqa: S106 + self.login(Account('admin', PASSWORD, key=key, needs_lock=False)) + self.assertEqual(requests, [('publickey', 'admin', public.get_base64())]) + + def testRejectedKey(self): + # Key logins do not fall back to the password. + requests = self.serve(allowed='publickey,password') + account = Account('admin', PASSWORD, key=PrivateKey.from_file(keyfile), needs_lock=False) + with self.assertRaises((paramiko.SSHException, LoginFailure)) as cm: + self.login(account) + self.assertNotIn(PASSWORD, str(cm.exception)) + public = paramiko.RSAKey(filename=keyfile) + self.assertEqual(requests, [('publickey', 'admin', public.get_base64())]) + + # A key without a file is no key. + account = Account('admin', PASSWORD, key=PrivateKey(), needs_lock=False) + with self.assertRaises((paramiko.SSHException, LoginFailure)): + self.login(account) + self.assertEqual(len(requests), 1) + + def testAgentKeys(self): + accepted, other = paramiko.RSAKey(filename=keyfile), paramiko.RSAKey.generate(1024) + paramiko.Agent.return_value.get_keys.return_value = [other, accepted] + requests = self.serve(allowed='publickey,password', keys=(accepted,)) + self.login() + self.assertEqual([r[0] for r in requests], ['none', 'publickey', 'publickey']) + + def testRejectedAgentKeysFallBackToPassword(self): + rejected = paramiko.RSAKey(filename=keyfile) + paramiko.Agent.return_value.get_keys.return_value = [rejected] + requests = self.serve(allowed='publickey,password') + self.assertTrue(self.login().is_protocol_authenticated()) + self.assertEqual( + requests, + [ + ('none', 'admin'), + ('publickey', 'admin', rejected.get_base64()), + ('password', 'admin', PASSWORD), + ], + ) + + def testAgentErrorFallsBackToPassword(self): + paramiko.Agent.side_effect = paramiko.SSHException('lost ssh-agent') + requests = self.serve(allowed='publickey,password') + self.login() + self.assertEqual(requests, [('none', 'admin'), ('password', 'admin', PASSWORD)]) + + def testKeyFilesOfTheUser(self): + os.mkdir(os.path.join(self.home, '.ssh')) + shutil.copy(keyfile, os.path.join(self.home, '.ssh', 'id_rsa')) + public = paramiko.RSAKey(filename=keyfile) + requests = self.serve(allowed='publickey,password', keys=(public,)) + self.login() + self.assertEqual(requests, [('none', 'admin'), ('publickey', 'admin', public.get_base64())]) + + def testNoneMethod(self): + # Some devices accept any user and ask for the password in the shell. + requests = self.serve(none=True) + self.assertTrue(self.login().is_protocol_authenticated()) + self.assertEqual(requests, [('none', 'admin')]) + + def testBanner(self): + self.serve(banner='Authorized users only!\r\n') + conn = self.login() + self.assertEqual(conn.get_banner(), 'Authorized users only!\r\n') + self.assertIn('Authorized users only!', conn.stdout.getvalue()) + conn.expect('users only!') + + def testPreferredMethodsNotAllowed(self): + requests = self.serve(allowed='password') + with self.assertRaisesRegex(paramiko.BadAuthenticationType, 'invalid preferred auth'): + self.login(preferred_authentications=['publickey']) + self.assertEqual(requests, [('none', 'admin')]) + + def testUnsupportedMethodsAreSkipped(self): + requests = self.serve(allowed='gssapi-with-mic,password') + self.login(preferred_authentications=['gssapi-with-mic', 'password']) + self.assertEqual(requests, [('none', 'admin'), ('password', 'admin', PASSWORD)]) + + def testServerOrderWithoutPreferences(self): + requests = self.serve( + allowed='keyboard-interactive,password', rounds=[[('Password: ', False)]] + ) + conn = self.client() + conn.preferred_authentications = [] + conn.protocol_authenticate(Account('admin', PASSWORD, needs_lock=False)) + self.assertEqual([r[0] for r in requests], ['none', 'keyboard-interactive', 'answers']) + + def testReconnectWhenTheServerDropsTheConnection(self): + requests = self.serve( + allowed='keyboard-interactive,password', + rounds=[[('Password: ', False)]], + answers=['other'], + drop_after_failure=True, + ) + conn = self.login() + self.assertTrue(conn.is_protocol_authenticated()) + self.assertEqual(len(self.server._transports), 2) + self.assertEqual(requests[-1], ('password', 'admin', PASSWORD)) + + def testNoShell(self): + # E.g. an account for file transfers only. + self.serve(channels=False) + with self.assertRaisesRegex(LoginFailure, 'Failed to open a channel with pty/shell'): + self.login() + + +class SessionTest(unittest.TestCase): + """Sending to and closing a session that the host ended or that has pending output.""" + + def setUp(self): + self.device = VirtualDevice(HOST, login_type=VirtualDevice.LOGIN_TYPE_NONE, strict=False) + self.device.add_command('ls', 'file1 file2') + self.daemon = SSHd(HOST, 0, self.device, key=key) + self.daemon.timeout = 0.05 + self.daemon.start() + self.addCleanup(self.daemon.join) + self.addCleanup(self.daemon.exit) + + def login(self, **kwargs): + conn = SSH2( + timeout=1, + connect_rate_limit=0, + verify_fingerprint=False, + preferred_authentications=['password'], + **kwargs, + ) + self.addCleanup(conn.close, force=True) + conn.connect(HOST, self.daemon.port) + conn.protocol_authenticate(Account('user', PASSWORD, needs_lock=False)) + conn.expect_prompt() + return conn + + def testSendAfterTheHostClosed(self): + conn = self.login(debug=1) + self.daemon.exit() + self.daemon.join() + self.assertRaises(ConnectionClosedException, conn.expect_prompt) + with self.assertLogs('Exscript.protocols.ssh2', logging.DEBUG) as logs: + self.assertRaises(OSError, conn.send, 'ls\r') + self.assertIn('Error while writing to connection', logs.output[-1]) + + def testGracefulCloseReceivesThePendingOutput(self): + stdout = io.StringIO() + conn = self.login(stdout=stdout) + conn.send('ls\r') + select.select([conn._channel], [], [], 5) # the output arrived, but was not read + conn.close() + self.assertIn('file1 file2', stdout.getvalue()) + + def suite(): loader = unittest.TestLoader() return unittest.TestSuite( @@ -427,6 +802,8 @@ def suite(): loader.loadTestsFromTestCase(PartialAuthenticationTest), loader.loadTestsFromTestCase(ConnectTest), loader.loadTestsFromTestCase(TerminalSizeTest), + loader.loadTestsFromTestCase(AuthenticationTest), + loader.loadTestsFromTestCase(SessionTest), ] ) diff --git a/tests/Exscript/protocols/TelnetTest.py b/tests/Exscript/protocols/TelnetTest.py index e501488b..f7ae8b33 100644 --- a/tests/Exscript/protocols/TelnetTest.py +++ b/tests/Exscript/protocols/TelnetTest.py @@ -1,7 +1,12 @@ +import io +import logging import socket import threading import unittest +from unittest import mock +from Exscript import Account +from Exscript.emulators import VirtualDevice from Exscript.protocols import Telnet from Exscript.protocols.telnetlib import DO, IAC, NAWS, SB, SE, WILL from Exscript.servers import Telnetd @@ -59,10 +64,58 @@ def testOptions(self): ) +class SessionTest(unittest.TestCase): + """Closing a session, and errors of the connection.""" + + def setUp(self): + self.device = VirtualDevice('r1', login_type=VirtualDevice.LOGIN_TYPE_NONE, strict=False) + self.daemon = Telnetd('127.0.0.1', 0, self.device) + self.daemon.timeout = 0.05 + self.device.add_command('exit', self.daemon.exit_command) + self.daemon.start() + self.addCleanup(self.daemon.join) + self.addCleanup(self.daemon.exit) + + def login(self, **kwargs): + conn = Telnet(timeout=1, connect_rate_limit=0, **kwargs) + self.addCleanup(conn.close, force=True) + conn.connect('127.0.0.1', self.daemon.port) + conn.protocol_authenticate(Account('user', 'password', needs_lock=False)) + conn.expect_prompt() + return conn + + def testGracefulCloseReceivesTheRest(self): + stdout = io.StringIO() + conn = self.login(stdout=stdout) + conn.close() # sends exit; the host answers and closes the connection + self.assertRegex(stdout.getvalue(), r'r1> exit\s+r1> $') + self.assertFalse(conn.connected) + + def testErrorWhileClosing(self): + # E.g. the host resets the connection after the logout. + conn = self.login() + with mock.patch.object(conn._tn, 'receive', side_effect=ConnectionResetError): + conn.close() + self.assertFalse(conn.connected) + + def testErrorWhileSending(self): + conn = self.login(debug=1) + with ( + mock.patch.object(conn._tn, 'write', side_effect=BrokenPipeError), + self.assertLogs('Exscript.protocols.telnet', logging.DEBUG) as logs, + ): + self.assertRaises(BrokenPipeError, conn.send, 'ls\r') + self.assertIn('Error while writing to connection', logs.output[-1]) + + def suite(): loader = unittest.TestLoader() return unittest.TestSuite( - [loader.loadTestsFromTestCase(TelnetTest), loader.loadTestsFromTestCase(TerminalSizeTest)] + [ + loader.loadTestsFromTestCase(TelnetTest), + loader.loadTestsFromTestCase(TerminalSizeTest), + loader.loadTestsFromTestCase(SessionTest), + ] ) diff --git a/tests/Exscript/protocols/replay.py b/tests/Exscript/protocols/replay.py index 1e964884..97f3b63a 100644 --- a/tests/Exscript/protocols/replay.py +++ b/tests/Exscript/protocols/replay.py @@ -149,6 +149,7 @@ def __init__(self, recording, chunks='recorded', seed=0): self._closing = threading.Event() self._sockets = [] self._transports = [] + self._session_threads = [] self._host_key = paramiko.ECDSAKey.generate() if recording.protocol != 'telnet' else None self._listener = socket.create_server(('127.0.0.1', 0)) self._listener.settimeout(0.1) @@ -174,6 +175,8 @@ def close(self): except OSError: pass sock.close() + for thread in self._session_threads: + thread.join(5) def _accept(self): while not self._closing.is_set(): @@ -206,7 +209,9 @@ def _serve(self, sock): def start_session(self, session): self.sessions.append(session) - threading.Thread(target=session.run, daemon=True).start() + thread = threading.Thread(target=session.run, daemon=True) + self._session_threads.append(thread) + thread.start() class _SSHServer(paramiko.ServerInterface): diff --git a/tests/Exscript/protocols/telnetlibTest.py b/tests/Exscript/protocols/telnetlibTest.py index 4c47c21e..ffb475d4 100644 --- a/tests/Exscript/protocols/telnetlibTest.py +++ b/tests/Exscript/protocols/telnetlibTest.py @@ -1,6 +1,7 @@ import logging import socket import unittest +from unittest import mock from hypothesis import given from hypothesis import strategies as st @@ -167,3 +168,34 @@ def testOpen(self): port = listener.getsockname()[1] with self.assertRaisesRegex(OSError, f'=> telnet://127.0.0.1:{port}'): Telnet('127.0.0.1', port, connect_timeout=5) + + def testOpenErrorNamesTheUrl(self): + error = ConnectionRefusedError(111, 'Connection refused') + with ( + mock.patch('socket.create_connection', side_effect=error), + self.assertRaisesRegex(OSError, 'Connection refused => telnet://router1:23') as cm, + ): + Telnet('router1', connect_timeout=5) + self.assertIs(cm.exception.__cause__, error) + + def testCloseTwice(self): + local, remote = socket.socketpair() + self.addCleanup(remote.close) + tn = Telnet() + tn.sock = local + tn.close() + tn.close() + self.assertTrue(tn.eof) + self.assertIsNone(tn.sock) + self.assertEqual(local.fileno(), -1) + self.assertFalse(tn.receive(1)) + + def testNawsWithoutSize(self): + # The size is sent once it is known. + tn = Telnet(termsize=None) + tn.sock = FakeSock([iac(DO, NAWS)]) + while tn.receive(None): + pass + self.assertEqual(tn.sock.sent, [iac(WILL, NAWS)]) + tn.set_window_size(24, 80) + self.assertEqual(tn.sock.sent[-1], iac(SB, NAWS, 0, 80, 0, 24) + iac(SE)) From 464511710a723ea5083bd72a3b63e16a37641fdb Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 10 Nov 2026 17:04:16 +0100 Subject: [PATCH 092/250] Remove Exscript.servers.HTTPd Its digest authentication never worked on Python 3: every login with a valid password failed with a TypeError. http.server or a web framework serve the purpose. --- CHANGES.md | 8 +- docs/migration.rst | 3 + pyproject.toml | 2 - src/Exscript/servers/__init__.py | 2 - src/Exscript/servers/httpd.py | 292 ------------------------------- 5 files changed, 9 insertions(+), 298 deletions(-) delete mode 100644 src/Exscript/servers/httpd.py diff --git a/CHANGES.md b/CHANGES.md index 1d403ecf..0a66365e 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -416,8 +416,12 @@ The names below refer to functions callable from templates, not protocol methods - **BREAKING - Threaded Servers:** The `Server` base class and its `SSHd`/`Telnetd` implementations now run as daemon threads rather than processes. Process-specific - APIs are no longer available; use `exit()` and `join()` for shutdown. `HTTPd` is - unaffected by this change. + APIs are no longer available; use `exit()` and `join()` for shutdown. +- **BREAKING - `HTTPd` Removed:** `Exscript.servers.HTTPd` and + `Exscript.servers.httpd.RequestHandler` are removed. Their HTTP digest + authentication never worked on Python 3: every login with a valid password failed + with a `TypeError`. Use `http.server` from the + standard library or a web framework instead. - **Client Handling:** `SSHd` and `Telnetd` keep listening after a client disconnects, allowing sequential connections. They share the supplied emulator device; concurrent clients are not supported. diff --git a/docs/migration.rst b/docs/migration.rst index a054dc78..592d70de 100644 --- a/docs/migration.rst +++ b/docs/migration.rst @@ -176,6 +176,9 @@ Emulators and test servers - The test servers ``SSHd`` and ``Telnetd`` run as threads; stop them with ``exit()`` and ``join()``. Their ``buf`` attribute and the ``encoding`` argument of ``Telnetd`` are removed. +- ``Exscript.servers.HTTPd`` and its ``RequestHandler`` are removed; their + digest authentication never worked on Python 3. Use ``http.server`` from + the standard library or a web framework. From the 2.9 development versions --------------------------------- diff --git a/pyproject.toml b/pyproject.toml index 9bb33656..dc4be15f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -111,7 +111,6 @@ ignore = [ "src/Exscript/protocols/protocol.py" = ["S101", "S110"] "src/Exscript/protocols/ssh2.py" = ["S110"] "src/Exscript/protocols/telnet.py" = ["S101", "S110"] -"src/Exscript/servers/httpd.py" = ["S105", "S324"] "src/Exscript/stdlib/mysys.py" = ["S603"] "src/Exscript/util/crypt.py" = ["S303"] "src/Exscript/util/file.py" = ["S107"] @@ -132,7 +131,6 @@ ignore_missing_imports = true # ncclient and others ship no type hints # it passes the strict checks; the list only shrinks. module = [ "Exscript.queue", - "Exscript.servers.httpd", "Exscript.util.sigint", "Exscript.workqueue.job", "Exscript.workqueue.mainloop", diff --git a/src/Exscript/servers/__init__.py b/src/Exscript/servers/__init__.py index 985f1e1d..f4964ac1 100644 --- a/src/Exscript/servers/__init__.py +++ b/src/Exscript/servers/__init__.py @@ -24,12 +24,10 @@ Very simple servers, useful for emulating a device for testing. """ -from .httpd import HTTPd from .sshd import SSHd from .telnetd import Telnetd __all__ = [ - 'HTTPd', 'SSHd', 'Telnetd', ] diff --git a/src/Exscript/servers/httpd.py b/src/Exscript/servers/httpd.py deleted file mode 100644 index 96d8d152..00000000 --- a/src/Exscript/servers/httpd.py +++ /dev/null @@ -1,292 +0,0 @@ -# -# Copyright (C) 2010-2017 Samuel Abels -# The MIT License (MIT) -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files -# (the "Software"), to deal in the Software without restriction, -# including without limitation the rights to use, copy, modify, merge, -# publish, distribute, sublicense, and/or sell copies of the Software, -# and to permit persons to whom the Software is furnished to do so, -# subject to the following conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY -# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, -# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE -# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -""" -A threaded HTTP server with support for HTTP/Digest authentication. -""" - -import hashlib -import logging -import time -import urllib.parse -from http.server import BaseHTTPRequestHandler, HTTPServer -from socketserver import ThreadingMixIn -from traceback import format_exc -from urllib.parse import parse_qs, urlparse -from urllib.request import parse_http_list, parse_keqv_list - -_logger = logging.getLogger(__name__) -_logger.addHandler(logging.NullHandler()) - - -def md5hex(x): - return hashlib.md5(x).hexdigest() - - -default_realm = 'exscript' - -# This is convoluted because there's no way to tell 2to3 to insert a -# byte literal. -_HEADER_NEWLINES = [x.encode('ascii') for x in ('\r\n', '\n', '')] - - -def _parse_url(path): - """Given a urlencoded path, returns the path and the dictionary of - query arguments, all in Unicode.""" - - o = urlparse(urllib.parse.unquote_plus(path)) - - path = o.path - args = {} - - # Convert parse_qs' str --> [str] dictionary to a str --> str - # dictionary since we never use multi-value GET arguments - # anyway. - multiargs = parse_qs(o.query, keep_blank_values=True) - for arg, value in list(multiargs.items()): - args[arg] = value[0] - - return path, args - - -def _error_401(handler, msg): - handler.send_response(401) - realm = handler.server.realm - nonce = ("%d:%s" % (time.time(), realm)).encode('utf8') - handler.send_header( - 'WWW-Authenticate', - 'Digest realm="%s",qop="auth",algorithm="MD5",nonce="%s"' % (realm, nonce), - ) - handler.end_headers() - handler.rfile.read() - handler.rfile.close() - handler.wfile.write(msg.encode('utf8')) - handler.wfile.close() - - -def _require_authenticate(func): - '''A decorator to add digest authorization checks to HTTP Request Handlers''' - - def wrapped(self): - if not hasattr(self, 'authenticated'): - self.authenticated = None - if self.authenticated: - return func(self) - - auth = self.headers.get('Authorization') - if auth is None: - msg = "You are not allowed to access this page. Please login first!" - return _error_401(self, msg) - - token, fields = auth.split(' ', 1) - if token != 'Digest': - return _error_401(self, 'Unsupported authentication type') - - # Check the header fields of the request. - cred = parse_http_list(fields) - cred = parse_keqv_list(cred) - keys = 'realm', 'username', 'nonce', 'uri', 'response' - if not all(cred.get(key) for key in keys): - return _error_401(self, 'Incomplete authentication header') - if cred['realm'] != self.server.realm: - return _error_401(self, 'Incorrect realm') - if 'qop' in cred and ('nc' not in cred or 'cnonce' not in cred): - return _error_401(self, 'qop with missing nc or cnonce') - - # Check the username. - username = cred['username'] - password = self.server.get_password(username) - if not username or password is None: - return _error_401(self, 'Invalid username or password') - - # Check the digest string. - location = '%s:%s' % (self.command, self.path) - location = md5hex(location.encode('utf8')) - pwhash = md5hex('%s:%s:%s' % (username, self.server.realm, password)) - - if 'qop' in cred: - info = (cred['nonce'], cred['nc'], cred['cnonce'], cred['qop'], location) - else: - info = cred['nonce'], location - - expect = '%s:%s' % (pwhash, ':'.join(info)) - expect = md5hex(expect.encode('utf8')) - if expect != cred['response']: - return _error_401(self, 'Invalid username or password') - - # Success! - self.authenticated = True - return func(self) - - return wrapped - - -class HTTPd(ThreadingMixIn, HTTPServer): - """ - An HTTP server, derived from Python's HTTPServer but with added - support for HTTP/Digest. Usage:: - - from Exscript.servers import HTTPd, RequestHandler - class MyHandler(RequestHandler): - def handle_GET(self): - self.send_response(200) - self.end_headers() - self.wfile.write('You opened ' + self.path) - - server = HTTPd(('', 8080), MyHandler) - server.add_account('testuser', 'testpassword') - print('started httpserver...') - server.serve_forever() - """ - - daemon_threads = True - - def __init__(self, addr, handler_cls, user_data=None): - """ - Constructor. - - :type address: (str, int) - :param address: The address and port number on which to bind. - :type handler_cls: :class:`RequestHandler` - :param handler_cls: The RequestHandler to use. - :type user_data: object - :param user_data: Optional data that, stored in self.user_data. - """ - self._logger = _logger - self.debug = False - self.realm = default_realm - self.accounts = {} - self.user_data = user_data - HTTPServer.__init__(self, addr, handler_cls) - - def add_account(self, username, password): - """ - Adds a username/password pair that HTTP clients may use to log in. - - :type username: str - :param username: The name of the user. - :type password: str - :param password: The user's password. - """ - self.accounts[username] = password - - def get_password(self, username): - """ - Returns the password of the user with the given name. - - :type username: str - :param username: The name of the user. - """ - return self.accounts.get(username) - - def _dbg(self, msg, exc_info=None): - if not self.debug: - return - self._logger.debug(msg, exc_info=exc_info) - - -class RequestHandler(BaseHTTPRequestHandler): - """ - A drop-in replacement for Python's BaseHTTPRequestHandler that - handles HTTP/Digest. - """ - - def _do_POSTGET(self, handler): - """handle an HTTP request""" - # at first, assume that the given path is the actual path and there are - # no arguments - self.server._dbg(self.path) - - self.path, self.args = _parse_url(self.path) - - # Extract POST data, if any. Clumsy syntax due to Python 2 and - # 2to3's lack of a byte literal. - self.data = b"" - length = self.headers.get('Content-Length') - if length and length.isdigit(): - self.data = self.rfile.read(int(length)) - - # POST data gets automatically decoded into Unicode. The bytestring - # will still be available in the bdata attribute. - self.bdata = self.data - try: - self.data = self.data.decode('utf8') - except UnicodeDecodeError: - self.data = None - - # Run the handler. - try: - handler() - except: - self.send_response(500) - self.end_headers() - self.wfile.write(format_exc().encode('utf8')) - - @_require_authenticate - def do_POST(self): - """ - Do not overwrite; instead, overwrite handle_POST(). - """ - self._do_POSTGET(self.handle_POST) - - @_require_authenticate - def do_GET(self): - """ - Do not overwrite; instead, overwrite handle_GET(). - """ - self._do_POSTGET(self.handle_GET) - - def handle_POST(self): - """ - Overwrite this method to handle a POST request. The default - action is to respond with "error 404 (not found)". - """ - self.send_response(404) - self.end_headers() - self.wfile.write(b'not found') - - def handle_GET(self): - """ - Overwrite this method to handle a GET request. The default - action is to respond with "error 404 (not found)". - """ - self.send_response(404) - self.end_headers() - self.wfile.write(b'not found') - - def send_response(self, code): - """ - See Python's BaseHTTPRequestHandler.send_response(). - """ - BaseHTTPRequestHandler.send_response(self, code) - self.send_header("Connection", "close") - - -if __name__ == '__main__': - try: - server = HTTPd(('', 8123), RequestHandler) - server.add_account('test', 'fo') - print('started httpserver...') - server.serve_forever() - except KeyboardInterrupt: - print('^C received, shutting down server') - server.socket.close() From 5e215fa3594be04bdbea97c9b682fce9c88debdd Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 10 Nov 2026 17:06:08 +0100 Subject: [PATCH 093/250] Send the last line of a template without a line break A template ignored its last line if the line did not end with a line break. The end of the input now ends the last line like a newline. --- CHANGES.md | 4 ++++ docs/migration.rst | 4 ++++ src/Exscript/interpreter/template.py | 6 ++++++ tests/Exscript/interpreter/CommandTest.py | 14 ++++++++++++++ tests/Exscript/interpreter/ParserPropertyTest.py | 6 +++--- 5 files changed, 31 insertions(+), 3 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 0a66365e..ee81a89a 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -362,6 +362,10 @@ marked with **BREAKING**; a migration guide will accompany the release. ## Template Engine +- **BREAKING - Last Line Without a Line Break:** A template also sends its last line + if the line does not end with a line break, e.g. `show version` in + `template.eval(conn, 'ls\nshow version')`. Before, such a line was ignored. + ### Standard Library The names below refer to functions callable from templates, not protocol methods. diff --git a/docs/migration.rst b/docs/migration.rst index 592d70de..8402b81d 100644 --- a/docs/migration.rst +++ b/docs/migration.rst @@ -99,6 +99,10 @@ Templates ``connection.close()`` Logs out first; ``connection.close(true)`` closes at once. +Last line + A template also sends its last line if the line does not end with a + line break. 2.x ignored such a line. + Command line tool ----------------- diff --git a/src/Exscript/interpreter/template.py b/src/Exscript/interpreter/template.py index 4cd391dc..b3066696 100644 --- a/src/Exscript/interpreter/template.py +++ b/src/Exscript/interpreter/template.py @@ -90,6 +90,12 @@ def __init__( while True: lexer.skip(['line_comment']) if self.exit_requested or lexer.current_is('EOF'): + # The end of the input ends the last line like a newline. + if lexer.current_is('EOF') and buffer.strip() != '': + # Execute expects the lexer to have consumed the delimiter + # behind the command; at the end of the input, it is empty. + lexer.forward(0) + self.add(Execute(lexer, parser, self, buffer, escape_offset)) break elif lexer.current_is('open_curly_bracket'): if buffer.strip() != '': diff --git a/tests/Exscript/interpreter/CommandTest.py b/tests/Exscript/interpreter/CommandTest.py index 8df429ac..d4102c33 100644 --- a/tests/Exscript/interpreter/CommandTest.py +++ b/tests/Exscript/interpreter/CommandTest.py @@ -33,6 +33,20 @@ def testCommentsAreNotSent(self): self.evaluate(text) self.assertSent('ls', 'show version # not a comment') + def testLastLineWithoutLineBreak(self): + variables = self.evaluate('ls\nshow version') + self.assertSent('ls', 'show version') + self.assertEqual(variables['__response__'], ['Version 1.0', 'Uptime 5 days']) + template.paste(self.conn, '{x = 1}ls') + self.assertSent('ls', 'show version', 'ls') + + def testUndefinedVariableInTheLastLine(self): + with self.assertRaises(ExecuteError) as cm: + self.evaluate('ls\nshow $nothing') + message = "Undefined variable 'nothing' in undefined:2" + self.assertError(cm.exception, ['show $nothing'], ' ^^^^^^^^', message) + self.assertSent() + def testCommandBeforeACodeBlock(self): variables = self.evaluate('show version{extract /^Version (\\S+)/ as version}\n') self.assertSent('show version') diff --git a/tests/Exscript/interpreter/ParserPropertyTest.py b/tests/Exscript/interpreter/ParserPropertyTest.py index e39f16e7..66a5b7fa 100644 --- a/tests/Exscript/interpreter/ParserPropertyTest.py +++ b/tests/Exscript/interpreter/ParserPropertyTest.py @@ -162,10 +162,10 @@ def testPiecesOfTemplates(self, text): self.assertCompiles(text) @settings(max_examples=100, deadline=None) - @given(plain_lines) - def testPlainTextIsSentAsCommands(self, lines): + @given(plain_lines, st.sampled_from(['', '\n'])) + def testPlainTextIsSentAsCommands(self, lines, end): del self.device.received[:] - template.paste(self.conn, '\n'.join(lines) + '\n') + template.paste(self.conn, '\n'.join(lines) + end) expected = [line.lstrip(' \t') + '\r' for line in lines if line.strip()] self.assertEqual(self.device.received, expected) From ed43f5f0dc6762f7e4ec2572a1978221605cb562 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 10 Nov 2026 17:08:40 +0100 Subject: [PATCH 094/250] Raise a CompileError for Unicode escapes above U+10FFFF An escape sequence such as \UFFFFFFFF raised a SyntaxError of Python, because ast.literal_eval() cannot decode it. Check the range first and raise a CompileError that shows the position of the sequence. --- CHANGES.md | 3 +++ src/Exscript/interpreter/string.py | 8 +++++++- src/Exscript/interpreter/template.py | 8 +++++++- tests/Exscript/interpreter/CommandTest.py | 9 +++++++++ tests/Exscript/interpreter/ParserPropertyTest.py | 7 ++----- tests/Exscript/interpreter/StringTest.py | 7 +++++++ 6 files changed, 35 insertions(+), 7 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index ee81a89a..06333d2f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -365,6 +365,9 @@ marked with **BREAKING**; a migration guide will accompany the release. - **BREAKING - Last Line Without a Line Break:** A template also sends its last line if the line does not end with a line break, e.g. `show version` in `template.eval(conn, 'ls\nshow version')`. Before, such a line was ignored. +- **Unicode Escape Sequences Above U+10FFFF:** A sequence such as `\UFFFFFFFF` in a + command or a string raises a `CompileError` that shows its position. Before, it + raised a `SyntaxError` of Python. ### Standard Library diff --git a/src/Exscript/interpreter/string.py b/src/Exscript/interpreter/string.py index 6a68fa5d..0cbf7d55 100644 --- a/src/Exscript/interpreter/string.py +++ b/src/Exscript/interpreter/string.py @@ -133,7 +133,8 @@ def eval_escape(self, ttype: str, token: str) -> str: unless the parser disables escaping; CR, LF and TAB are always evaluated. An escaped dollar sign stays escaped for the substitution of variables; any other escaped character is returned without the - backslash. + backslash. Raises a CompileError at the current position of the + lexer for a Unicode sequence above U+10FFFF. :param ttype: The token type of the escape sequence. :param token: The escape sequence. @@ -149,6 +150,11 @@ def eval_escape(self, ttype: str, token: str) -> str: ): if token == '\\e': # Invalid sequence in Python. return '\x1b' + if ttype == 'escaped_unicode_32bit' and int(token[2:], 16) > 0x10FFFF: + self.mark_start() + self.mark_end(distance=len(token)) + msg = 'Invalid escape sequence %s: above U+10FFFF' % repr(token) + self.lexer.syntax_error(msg, self) return cast(str, ast.literal_eval('"' + token + '"')) if ttype == 'escaped_common_controls': return cast(str, ast.literal_eval('"' + token + '"')) diff --git a/src/Exscript/interpreter/template.py b/src/Exscript/interpreter/template.py index b3066696..3698c16f 100644 --- a/src/Exscript/interpreter/template.py +++ b/src/Exscript/interpreter/template.py @@ -153,7 +153,8 @@ def eval_escape(self, ttype: str, token: str) -> str: hexadecimal and octal sequences and control characters are evaluated unless the parser disables escaping. An escaped dollar sign stays escaped for the Execute token; any other escaped character is - returned without the backslash. + returned without the backslash. Raises a CompileError at the current + position of the lexer for a Unicode sequence above U+10FFFF. :param ttype: The token type of the escape sequence. :param token: The escape sequence. @@ -170,6 +171,11 @@ def eval_escape(self, ttype: str, token: str) -> str: ): if token == '\\e': # Invalid sequence in Python. return '\x1b' + if ttype == 'escaped_unicode_32bit' and int(token[2:], 16) > 0x10FFFF: + self.mark_start() + self.mark_end(distance=len(token)) + msg = 'Invalid escape sequence %s: above U+10FFFF' % repr(token) + self.lexer.syntax_error(msg, self) return cast(str, ast.literal_eval('"' + token + '"')) # Simple escape sequence with one character. if len(token) == 2: diff --git a/tests/Exscript/interpreter/CommandTest.py b/tests/Exscript/interpreter/CommandTest.py index d4102c33..7c9760fe 100644 --- a/tests/Exscript/interpreter/CommandTest.py +++ b/tests/Exscript/interpreter/CommandTest.py @@ -101,6 +101,15 @@ def testNoEscaping(self): self.evaluate(text, no_escaping=True) self.assertSent('show x41 101 u00e9 U0001F600 t e') + def testUnicodeEscapeSequenceAboveTheRange(self): + with self.assertRaises(CompileError) as cm: + self.evaluate('ls\nshow \\UFFFFFFFF\n') + message = "Invalid escape sequence '\\\\UFFFFFFFF': above U+10FFFF in undefined:2" + self.assertError(cm.exception, ['show \\UFFFFFFFF'], ' ' + '^' * 10, message) + self.assertSent() + self.evaluate('show \\UFFFFFFFF\n', no_escaping=True) + self.assertSent('show UFFFFFFFF') + def testEscapedSpecialCharacters(self): self.evaluate('show \\{a\\} \\\\ \\$x \\# \\"\n') self.assertSent('show {a} \\ $x # "') diff --git a/tests/Exscript/interpreter/ParserPropertyTest.py b/tests/Exscript/interpreter/ParserPropertyTest.py index 66a5b7fa..195a9209 100644 --- a/tests/Exscript/interpreter/ParserPropertyTest.py +++ b/tests/Exscript/interpreter/ParserPropertyTest.py @@ -2,7 +2,7 @@ import unittest import warnings -from hypothesis import assume, given, settings +from hypothesis import given, settings from hypothesis import strategies as st from Exscript.interpreter.exception import PermissionError @@ -64,6 +64,7 @@ '\\x41', '\\u00e9', '\\U0010FFFF', + '\\U00110000', 'if ', 'else', 'end', @@ -98,9 +99,6 @@ '\u00e9', ] -# Escape sequences above U+10FFFF raise SyntaxError instead of CompileError (reported). -INVALID_ESCAPE = re.compile(r'\\U([0-9a-fA-F]{8})') - # Text that the template language does not interpret: no code blocks, no # escape sequences, no variables, no comments and no elements. plain_text = st.text( @@ -126,7 +124,6 @@ def assertCompiles(self, text): """ Compiles the text, and checks that an error shows a position in it. """ - assume(not any(int(code, 16) > 0x10FFFF for code in INVALID_ESCAPE.findall(text))) try: with warnings.catch_warnings(): warnings.simplefilter('error', CompileWarning) diff --git a/tests/Exscript/interpreter/StringTest.py b/tests/Exscript/interpreter/StringTest.py index 78c92f35..39d4d25e 100644 --- a/tests/Exscript/interpreter/StringTest.py +++ b/tests/Exscript/interpreter/StringTest.py @@ -25,6 +25,13 @@ def testNoEscaping(self): text = '\\x41\\101\\u00e9\\e|\\t|\\n|\\r' self.assertString(text, 'x41101u00e9e|\t|\n|\r', no_escaping=True) + def testUnicodeEscapeSequenceAboveTheRange(self): + with self.assertRaises(CompileError) as cm: + self.evaluate('{x = "a\\U00110000"}') + message = "Invalid escape sequence '\\\\U00110000': above U+10FFFF in undefined:1" + self.assertError(cm.exception, ['{x = "a\\U00110000"}'], ' ' * 7 + '^' * 10, message) + self.assertString('\\U00110000', 'U00110000', no_escaping=True) + def testVariables(self): variables = self.evaluate( '{a = "1"}{b = list.new("x", "y")}{n = 2 * 3}{e = list.new()}' From 2181005801506a3004f2f12a44946e4287f43885 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 10 Nov 2026 17:10:16 +0100 Subject: [PATCH 095/250] Require a regular expression after extract extract took the character behind the keyword as the delimiter of its regular expression, so a quoted string worked, and other terms raised confusing errors. Raise a CompileError unless a slash follows the keyword. --- CHANGES.md | 5 +++++ docs/migration.rst | 4 ++++ src/Exscript/interpreter/extract.py | 6 ++++++ tests/Exscript/interpreter/ExtractTest.py | 7 +++++++ tests/Exscript/interpreter/ParserPropertyTest.py | 3 +-- 5 files changed, 23 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 06333d2f..24e78ec1 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -368,6 +368,11 @@ marked with **BREAKING**; a migration guide will accompany the release. - **Unicode Escape Sequences Above U+10FFFF:** A sequence such as `\UFFFFFFFF` in a command or a string raises a `CompileError` that shows its position. Before, it raised a `SyntaxError` of Python. +- **BREAKING - `extract` Requires a Regular Expression:** `extract` raises a + `CompileError` if no regular expression between slashes follows, e.g. in + `{extract "(a)" as y from "a"}` or `{extract x as y}`. Before, the character after + `extract` was taken as the delimiter of the regular expression, so a string in + double quotes worked, and other terms caused confusing errors such as `re.error`. ### Standard Library diff --git a/docs/migration.rst b/docs/migration.rst index 8402b81d..115389c2 100644 --- a/docs/migration.rst +++ b/docs/migration.rst @@ -103,6 +103,10 @@ Last line A template also sends its last line if the line does not end with a line break. 2.x ignored such a line. +``extract`` + Requires a regular expression between slashes. 2.x also accepted a + string in double quotes, e.g. ``{extract "(a)" as y}``. + Command line tool ----------------- diff --git a/src/Exscript/interpreter/extract.py b/src/Exscript/interpreter/extract.py index 85266634..9f9cec0c 100644 --- a/src/Exscript/interpreter/extract.py +++ b/src/Exscript/interpreter/extract.py @@ -70,6 +70,12 @@ def __init__(self, lexer: Lexer, parser: 'Parser', parent: Scope) -> None: lexer.expect(self, 'keyword', 'extract') lexer.expect(self, 'whitespace') _regex_start = lexer.current_char # Helps with later Regex errors. + if not lexer.current_is('regex_delimiter'): + ttype, token = lexer.token() + self.mark_start() + self.mark_end(distance=len(token)) + msg = 'Expected regular expression but got %s %s' % (ttype, repr(token)) + lexer.syntax_error(msg, self) self.regex = Regex(lexer, parser, parent) # Expect "as" keyword. diff --git a/tests/Exscript/interpreter/ExtractTest.py b/tests/Exscript/interpreter/ExtractTest.py index b539b640..140903ea 100644 --- a/tests/Exscript/interpreter/ExtractTest.py +++ b/tests/Exscript/interpreter/ExtractTest.py @@ -114,6 +114,13 @@ def testSyntaxErrors(self): ' ' * 14 + '^', "Expected one of whitespace, newline but got close_curly_bracket '}'", ), + ('{extract x as y}', ' ' * 9 + '^', "Expected regular expression but got varname 'x'"), + ('{extract 1 as y}', ' ' * 9 + '^', "Expected regular expression but got number '1'"), + ( + '{extract "(a)" as y from "a"}', + ' ' * 9 + '^', + "Expected regular expression but got string_delimiter '\"'", + ), ): with self.subTest(text=text): with self.assertRaises(CompileError) as cm: diff --git a/tests/Exscript/interpreter/ParserPropertyTest.py b/tests/Exscript/interpreter/ParserPropertyTest.py index 195a9209..46cc22e2 100644 --- a/tests/Exscript/interpreter/ParserPropertyTest.py +++ b/tests/Exscript/interpreter/ParserPropertyTest.py @@ -15,8 +15,6 @@ COMPILE_ERRORS = (LexerException, NameError, PermissionError, CompileWarning) # Pieces of templates, to reach more paths of the parser than random text. -# The regular expressions of extract statements are always given: without one, -# the parser fails with re.error (reported). FRAGMENTS = [ '{', '}', @@ -74,6 +72,7 @@ ' as ', 'into ', 'extract /', + 'extract ', 'append ', 'fail ', 'try', From 707c895f6b8811828ef09e5a0fe2fa6690940eaa Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 10 Nov 2026 17:12:54 +0100 Subject: [PATCH 096/250] Keep the variables of the caller in compile-time declarations Assignments, append, extract and loop declared their variables when the template was compiled, which replaced the values that the caller had passed. Declare them with the new Scope.declare(), which defines only the variables that are not defined yet. --- CHANGES.md | 7 +++++++ docs/migration.rst | 6 ++++++ src/Exscript/interpreter/append.py | 6 +++--- src/Exscript/interpreter/assign.py | 8 ++++---- src/Exscript/interpreter/extract.py | 11 ++++++----- src/Exscript/interpreter/loop.py | 12 ++++++------ src/Exscript/interpreter/scope.py | 8 ++++++++ tests/Exscript/interpreter/AssignTest.py | 11 +++++++++++ tests/Exscript/interpreter/ExtractTest.py | 4 ++++ tests/Exscript/interpreter/LoopTest.py | 6 ++++++ tests/Exscript/interpreter/ParserTest.py | 5 +++++ 11 files changed, 66 insertions(+), 18 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 24e78ec1..2fcb57c3 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -373,6 +373,13 @@ marked with **BREAKING**; a migration guide will accompany the release. `{extract "(a)" as y from "a"}` or `{extract x as y}`. Before, the character after `extract` was taken as the delimiter of the regular expression, so a string in double quotes worked, and other terms caused confusing errors such as `re.error`. +- **BREAKING - Variables of the Caller Keep Their Values:** A variable that the caller + passes to a template keeps its value until a statement of the template changes it + when the template runs. Before, every assignment, `append`, `extract` and `loop` + statement reset its variables when the template was compiled, even if it never ran: + with `x='hi'`, `show $x\n{x = "new"}` sent `show None`, `{append "b" to x}` with + `x=['a']` gave `['b']`, and `extract ... into x` did not append to the list of the + caller. ### Standard Library diff --git a/docs/migration.rst b/docs/migration.rst index 115389c2..fa2d4274 100644 --- a/docs/migration.rst +++ b/docs/migration.rst @@ -107,6 +107,12 @@ Last line Requires a regular expression between slashes. 2.x also accepted a string in double quotes, e.g. ``{extract "(a)" as y}``. +Variables of the caller + Keep their values until a statement of the template changes them. In + 2.x, assignments, ``append``, ``extract`` and ``loop`` reset their + variables when the template was compiled, e.g. ``append`` and + ``extract ... into`` started with an empty list. + Command line tool ----------------- diff --git a/src/Exscript/interpreter/append.py b/src/Exscript/interpreter/append.py index 1a4d80af..977871c4 100644 --- a/src/Exscript/interpreter/append.py +++ b/src/Exscript/interpreter/append.py @@ -42,8 +42,8 @@ class Append(Token): def __init__(self, lexer: Lexer, parser: 'Parser', parent: Scope) -> None: """ - Constructor. Parses the statement and defines the variable as an - empty list. + Constructor. Parses the statement and declares the variable as an + empty list, unless it is defined already. :param lexer: The lexer that reads the template. :param parser: The parser of the template. @@ -71,7 +71,7 @@ def __init__(self, lexer: Lexer, parser: 'Parser', parent: Scope) -> None: lexer.expect(self, 'whitespace') _, self.varname = lexer.token() lexer.expect(self, 'varname') - self.parent.define(**{self.varname: []}) + self.parent.declare(**{self.varname: []}) # End of statement. self.mark_end(rstrip=True) diff --git a/src/Exscript/interpreter/assign.py b/src/Exscript/interpreter/assign.py index faac8dc6..593b476e 100644 --- a/src/Exscript/interpreter/assign.py +++ b/src/Exscript/interpreter/assign.py @@ -42,9 +42,9 @@ class Assign(Token): def __init__(self, lexer: Lexer, parser: 'Parser', parent: Scope) -> None: """ - Constructor. Parses the statement and declares the variable. Raises - a CompileError if the name starts and ends with two underscores, as - the names of internal variables do. + Constructor. Parses the statement and declares the variable, unless + it is defined already. Raises a CompileError if the name starts and + ends with two underscores, as the names of internal variables do. :param lexer: The lexer that reads the template. :param parser: The parser of the template. @@ -75,7 +75,7 @@ def __init__(self, lexer: Lexer, parser: 'Parser', parent: Scope) -> None: lexer.syntax_error(msg, self) self.expression = Expression(lexer, parser, parent) - self.parent.define(**{self.varname: None}) + self.parent.declare(**{self.varname: None}) # End of statement. self.mark_end(rstrip=True) diff --git a/src/Exscript/interpreter/extract.py b/src/Exscript/interpreter/extract.py index 9f9cec0c..2c8c8392 100644 --- a/src/Exscript/interpreter/extract.py +++ b/src/Exscript/interpreter/extract.py @@ -46,10 +46,11 @@ class Extract(Token): def __init__(self, lexer: Lexer, parser: 'Parser', parent: Scope) -> None: """ - Constructor. Parses the statement and defines the variables as empty - lists. Raises a CompileError on syntax errors, if the parser has - no_prompt set (then there is no response), or if the number of - variables differs from the number of capture groups. + Constructor. Parses the statement and declares the variables as empty + lists, unless they are defined already. Raises a CompileError on + syntax errors, if the parser has no_prompt set (then there is no + response), or if the number of variables differs from the number of + capture groups. :param lexer: The lexer that reads the template. :param parser: The parser of the template. @@ -118,7 +119,7 @@ def __init__(self, lexer: Lexer, parser: 'Parser', parent: Scope) -> None: if lexer.next_if('comma'): continue break - self.parent.define(**self.variables) + self.parent.declare(**self.variables) if len(self.varnames) != self.regex.groups: count = (len(self.varnames), self.regex.groups) diff --git a/src/Exscript/interpreter/loop.py b/src/Exscript/interpreter/loop.py index a24ad266..8ff4d394 100644 --- a/src/Exscript/interpreter/loop.py +++ b/src/Exscript/interpreter/loop.py @@ -49,9 +49,9 @@ class Loop(Token): def __init__(self, lexer: Lexer, parser: 'Parser', parent: Scope) -> None: """ Constructor. Parses the statement up to and including its end - keyword, and defines the loop variables as empty lists. Raises a - CompileError if the number of lists differs from the number of - variables. + keyword, and declares the loop variables as empty lists, unless they + are defined already. Raises a CompileError if the number of lists + differs from the number of variables. :param lexer: The lexer that reads the template. :param parser: The parser of the template. @@ -87,14 +87,14 @@ def __init__(self, lexer: Lexer, parser: 'Parser', parent: Scope) -> None: lexer.next_if('whitespace') _, iter_varname = lexer.token() lexer.expect(self, 'varname') - parent.define(**{iter_varname: []}) + parent.declare(**{iter_varname: []}) self.iter_varnames = [iter_varname] lexer.next_if('whitespace') while lexer.next_if('comma'): lexer.skip(['whitespace', 'newline']) _, iter_varname = lexer.token() lexer.expect(self, 'varname') - parent.define(**{iter_varname: []}) + parent.declare(**{iter_varname: []}) self.iter_varnames.append(iter_varname) lexer.skip(['whitespace', 'newline']) @@ -120,7 +120,7 @@ def __init__(self, lexer: Lexer, parser: 'Parser', parent: Scope) -> None: lexer.next_if('whitespace') else: iter_varname = 'counter' - parent.define(**{iter_varname: []}) + parent.declare(**{iter_varname: []}) self.iter_varnames = [iter_varname] # Check if this is a "while" loop. diff --git a/src/Exscript/interpreter/scope.py b/src/Exscript/interpreter/scope.py index 6d078d05..f6bc2288 100644 --- a/src/Exscript/interpreter/scope.py +++ b/src/Exscript/interpreter/scope.py @@ -100,6 +100,14 @@ def define_object(self, **kwargs: Any) -> None: """ self.variables.update(kwargs) + def declare(self, **kwargs: Any) -> None: + """ + Defines the given variables like define(), except the ones that are + defined already, which keep their values. Statements declare their + variables with it when the template is compiled. + """ + self.define(**{key: value for key, value in kwargs.items() if not self.is_defined(key)}) + def is_defined(self, name: str) -> int: """ Returns 1 if a variable with the given name is defined in this scope diff --git a/tests/Exscript/interpreter/AssignTest.py b/tests/Exscript/interpreter/AssignTest.py index 4b94e0e9..07536945 100644 --- a/tests/Exscript/interpreter/AssignTest.py +++ b/tests/Exscript/interpreter/AssignTest.py @@ -24,6 +24,13 @@ def testAssignmentsInBlocksAreVisibleOutside(self): self.evaluate('{if 1}{x = "eth0"}{end}\nshow interface $x\n') self.assertSent('show interface eth0') + def testVariableOfTheCallerBeforeTheAssignment(self): + variables = self.evaluate('show $x\n{x = "new"}\n', x='hi') + self.assertSent('show hi') + self.assertEqual(variables['x'], ['new']) + variables = self.evaluate('{if 0}{x = "new"}{end}', x='hi') + self.assertEqual(variables['x'], ['hi']) + def testInternalVariables(self): with self.assertRaises(CompileError) as cm: self.evaluate('{__hostname__ = "r2"}') @@ -66,6 +73,10 @@ def testAppendAList(self): variables = self.evaluate('{x = "a"}{append list.new("b", "c") to x}') self.assertEqual(variables['x'], ['a', 'b', 'c']) + def testAppendToAVariableOfTheCaller(self): + variables = self.evaluate('{append "b" to x}', x=['a']) + self.assertEqual(variables['x'], ['a', 'b']) + def testAppendInALoop(self): variables = self.evaluate('{loop from 0 to 3 as i}{append i * 2 to doubles}{end}') self.assertEqual(variables['doubles'], [0, 2, 4]) diff --git a/tests/Exscript/interpreter/ExtractTest.py b/tests/Exscript/interpreter/ExtractTest.py index 140903ea..0512bdd5 100644 --- a/tests/Exscript/interpreter/ExtractTest.py +++ b/tests/Exscript/interpreter/ExtractTest.py @@ -26,6 +26,10 @@ def testIntoAppendsTheValues(self): variables = self.evaluate(text) self.assertEqual(variables['files'], ['old', 'file1', 'file2', 'file1']) + def testIntoAVariableOfTheCaller(self): + variables = self.evaluate('{extract /(a)/ into x from "a"}', x=['b']) + self.assertEqual(variables['x'], ['b', 'a']) + def testSeveralExtracts(self): text = ( 'dir {\n' diff --git a/tests/Exscript/interpreter/LoopTest.py b/tests/Exscript/interpreter/LoopTest.py index 8b8f147d..4de03b68 100644 --- a/tests/Exscript/interpreter/LoopTest.py +++ b/tests/Exscript/interpreter/LoopTest.py @@ -59,6 +59,12 @@ def testLoopFromToWithoutNumbers(self): self.evaluate('{loop from 3 to 3}\nls\n{end}\n{loop from 3 to 1}\nls\n{end}') self.assertSent() + def testVariablesOfTheCallerWithoutIterations(self): + text = '{loop from 0 to 0}{end}{loop list.new() as x}{end}{loop from 1 to 1 as y}{end}' + variables = self.evaluate(text, counter='c', x='hi', y=['a', 'b']) + self.assertEqual((variables['counter'], variables['x']), (['c'], ['hi'])) + self.assertEqual(variables['y'], ['a', 'b']) + def testLoopWhile(self): variables = self.evaluate('{n = 0}{loop while n lt 3}{n = n + 1}\nls $n\n{end}') self.assertSent('ls 1', 'ls 2', 'ls 3') diff --git a/tests/Exscript/interpreter/ParserTest.py b/tests/Exscript/interpreter/ParserTest.py index 7b7ea8bd..0478a8e3 100644 --- a/tests/Exscript/interpreter/ParserTest.py +++ b/tests/Exscript/interpreter/ParserTest.py @@ -165,6 +165,11 @@ def testDefineObject(self): self.assertEqual(self.child.variables, {'b': 'x'}) self.assertEqual(self.root.variables, {'a': [1]}) + def testDeclare(self): + self.child.declare(a=['x'], b='y', **{'f.g': 'function'}) + self.assertEqual(self.root.variables, {'a': [1], 'b': ['y'], 'f.g': 'function'}) + self.assertEqual(self.child.variables, {}) + def testGet(self): self.child.define_object(b='x') self.assertEqual(self.child.get('a'), [1]) From e3e03fe43edd0c20b4aeebf082bee3ccff8093c3 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 10 Nov 2026 17:18:27 +0100 Subject: [PATCH 097/250] Evaluate operators of the same priority from left to right They were evaluated from right to left. Expression.prioritize() now builds the tree with the shunting-yard algorithm and the priorities of docs/templates.rst, which gives % the priority of * and /. The attributes that only the old algorithm used are removed. --- CHANGES.md | 6 + docs/migration.rst | 6 + docs/templates.rst | 8 +- src/Exscript/interpreter/expression.py | 140 +++++++------------ src/Exscript/interpreter/term.py | 10 -- tests/Exscript/interpreter/ExpressionTest.py | 18 +++ 6 files changed, 91 insertions(+), 97 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 2fcb57c3..4a433c00 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -380,6 +380,12 @@ marked with **BREAKING**; a migration guide will accompany the release. with `x='hi'`, `show $x\n{x = "new"}` sent `show None`, `{append "b" to x}` with `x=['a']` gave `['b']`, and `extract ... into x` did not append to the list of the caller. +- **BREAKING - Operators of the Same Priority:** Operators of the same priority are + evaluated from left to right: `10 - 2 - 3` is 5, `100 / 10 * 2` is 20, and + `0 and 1 or 1` is 1. Before, they were evaluated from right to left, so these + expressions gave 11, 5 and 0, and `20 / 4 / 5` failed with a `ZeroDivisionError`. + As documented, `and` and `or` have the same priority, so `1 or 0 and 0` is 0. `%`, + which the documentation did not list, has the priority of `*` and `/`. ### Standard Library diff --git a/docs/migration.rst b/docs/migration.rst index fa2d4274..4d3b57c1 100644 --- a/docs/migration.rst +++ b/docs/migration.rst @@ -113,6 +113,12 @@ Variables of the caller variables when the template was compiled, e.g. ``append`` and ``extract ... into`` started with an empty list. +Operators + Operators of the same priority are evaluated from left to right, e.g. + ``10 - 2 - 3`` is 5 and ``0 and 1 or 1`` is 1. 2.x evaluated them from + right to left. ``and`` and ``or`` have the same priority, so + ``1 or 0 and 0`` is 0. + Command line tool ----------------- diff --git a/docs/templates.rst b/docs/templates.rst index 96415f56..6aa07b7c 100644 --- a/docs/templates.rst +++ b/docs/templates.rst @@ -105,7 +105,10 @@ value of this particular expression is *true*. In Exscript, expressions are used in many places, such as if-conditions or variable assignments. The following operators may be -used in an expression. +used in an expression. Operators with a lower priority number are +evaluated first, and operators of the same priority from left to right. +For example, ``2 + 3 * 4`` is 14, ``10 - 2 - 3`` is 5, and +``1 or 0 and 0`` is 0. Priority 1 Operators ~~~~~~~~~~~~~~~~~~~~ @@ -114,6 +117,9 @@ Priority 1 Operators #. ``/`` divides the operators (numerically). +#. ``%`` returns the remainder of the division of the operators + (numerically). + Priority 2 Operators ~~~~~~~~~~~~~~~~~~~~ diff --git a/src/Exscript/interpreter/expression.py b/src/Exscript/interpreter/expression.py index 7072f542..c3aecea6 100644 --- a/src/Exscript/interpreter/expression.py +++ b/src/Exscript/interpreter/expression.py @@ -37,18 +37,14 @@ class ExpressionNode(Token): """ - A node of an expression: a term, optionally followed by an operator and - the rest of the expression, which is the right-hand node. An expression - that starts with "not" has no term. + A node of an expression. The constructor parses a chain of nodes: a + term, optionally followed by an operator and the rest of the expression + as the right-hand node; a node that starts with "not" has no term. + Expression.prioritize() turns the chain into a tree, in which the + left-hand and right-hand nodes are the operands of the operator. """ - def __init__( - self, - lexer: Lexer, - parser: 'Parser', - parent: Scope, - parent_node: 'ExpressionNode | None' = None, - ) -> None: + def __init__(self, lexer: Lexer, parser: 'Parser', parent: Scope) -> None: """ Constructor. Parses a term and, if an operator follows, the operator and the rest of the expression. @@ -56,8 +52,6 @@ def __init__( :param lexer: The lexer that reads the template. :param parser: The parser of the template. :param parent: The scope that contains the expression. - :param parent_node: The node whose right-hand node this node is, or - None. """ # Skip whitespace before initializing the token to make sure that self.start # points to the beginning of the expression (which makes for prettier error @@ -69,7 +63,6 @@ def __init__( self.rgt: Exscript.interpreter.term.Term | ExpressionNode | None = None self.op = None self.op_type = None - self.parent_node = parent_node # The "not" operator requires special treatment because it is # positioned left of the term. @@ -102,21 +95,19 @@ def __init__( lexer.syntax_error(msg, self) # Expect the second term. - self.rgt = ExpressionNode(lexer, parser, parent, self) + self.rgt = ExpressionNode(lexer, parser, parent) # End of statement. self.mark_end(rstrip=True) def priority(self) -> int: """ Returns the priority of the operator of the node, from 1 for "and" - and "or" to 7 for "%", or 8 if the node has no operator. Raises an - Exception for an unknown operator. + and "or" to 6 for "*", "/" and "%", or 8 if the node has no operator. + Raises an Exception for an unknown operator. """ if self.op is None: return 8 - elif self.op_type == 'arithmetic_operator' and self.op == '%': - return 7 - elif self.op_type == 'arithmetic_operator' and self.op == '*': + elif self.op_type == 'arithmetic_operator' and self.op in ('*', '%'): return 6 elif self.op_type == 'regex_delimiter': return 6 @@ -252,7 +243,7 @@ class Expression(Token): def __init__(self, lexer: Lexer, parser: 'Parser', parent: Scope) -> None: """ - Constructor. Parses the expression and reorders its nodes by the + Constructor. Parses the expression and orders its nodes by the priority of the operators. :param lexer: The lexer that reads the template. @@ -261,78 +252,55 @@ def __init__(self, lexer: Lexer, parser: 'Parser', parent: Scope) -> None: """ Token.__init__(self, self.__class__.__name__, lexer, parser, parent) - # Parse the expression. - self.root = ExpressionNode(lexer, parser, parent) - - # Reorder the tree according to the operator priorities. - self.prioritize(self.root) + # Parse the expression, and order the nodes by the operator priorities. + self.root = self.prioritize(ExpressionNode(lexer, parser, parent)) # End of statement. self.mark_end(rstrip=True) - # start is an ExpressionNode, a Term or None. It is Any because the code - # walks Terms like nodes (a Term has lft, rgt and op set to None, and the - # priority 6) and relies on Terms never reaching the reparenting part, - # which needs attributes that only nodes have. - def prioritize(self, start: Any, prio: int = 1) -> None: + def prioritize( + self, start: ExpressionNode + ) -> 'Exscript.interpreter.term.Term | ExpressionNode': """ - Reorders the tree from the given node on, so that operators with a - higher priority are evaluated first: the first node with the given - priority becomes the parent of the preceding nodes with a higher - priority. Continues recursively with the priorities up to 5. - - :param start: The node at which the reordering starts. - :param prio: The priority of the operators to move up. + Turns the chain of nodes that the parser created into a tree, in + which the operators with a higher priority take their operands + first, and operators with the same priority from left to right. A + "not" takes the expression up to the next operator whose priority is + not higher than its own. The marks of each node span its operands. + + :param start: The first node of the chain. + :return: The root of the tree. """ - # print("Prioritizing from", start.op, "with prio", prio, (start.lft,) - # start.rgt) - if prio == 6: - return - - # Search the tree for the first node that has at least the - # given prio. - root = start - while root is not None and root.priority() <= prio: - root = root.rgt - - # If no such node exists, search for weaker priorities. - if root is None: - self.prioritize(start, prio + 1) - return - - # Check if there is any child node that has a better priority. - previous = root - current = root.rgt - while current is not None and current.priority() != prio: - previous = current - current = current.rgt - - # If none was found, continue with sorting weaker priorities. - if current is None: - self.prioritize(start, prio + 1) - return - - # So we found a node that has a better prio than it's parent. - # Reparent the expressions. - # print("Prio of", root.op, 'is higher than', current.op) - previous.rgt = current.lft - current.lft = root - - # Change the pointer of the parent of the root node. - # If this was the root of the entire tree we need to change that as - # well. - if root.parent_node is None: - self.root = current - elif root.parent_node.lft == root: - root.parent_node.lft = current - elif root.parent_node.rgt == root: - root.parent_node.rgt = current - - root.parent_node = current - - # Go ahead prioritizing the children. - self.prioritize(current.lft, prio + 1) - self.prioritize(current.rgt, prio) + operands: list[Exscript.interpreter.term.Term | ExpressionNode] = [] + operators: list[ExpressionNode] = [] + + def reduce() -> None: + # Gives the last operator its operands, and makes it an operand. + node = operators.pop() + node.rgt = operands.pop() + if node.op != 'not': + node.lft = operands.pop() + node.mark_start(node.lft.start) + node.mark_end(node.rgt.end) + operands.append(node) + + node: ExpressionNode | None = start + while node is not None: + # In the chain, the right-hand node of a node is the next node. + following = cast('ExpressionNode | None', node.rgt) + if node.op is None: + operands.append(node) + else: + if node.op != 'not': + # The parser gives the nodes with a binary operator a term. + operands.append(cast(Exscript.interpreter.term.Term, node.lft)) + while operators and operators[-1].priority() >= node.priority(): + reduce() + operators.append(node) + node = following + while operators: + reduce() + return operands[0] # Any: see ExpressionNode.value(). def value(self, context: Scope) -> Any: diff --git a/src/Exscript/interpreter/term.py b/src/Exscript/interpreter/term.py index 52d582c9..b2d65ea7 100644 --- a/src/Exscript/interpreter/term.py +++ b/src/Exscript/interpreter/term.py @@ -56,9 +56,6 @@ def __init__(self, lexer: Lexer, parser: 'Parser', parent: Scope) -> None: """ Token.__init__(self, self.__class__.__name__, lexer, parser, parent) self.term: Variable | FunctionCall | String | Number | None = None - self.lft = None - self.rgt = None - self.op = None # Expect a term. ttype, token = lexer.token() @@ -92,13 +89,6 @@ def __init__(self, lexer: Lexer, parser: 'Parser', parent: Scope) -> None: # End of statement. self.mark_end(rstrip=True) - def priority(self) -> int: - """ - Returns 6, the priority that Expression.prioritize() assumes for a - term. - """ - return 6 - # Any: the value of a variable or the result of a function. def value(self, context: Scope) -> Any: """ diff --git a/tests/Exscript/interpreter/ExpressionTest.py b/tests/Exscript/interpreter/ExpressionTest.py index ebec68d1..9be7848a 100644 --- a/tests/Exscript/interpreter/ExpressionTest.py +++ b/tests/Exscript/interpreter/ExpressionTest.py @@ -115,6 +115,24 @@ def testComparisonBeforeAndOr(self): text = '{a = 1 is 1 and 2 is 3}{b = 1 is 2 or 2 is 2}' self.assertValues(text, a=[False], b=[True]) + def testOperatorsOfTheSamePriorityFromLeftToRight(self): + text = ( + '{a = 10 - 2 - 3}{b = 1 - 2 + 3}{c = 100 / 10 * 2}{d = 20 / 4 / 5}' + '{e = 7 % 3 * 2}{f = 2 * 7 % 4}{g = 2 is 2 is 1}' + '{h = 0 and 1 or 1}{i = 1 or 0 and 0}{j = not 0 and 0 or 1}' + '{k = 1 + 2 * 3 - 4 / 2 . "x" is "5x"}' + ) + self.assertValues( + text, a=[5], b=[2], c=[20], d=[1], e=[2], f=[2], g=[True], h=[1], i=[0], j=[1], k=[True] + ) + + def testErrorInAnOperandWithOperators(self): + text = '{x = "a" matches "b" . "c" and 1}' + with self.assertRaises(ExecuteError) as cm: + self.evaluate(text) + message = 'Right hand operator is not a regular expression in undefined:1' + self.assertError(cm.exception, [text], ' ' * 17 + '^' * 9, message) + def testExpressionOnSeveralLines(self): text = '{a = 1 is 1\n and 2\n is 2\n}' self.assertValues(text, a=[True]) From 838faeded20f15bf99b68804dbf7723eafaa3b50 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 10 Nov 2026 17:20:20 +0100 Subject: [PATCH 098/250] Evaluate the right operand of and and or only if needed Both operands were always evaluated, although docs/templates.rst says otherwise, so a function on the right, e.g. one that sends a command, was always called. --- CHANGES.md | 5 +++++ docs/migration.rst | 5 +++++ src/Exscript/interpreter/expression.py | 8 ++++++-- tests/Exscript/interpreter/ExpressionTest.py | 16 ++++++++++++++++ 4 files changed, 32 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 4a433c00..68d2e947 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -386,6 +386,11 @@ marked with **BREAKING**; a migration guide will accompany the release. expressions gave 11, 5 and 0, and `20 / 4 / 5` failed with a `ZeroDivisionError`. As documented, `and` and `or` have the same priority, so `1 or 0 and 0` is 0. `%`, which the documentation did not list, has the priority of `*` and `/`. +- **BREAKING - `and` and `or` Evaluate the Right Operand Only If Needed:** As + documented, `and` does not evaluate its right operand if the left one is false, and + `or` not if the left one is true; e.g. `{x = 0 and list.get(list.new(), 1)}` sets `x` + to 0. Before, both operands were always evaluated, so the right one could raise an + error or call a function, e.g. one that sends a command. ### Standard Library diff --git a/docs/migration.rst b/docs/migration.rst index 4d3b57c1..d444a88e 100644 --- a/docs/migration.rst +++ b/docs/migration.rst @@ -119,6 +119,11 @@ Operators right to left. ``and`` and ``or`` have the same priority, so ``1 or 0 and 0`` is 0. +``and``, ``or`` + Evaluate the right operand only if the left one does not decide the + result, as documented. 2.x always evaluated both, e.g. called the + functions in both. + Command line tool ----------------- diff --git a/src/Exscript/interpreter/expression.py b/src/Exscript/interpreter/expression.py index c3aecea6..2ac24118 100644 --- a/src/Exscript/interpreter/expression.py +++ b/src/Exscript/interpreter/expression.py @@ -128,8 +128,9 @@ def priority(self) -> int: def value(self, context: Scope) -> Any: """ Evaluates the node. A list operand stands for its first item, or '' - if it is empty. Arithmetic operators raise an ExecuteError if an - operand is not a number. + if it is empty. "and" and "or" evaluate the right operand only if + the left one does not decide the result. Arithmetic operators raise + an ExecuteError if an operand is not a number. :param context: The scope in which the template runs. :return: The value of the term if the node has no operator, or a @@ -153,6 +154,9 @@ def value(self, context: Scope) -> Any: lft: Any = lft_lst[0] else: lft = '' + # The left operand of "and" and "or" may decide the result alone. + if (self.op == 'and' and not lft) or (self.op == 'or' and lft): + return [lft] rgt_lst = cast(Token, self.rgt).value(context) if type(rgt_lst) == type([]): if len(rgt_lst) > 0: diff --git a/tests/Exscript/interpreter/ExpressionTest.py b/tests/Exscript/interpreter/ExpressionTest.py index 9be7848a..fc6c21bd 100644 --- a/tests/Exscript/interpreter/ExpressionTest.py +++ b/tests/Exscript/interpreter/ExpressionTest.py @@ -94,6 +94,22 @@ def testLogicalOperators(self): text = '{a = 1 and 0}{b = 0 or 1}{c = not 0}{d = not 1}{e = 1 and 1 and 0}' self.assertValues(text, a=[0], b=[1], c=[True], d=[False], e=[0]) + def testAndOrEvaluateTheRightOperandOnlyIfNeeded(self): + calls = [] + + def record(scope, value): + calls.append(value[0]) + return value + + text = ( + '{a = 0 and record(1)}{b = "x" or record(2)}{c = 1 and record(3)}' + '{d = "" or record(4)}{e = 0 and list.get(list.new(), 1)}' + ) + variables = self.evaluate(text, record=record) + actual = [variables[name] for name in 'abcde'] + self.assertEqual(actual, [[0], ['x'], [3], [4], [0]]) + self.assertEqual(calls, [3, 4]) + def testMultiplicationBeforeAddition(self): text = '{a = 2 + 3 * 4}{b = 2 * 3 + 4}{c = 20 - 12 / 4}{d = 20 / 4 - 1}{e = 1 + 7 % 3}' self.assertValues(text, a=[14], b=[10], c=[17], d=[4], e=[2]) From 8bc5dfa1fb61d2d82467186a2d9fa49e4abe35de Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 10 Nov 2026 17:26:29 +0100 Subject: [PATCH 099/250] Drop lines with only whitespace from the next command Their whitespace was put in front of the next command, and textwrap.dedent() removes only spaces and tabs, so other whitespace such as a no-break space was sent. Empty the command buffer of the template at every newline. --- CHANGES.md | 3 +++ src/Exscript/interpreter/template.py | 5 +++-- tests/Exscript/interpreter/CommandTest.py | 5 +++++ 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 68d2e947..114f9d21 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -391,6 +391,9 @@ marked with **BREAKING**; a migration guide will accompany the release. `or` not if the left one is true; e.g. `{x = 0 and list.get(list.new(), 1)}` sets `x` to 0. Before, both operands were always evaluated, so the right one could raise an error or call a function, e.g. one that sends a command. +- **BREAKING - Lines With Only Whitespace:** A line with only whitespace is ignored, + also if the whitespace is not a space or a tab, e.g. a no-break space. Before, such + whitespace was sent at the start of the command in the next line. ### Standard Library diff --git a/src/Exscript/interpreter/template.py b/src/Exscript/interpreter/template.py index 3698c16f..8e831977 100644 --- a/src/Exscript/interpreter/template.py +++ b/src/Exscript/interpreter/template.py @@ -136,8 +136,9 @@ def __init__( elif lexer.next_if('newline'): if buffer.strip() != '': self.add(Execute(lexer, parser, self, buffer, escape_offset)) - buffer = '' - escape_offset = 0 + # A line with only whitespace is dropped. + buffer = '' + escape_offset = 0 else: ttype, token = lexer.token() self.mark_start() diff --git a/tests/Exscript/interpreter/CommandTest.py b/tests/Exscript/interpreter/CommandTest.py index 7c9760fe..a43000db 100644 --- a/tests/Exscript/interpreter/CommandTest.py +++ b/tests/Exscript/interpreter/CommandTest.py @@ -22,6 +22,11 @@ def testBlankLinesAndIndentationAreNotSent(self): self.evaluate('\n show version\n\t\n ls\n\n') self.assertSent('show version', 'ls') + def testLinesWithOnlyWhitespace(self): + # textwrap.dedent() removes only spaces and tabs from the commands. + self.evaluate('\u00a0\nls\n \u3000\nshow version\n') + self.assertSent('ls', 'show version') + def testCommentsAreNotSent(self): text = ( '# a comment\n' From a3ed5c539a8d38444933d49945be25301f85c63b Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 10 Nov 2026 17:27:10 +0100 Subject: [PATCH 100/250] Raise ExecuteErrors for runtime errors in expressions and statements Errors such as a division by zero, a non-numeric operand of / or gt, or an empty list as a condition leaked as exceptions of Python without a position. Check the operands and the bounds of range loops, match the items of extract as text, and take an empty list as false. --- CHANGES.md | 11 ++++++++ docs/templates.rst | 3 +++ src/Exscript/interpreter/expression.py | 26 +++++++++++++----- src/Exscript/interpreter/extract.py | 5 ++-- src/Exscript/interpreter/fail.py | 9 ++++--- src/Exscript/interpreter/ifcondition.py | 6 ++--- src/Exscript/interpreter/loop.py | 23 +++++++++------- tests/Exscript/interpreter/ExpressionTest.py | 25 +++++++++++++++-- tests/Exscript/interpreter/ExtractTest.py | 6 +++++ tests/Exscript/interpreter/FailTest.py | 7 +++++ tests/Exscript/interpreter/IfConditionTest.py | 5 ++++ tests/Exscript/interpreter/LoopTest.py | 27 +++++++++++++++++++ 12 files changed, 127 insertions(+), 26 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 114f9d21..03747861 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -394,6 +394,17 @@ marked with **BREAKING**; a migration guide will accompany the release. - **BREAKING - Lines With Only Whitespace:** A line with only whitespace is ignored, also if the whitespace is not a space or a tab, e.g. a no-break space. Before, such whitespace was sent at the start of the command in the next line. +- **Errors at Runtime:** A division by zero (`1 / 0`, `1 % 0`), an operand of `/`, + `ge`, `gt`, `le` or `lt` that is not a number (`"a" / 2`), an operand of an + arithmetic operator without a value, such as a group that `extract` did not match, + and a bound of `loop from ... to ...` that is not a number raise an `ExecuteError` + that shows their position. Before, they raised a `ZeroDivisionError`, `ValueError` + or `TypeError` of Python. Other such exceptions are gone: an empty list is false as + the condition of `if`, `fail` and `loop`, and as the operand of `not`, and `fail` + with an empty list has an empty message (before: `IndexError`); `extract ... from` + matches numbers as text, as `matches` does, e.g. in `{extract /(\d)/ as d from 42}` + (before: `TypeError`); and the bounds of `loop from ... to ...` may be strings of + digits, like the operands of arithmetic operators. ### Standard Library diff --git a/docs/templates.rst b/docs/templates.rst index 6aa07b7c..131689ee 100644 --- a/docs/templates.rst +++ b/docs/templates.rst @@ -297,6 +297,9 @@ The following Exscript template executes the ``ls`` command only if ls {end} +A condition that is an empty list is false, so ``{if not found}`` works +as well. + if ... else ... end ^^^^^^^^^^^^^^^^^^^ diff --git a/src/Exscript/interpreter/expression.py b/src/Exscript/interpreter/expression.py index 2ac24118..748bc233 100644 --- a/src/Exscript/interpreter/expression.py +++ b/src/Exscript/interpreter/expression.py @@ -129,8 +129,9 @@ def value(self, context: Scope) -> Any: """ Evaluates the node. A list operand stands for its first item, or '' if it is empty. "and" and "or" evaluate the right operand only if - the left one does not decide the result. Arithmetic operators raise - an ExecuteError if an operand is not a number. + the left one does not decide the result. Arithmetic operators and + ge, gt, le and lt raise an ExecuteError if an operand is not a + number, and / and % if the right operand is 0. :param context: The scope in which the template runs. :return: The value of the term if the node has no operator, or a @@ -142,7 +143,8 @@ def value(self, context: Scope) -> Any: if self.op is None: return cast(Token, self.lft).value(context) elif self.op == 'not': - return [not cast(Token, self.rgt).value(context)[0]] + rgt_lst = cast(Token, self.rgt).value(context) + return [not (rgt_lst[0] if len(rgt_lst) > 0 else '')] # There are only two types of values: Regular expressions and lists. # We also have to make sure that empty lists do not cause an error. @@ -164,16 +166,18 @@ def value(self, context: Scope) -> Any: else: rgt = '' - if self.op_type == 'arithmetic_operator' and self.op != '.': + if self.op in ('*', '/', '%', '+', '-', 'ge', 'gt', 'le', 'lt'): error = 'Operand for %s is not a number' % (self.op) try: lft = int(lft) - except ValueError: + except (TypeError, ValueError): self.lexer.runtime_error(error, self.lft) try: rgt = int(rgt) - except ValueError: + except (TypeError, ValueError): self.lexer.runtime_error(error, self.rgt) + if self.op in ('/', '%') and rgt == 0: + self.lexer.runtime_error('Division by zero', self.rgt) # Two-term expressions. if self.op == 'is': @@ -317,6 +321,16 @@ def value(self, context: Scope) -> Any: """ return self.root.value(context) + def is_true(self, context: Scope) -> bool: + """ + Evaluates the expression as a condition, which is true if the first + item of the value is true. An empty list is false. + + :param context: The scope in which the template runs. + """ + values = self.value(context) + return len(values) > 0 and bool(values[0]) + def dump(self, indent: int = 0) -> None: """ Prints the expression tree to stdout. diff --git a/src/Exscript/interpreter/extract.py b/src/Exscript/interpreter/extract.py index 2c8c8392..2a8e1376 100644 --- a/src/Exscript/interpreter/extract.py +++ b/src/Exscript/interpreter/extract.py @@ -158,9 +158,10 @@ def extract(self, context: Scope) -> None: # print("Buffer contains", buffer) # Walk through all lines, matching each one against the regular - # expression. + # expression. Items that are not strings, such as numbers, are matched + # as text. for line in buffer: - match = self.regex.value(context).search(line) + match = self.regex.value(context).search(str(line)) if match is None: continue diff --git a/src/Exscript/interpreter/fail.py b/src/Exscript/interpreter/fail.py index a0a919d0..505f0664 100644 --- a/src/Exscript/interpreter/fail.py +++ b/src/Exscript/interpreter/fail.py @@ -76,14 +76,15 @@ def __init__(self, lexer: Lexer, parser: 'Parser', parent: Scope) -> None: def value(self, context: Scope) -> int: """ - Raises a FailException with the first item of the message, if the - condition is true or not given. + Raises a FailException with the first item of the message, or '' if + the message is an empty list, if the condition is true or not given. :param context: The scope in which the template runs. :return: 1 if the condition is false. """ - if self.expression is None or self.expression.value(context)[0]: - raise FailException(self.msg.value(context)[0]) + if self.expression is None or self.expression.is_true(context): + msg = self.msg.value(context) + raise FailException(msg[0] if len(msg) > 0 else '') return 1 def dump(self, indent: int = 0) -> None: diff --git a/src/Exscript/interpreter/ifcondition.py b/src/Exscript/interpreter/ifcondition.py index 2e13ae19..049a7314 100644 --- a/src/Exscript/interpreter/ifcondition.py +++ b/src/Exscript/interpreter/ifcondition.py @@ -88,13 +88,13 @@ def __init__(self, lexer: Lexer, parser: 'Parser', parent: Scope) -> None: def value(self, context: Scope) -> int: """ - Evaluates the if block if the first item of the condition is true, - and the else block otherwise. + Evaluates the if block if the condition is true, and the else block + otherwise. :param context: The scope in which the template runs. :return: 1. """ - if self.expression.value(context)[0]: + if self.expression.is_true(context): self.if_block.value(context) elif self.else_block is not None: self.else_block.value(context) diff --git a/src/Exscript/interpreter/loop.py b/src/Exscript/interpreter/loop.py index 8ff4d394..399ab1dd 100644 --- a/src/Exscript/interpreter/loop.py +++ b/src/Exscript/interpreter/loop.py @@ -146,8 +146,8 @@ def value(self, context: Scope) -> int: """ Runs the loop. A range loop counts from the first number up to, but not including, the second. A while or until condition is checked - before each iteration. Raises an ExecuteError if the lists have - different lengths. + before each iteration. Raises an ExecuteError if a bound of a range + loop is not a number, or if the lists have different lengths. :param context: The scope in which the template runs. :return: 1. @@ -156,22 +156,27 @@ def value(self, context: Scope) -> int: # If this is a "while" loop, iterate as long as the condition is # True. if self.during is not None: - while self.during.value(context)[0]: + while self.during.is_true(context): self.block.value(context) return 1 # If this is an "until" loop, iterate until the condition is True. if self.until is not None: - while not self.until.value(context)[0]: + while not self.until.is_true(context): self.block.value(context) return 1 # Retrieve the lists from the list terms. if self.thefrom: - start = self.thefrom.value(context)[0] + bounds = [] # The constructor sets theto together with thefrom. - stop = cast(Expression, self.theto).value(context)[0] - lists = [list(range(start, stop))] + for bound in self.thefrom, cast(Expression, self.theto): + values = bound.value(context) + try: + bounds.append(int(values[0] if len(values) > 0 else '')) + except (TypeError, ValueError): + self.lexer.runtime_error('Loop bound is not a number', bound.root) + lists = [list(range(*bounds))] else: lists = [var.value(context) for var in self.list_variables] varnames = self.iter_varnames @@ -188,9 +193,9 @@ def value(self, context: Scope) -> int: for thelist in lists: self.block.define(**{varnames[f]: [thelist[i]]}) f += 1 - if self.until is not None and self.until.value(context)[0]: + if self.until is not None and self.until.is_true(context): break - if self.during is not None and not self.during.value(context)[0]: + if self.during is not None and not self.during.is_true(context): break self.block.value(context) return 1 diff --git a/tests/Exscript/interpreter/ExpressionTest.py b/tests/Exscript/interpreter/ExpressionTest.py index fc6c21bd..a8585ba4 100644 --- a/tests/Exscript/interpreter/ExpressionTest.py +++ b/tests/Exscript/interpreter/ExpressionTest.py @@ -54,6 +54,11 @@ def testOperandIsNotANumber(self): ('{x = 1 + "b"}', ' ^^^'), ('{x = "a" - 1}', ' ^^^'), ('{x = 5 % "b"}', ' ^^^'), + ('{x = "a" / 2}', ' ^^^'), + ('{x = "a" ge 1}', ' ^^^'), + ('{x = 1 gt "b"}', ' ' * 10 + '^^^'), + ('{x = "a" le 1}', ' ^^^'), + ('{x = 1 lt "b"}', ' ' * 10 + '^^^'), ): with self.subTest(text=text): with self.assertRaises(ExecuteError) as cm: @@ -62,6 +67,22 @@ def testOperandIsNotANumber(self): message = 'Operand for %s is not a number in undefined:1' % op self.assertError(cm.exception, [text], marker, message) + def testOperandWithoutValue(self): + # A group that did not match extracts None. + text = '{extract /(x)?(y)/ as a, b from "y"}{z = a + 1}' + with self.assertRaises(ExecuteError) as cm: + self.evaluate(text) + message = 'Operand for + is not a number in undefined:1' + self.assertError(cm.exception, [text], ' ' * 41 + '^', message) + + def testDivisionByZero(self): + for text in '{x = 1 / 0}', '{x = 7 % 0}': + with self.subTest(text=text): + with self.assertRaises(ExecuteError) as cm: + self.evaluate(text) + message = 'Division by zero in undefined:1' + self.assertError(cm.exception, [text], ' ' * 9 + '^', message) + def testEquality(self): text = '{a = 1 is 1}{b = "a" is "b"}{c = 1 is not 2}{d = "a" is not "a"}' self.assertValues(text, a=[True], b=[False], c=[True], d=[False]) @@ -156,9 +177,9 @@ def testExpressionOnSeveralLines(self): def testListOperands(self): text = ( '{l = list.new("a", "b")}{e = list.new()}' - '{a = l . "c"}{b = e . "c"}{c = e is ""}{d = "c" . e}' + '{a = l . "c"}{b = e . "c"}{c = e is ""}{d = "c" . e}{f = not e}' ) - self.assertValues(text, a=['ac'], b=['c'], c=[True], d=['c']) + self.assertValues(text, a=['ac'], b=['c'], c=[True], d=['c'], f=[True]) def testRegularExpressionTerm(self): variables = self.evaluate('{r = /a.c/i}') diff --git a/tests/Exscript/interpreter/ExtractTest.py b/tests/Exscript/interpreter/ExtractTest.py index 0512bdd5..f1e22101 100644 --- a/tests/Exscript/interpreter/ExtractTest.py +++ b/tests/Exscript/interpreter/ExtractTest.py @@ -55,6 +55,12 @@ def testExtractFromAFunctionResult(self): variables = self.evaluate('{extract /^(\\w)/ as first from string.split("ab,cd", ",")}') self.assertEqual(variables['first'], ['a', 'c']) + def testExtractFromNumbers(self): + variables = self.evaluate( + '{extract /^(\\d)/ as a from 42}{extract /(\\d)$/ as b from l}', l=[17, 'x3'] + ) + self.assertEqual((variables['a'], variables['b']), (['4'], ['7', '3'])) + def testNoMatch(self): variables = self.evaluate('ls{extract /(nomatch)/ as found}\n') self.assertEqual(variables['found'], []) diff --git a/tests/Exscript/interpreter/FailTest.py b/tests/Exscript/interpreter/FailTest.py index 26ca93eb..a453fe6c 100644 --- a/tests/Exscript/interpreter/FailTest.py +++ b/tests/Exscript/interpreter/FailTest.py @@ -33,6 +33,13 @@ def testMessageWithVariables(self): self.evaluate('{fail "$x on $__hostname__" if x is "bad"}', x='bad') self.assertEqual(str(cm.exception), 'bad on r1') + def testEmptyList(self): + self.evaluate('{e = list.new()}{fail "x" if e}\nls\n') + self.assertSent('ls') + with self.assertRaises(FailException) as cm: + self.evaluate('{fail list.new()}') + self.assertEqual(str(cm.exception), '') + def testInABlock(self): with self.assertRaises(FailException) as cm: self.evaluate('{loop from 0 to 5 as i}\nls $i\n{fail "stop at " . i if i is 2}{end}') diff --git a/tests/Exscript/interpreter/IfConditionTest.py b/tests/Exscript/interpreter/IfConditionTest.py index 3a6965bb..5bd7935d 100644 --- a/tests/Exscript/interpreter/IfConditionTest.py +++ b/tests/Exscript/interpreter/IfConditionTest.py @@ -40,6 +40,11 @@ def testElseIfWithoutElse(self): self.evaluate('{if 0}\nshow one\n{else if 0}\nshow two\n{end}\nls\n') self.assertSent('ls') + def testEmptyListIsFalse(self): + text = '{e = list.new()}{if e}\nshow one\n{else if not e}\nshow two\n{end}' + self.evaluate(text) + self.assertSent('show two') + def testOnOneLine(self): self.evaluate('{if 1}ls{end}\n{if 0}show version{else}show other{end}\n') self.assertSent('ls', 'show other') diff --git a/tests/Exscript/interpreter/LoopTest.py b/tests/Exscript/interpreter/LoopTest.py index 4de03b68..8b31221f 100644 --- a/tests/Exscript/interpreter/LoopTest.py +++ b/tests/Exscript/interpreter/LoopTest.py @@ -65,6 +65,33 @@ def testVariablesOfTheCallerWithoutIterations(self): self.assertEqual((variables['counter'], variables['x']), (['c'], ['hi'])) self.assertEqual(variables['y'], ['a', 'b']) + def testLoopFromToWithDigitStrings(self): + variables = self.evaluate('{loop from "1" to n as i}{append i to l}{end}', n='3') + self.assertEqual(variables['l'], [1, 2]) + + def testBoundIsNotANumber(self): + for text, marker in ( + ('{loop from "a" to 3}', ' ' * 11 + '^^^'), + ('{loop from 1 to e}', ' ' * 16 + '^'), + ('{loop from 1 to e . "x"}', ' ' * 16 + '^' * 7), + ): + with self.subTest(text=text): + with self.assertRaises(ExecuteError) as cm: + self.evaluate(text + '\nls\n{end}', e=[]) + message = 'Loop bound is not a number in undefined:1' + self.assertError(cm.exception, [text], marker, message) + self.assertSent() + + def testConditionsOnAnEmptyList(self): + text = ( + '{e = list.new()}{n = 0}' + '{loop while e}{n = n + 1}{end}' + '{loop until e}{n = n + 1}{if n is 2}{append 1 to e}{end}{end}' + '{loop list.new("a", "b") as x while list.new()}{n = n + 10}{end}' + '{loop list.new("a", "b") as x until list.new()}{n = n + 100}{end}' + ) + self.assertEqual(self.evaluate(text)['n'], [202]) + def testLoopWhile(self): variables = self.evaluate('{n = 0}{loop while n lt 3}{n = n + 1}\nls $n\n{end}') self.assertSent('ls 1', 'ls 2', 'ls 3') From 3903c2403654a505e7b12b4bf1b55e9196e81a3d Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 10 Nov 2026 17:28:57 +0100 Subject: [PATCH 101/250] Raise an ExecuteError for regular expressions as operands A regular expression is only allowed as the right operand of matches. As another operand, it raised an UnboundLocalError or a TypeError. --- CHANGES.md | 4 ++++ src/Exscript/interpreter/expression.py | 18 +++++++++++++++++- tests/Exscript/interpreter/ExpressionTest.py | 17 +++++++++++++++++ 3 files changed, 38 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 03747861..20ba544d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -405,6 +405,10 @@ marked with **BREAKING**; a migration guide will accompany the release. matches numbers as text, as `matches` does, e.g. in `{extract /(\d)/ as d from 42}` (before: `TypeError`); and the bounds of `loop from ... to ...` may be strings of digits, like the operands of arithmetic operators. +- **Regular Expressions as Operands:** A regular expression that is not the right + operand of `matches`, e.g. in `/a/ matches /a/`, `/a/ . "b"`, `"a" is /a/` or + `not /a/`, raises an `ExecuteError` that shows its position. Before, it raised an + `UnboundLocalError` or a `TypeError` of Python. ### Standard Library diff --git a/src/Exscript/interpreter/expression.py b/src/Exscript/interpreter/expression.py index 748bc233..a0f13247 100644 --- a/src/Exscript/interpreter/expression.py +++ b/src/Exscript/interpreter/expression.py @@ -24,6 +24,7 @@ Expressions of the template language. """ +import re from typing import TYPE_CHECKING, Any, cast import Exscript.interpreter.term @@ -124,6 +125,16 @@ def priority(self) -> int: else: raise Exception('Invalid operator: ' + repr(self.op)) + # Any: the value of an operand. + def _reject_regex(self, value: Any, operand: Token) -> None: + """ + Raises an ExecuteError at the given operand if its given value is a + regular expression. Only "matches" accepts one, as its right operand. + """ + if isinstance(value, re.Pattern): + msg = "A regular expression is only allowed as the right operand of 'matches'" + self.lexer.runtime_error(msg, operand) + # Any: the value of a term, or a list with the result of the operator. def value(self, context: Scope) -> Any: """ @@ -131,7 +142,8 @@ def value(self, context: Scope) -> Any: if it is empty. "and" and "or" evaluate the right operand only if the left one does not decide the result. Arithmetic operators and ge, gt, le and lt raise an ExecuteError if an operand is not a - number, and / and % if the right operand is 0. + number, and / and % if the right operand is 0. A regular expression + raises an ExecuteError unless it is the right operand of "matches". :param context: The scope in which the template runs. :return: The value of the term if the node has no operator, or a @@ -144,6 +156,7 @@ def value(self, context: Scope) -> Any: return cast(Token, self.lft).value(context) elif self.op == 'not': rgt_lst = cast(Token, self.rgt).value(context) + self._reject_regex(rgt_lst, cast(Token, self.rgt)) return [not (rgt_lst[0] if len(rgt_lst) > 0 else '')] # There are only two types of values: Regular expressions and lists. @@ -151,6 +164,7 @@ def value(self, context: Scope) -> Any: # The first item of a list can be a string, a number or anything else # that a function returned, hence Any. lft_lst = cast(Token, self.lft).value(context) + self._reject_regex(lft_lst, cast(Token, self.lft)) if type(lft_lst) == type([]): if len(lft_lst) > 0: lft: Any = lft_lst[0] @@ -160,6 +174,8 @@ def value(self, context: Scope) -> Any: if (self.op == 'and' and not lft) or (self.op == 'or' and lft): return [lft] rgt_lst = cast(Token, self.rgt).value(context) + if self.op != 'matches': + self._reject_regex(rgt_lst, cast(Token, self.rgt)) if type(rgt_lst) == type([]): if len(rgt_lst) > 0: rgt: Any = rgt_lst[0] diff --git a/tests/Exscript/interpreter/ExpressionTest.py b/tests/Exscript/interpreter/ExpressionTest.py index a8585ba4..38215f15 100644 --- a/tests/Exscript/interpreter/ExpressionTest.py +++ b/tests/Exscript/interpreter/ExpressionTest.py @@ -111,6 +111,23 @@ def testMatchesRequiresARegularExpression(self): message = 'Right hand operator is not a regular expression in undefined:1' self.assertError(cm.exception, ['{x = "abc" matches "b"}'], ' ' * 19 + '^^^', message) + def testRegularExpressionAsAnotherOperand(self): + for text, marker in ( + ('{x = /a/ matches /a/}', ' ^^^'), + ('{x = /a/i . "b"}', ' ^^^^'), + ('{x = "a" is /a/}', ' ' * 12 + '^^^'), + ('{x = not /a/}', ' ' * 9 + '^^^'), + ('{x = 1 and /a/}', ' ' * 11 + '^^^'), + ): + with self.subTest(text=text): + with self.assertRaises(ExecuteError) as cm: + self.evaluate(text) + message = ( + "A regular expression is only allowed as the right operand of 'matches'" + ' in undefined:1' + ) + self.assertError(cm.exception, [text], marker, message) + def testLogicalOperators(self): text = '{a = 1 and 0}{b = 0 or 1}{c = not 0}{d = not 1}{e = 1 and 1 and 0}' self.assertValues(text, a=[0], b=[1], c=[True], d=[False], e=[0]) From ae87fa60dfc96b8bb73e588240a45e110295f025 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 10 Nov 2026 17:32:35 +0100 Subject: [PATCH 102/250] Point error positions behind escape sequences to the variable Errors about a variable behind an escape sequence pointed too far to the left, because the position was taken from the text after the escapes were evaluated. String now records each escape sequence and maps the positions of variables back to the template. --- CHANGES.md | 5 +++++ src/Exscript/interpreter/execute.py | 24 ++++++++++++++++------- src/Exscript/interpreter/string.py | 24 ++++++++++++++++++++--- src/Exscript/interpreter/template.py | 21 +++++++++++--------- tests/Exscript/interpreter/CommandTest.py | 13 ++++++++++++ tests/Exscript/interpreter/StringTest.py | 15 ++++++++++++++ 6 files changed, 83 insertions(+), 19 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 20ba544d..d2866fbd 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -409,6 +409,11 @@ marked with **BREAKING**; a migration guide will accompany the release. operand of `matches`, e.g. in `/a/ matches /a/`, `/a/ . "b"`, `"a" is /a/` or `not /a/`, raises an `ExecuteError` that shows its position. Before, it raised an `UnboundLocalError` or a `TypeError` of Python. +- **Error Positions After Escape Sequences:** An error about a variable in a command, + a string or a regular expression points to the variable also behind escape + sequences, e.g. to `$nothing` in `show \x41 $nothing`. Before, it pointed too far to + the left, by the number of characters by which the escape sequences are longer than + their text. ### Standard Library diff --git a/src/Exscript/interpreter/execute.py b/src/Exscript/interpreter/execute.py index 45de6f25..c5ec4166 100644 --- a/src/Exscript/interpreter/execute.py +++ b/src/Exscript/interpreter/execute.py @@ -26,6 +26,7 @@ import re import textwrap +from collections.abc import Sequence from typing import TYPE_CHECKING, cast from ..parselib import Lexer, Token @@ -54,7 +55,12 @@ class Execute(String): """ def __init__( - self, lexer: Lexer, parser: 'Parser', parent: Scope, command: str, escape_offset: int = 0 + self, + lexer: Lexer, + parser: 'Parser', + parent: Scope, + command: str, + escapes: Sequence[tuple[int, int]] = (), ) -> None: """ Constructor. Checks that the variables in the command are defined, @@ -68,9 +74,10 @@ def __init__( :param parser: The parser of the template. :param parent: The scope that contains the command. :param command: The command as read from the template. - :param escape_offset: The number of characters by which the - evaluation of escape sequences shortened the command, for the - position of the command in error messages. + :param escapes: The escape sequences in the command that are longer + than their text, as pairs of the position behind the text in the + command and the number of additional characters, for the + positions in error messages. """ Token.__init__(self, self.__class__.__name__, lexer, parser, parent) self.delimiter = '' # Command strings are not delimited in Execute tokens. @@ -82,11 +89,14 @@ def __init__( # (newline, open curly bracket, RPC tag). We want the debugger point to the # beginning of the command. The offset is expected to be the length of the # additionally consumed command delimiter before entering this Execute token. - # Any offset caused by prior escape evaluation is also taken into account. + # The escape sequences of the left-stripped command, which are longer in the + # template than in the command, are also taken into account. + stripped = len(command) - len(command.lstrip()) command = command.lstrip() + self.escapes = [(end - stripped, extra) for end, extra in escapes if end > stripped] delimiter_offset = lexer.current_char - lexer.last_char - self.mark_start(self.start - len(command) - delimiter_offset - escape_offset) - self.mark_end(distance=len(command.strip()) + escape_offset) + self.mark_start(self.start - delimiter_offset - self._template_offset(len(command))) + self.mark_end(distance=self._template_offset(len(command.rstrip()))) # Make sure that any variables specified in the command are declared. # Once more we use the left-stripped command to align with the start mark. diff --git a/src/Exscript/interpreter/string.py b/src/Exscript/interpreter/string.py index 0cbf7d55..aeb981dc 100644 --- a/src/Exscript/interpreter/string.py +++ b/src/Exscript/interpreter/string.py @@ -93,6 +93,10 @@ def __init__( # Begin parsing the string. self.string = '' + # The escape sequences that are longer than their text, as pairs of the + # position behind the text in the string and the number of additional + # characters, for the positions in error messages. + self.escapes: list[tuple[int, int]] = [] lexer.expect(self, purpose + '_delimiter') while True: if lexer.current_is(purpose + '_data'): @@ -109,7 +113,11 @@ def __init__( 'escaped_data', ] ): - self.string += self.eval_escape(*lexer.token()) + ttype, token = lexer.token() + text = self.eval_escape(ttype, token) + self.string += text + if len(token) > len(text): + self.escapes.append((len(self.string), len(token) - len(text))) next(lexer) elif lexer.next_if(purpose + '_delimiter'): break @@ -166,6 +174,14 @@ def eval_escape(self, ttype: str, token: str) -> str: # Simply unescape anything else. return token[1:] + def _template_offset(self, pos: int) -> int: + """ + Returns the offset in the template of the given position of the + string, from the start of the string: the position plus the + additional characters of the escape sequences before it. + """ + return pos + sum(extra for end, extra in self.escapes if end <= pos) + def _variable_error(self, field: str, field_start: int, msg: str) -> None: """ Moves the marks of the token to the given variable reference, and @@ -176,8 +192,10 @@ def _variable_error(self, field: str, field_start: int, msg: str) -> None: :param msg: The error message. """ # Narrow down the marks on the variable that caused the error. - self.mark_start(self.start + len(self.delimiter) + field_start) - self.mark_end(distance=len(field)) + start = self.start + len(self.delimiter) + end = start + self._template_offset(field_start + len(field)) + self.mark_start(start + self._template_offset(field_start)) + self.mark_end(end) self.lexer.runtime_error(msg, self) # Tokens that include variables in a string may use this callback to diff --git a/src/Exscript/interpreter/template.py b/src/Exscript/interpreter/template.py index 8e831977..2b3b8cbf 100644 --- a/src/Exscript/interpreter/template.py +++ b/src/Exscript/interpreter/template.py @@ -86,7 +86,9 @@ def __init__( Scope.__init__(self, self.__class__.__name__, lexer, parser, parent, **kwargs) lexer.set_grammar(grammar_c) buffer = '' - escape_offset = 0 + # The escape sequences in the buffer that are longer than their text, + # see Execute. + escapes: list[tuple[int, int]] = [] while True: lexer.skip(['line_comment']) if self.exit_requested or lexer.current_is('EOF'): @@ -95,13 +97,13 @@ def __init__( # Execute expects the lexer to have consumed the delimiter # behind the command; at the end of the input, it is empty. lexer.forward(0) - self.add(Execute(lexer, parser, self, buffer, escape_offset)) + self.add(Execute(lexer, parser, self, buffer, escapes)) break elif lexer.current_is('open_curly_bracket'): if buffer.strip() != '': - self.add(Execute(lexer, parser, self, buffer, escape_offset)) + self.add(Execute(lexer, parser, self, buffer, escapes)) buffer = '' - escape_offset = 0 + escapes = [] if isinstance(parent, Code): # Need to consume the bracket if the Code token is not called # due to nesting in IfCondition, Loop and Try tokens. @@ -110,9 +112,9 @@ def __init__( self.add(Code(lexer, parser, self)) elif lexer.current_is('open_rpc_tag'): if buffer.strip() != '': - self.add(Execute(lexer, parser, self, buffer, escape_offset)) + self.add(Execute(lexer, parser, self, buffer, escapes)) buffer = '' - escape_offset = 0 + escapes = [] self.add(RPCBody(lexer, parser, self)) elif lexer.current_is(['raw_data_before_tag', 'raw_data']): buffer += lexer.token()[1] @@ -131,14 +133,15 @@ def __init__( ttype, token = lexer.token() result = self.eval_escape(ttype, token) buffer += result - escape_offset += len(token) - len(result) + if len(token) > len(result): + escapes.append((len(buffer), len(token) - len(result))) next(lexer) elif lexer.next_if('newline'): if buffer.strip() != '': - self.add(Execute(lexer, parser, self, buffer, escape_offset)) + self.add(Execute(lexer, parser, self, buffer, escapes)) # A line with only whitespace is dropped. buffer = '' - escape_offset = 0 + escapes = [] else: ttype, token = lexer.token() self.mark_start() diff --git a/tests/Exscript/interpreter/CommandTest.py b/tests/Exscript/interpreter/CommandTest.py index a43000db..ac6b8949 100644 --- a/tests/Exscript/interpreter/CommandTest.py +++ b/tests/Exscript/interpreter/CommandTest.py @@ -91,6 +91,19 @@ def testUndefinedVariableAfterACodeBlock(self): message = "Undefined variable 'nothing' in undefined:2" self.assertError(cm.exception, [' show $x $nothing'], ' ^^^^^^^^', message) + def testUndefinedVariableAfterEscapeSequences(self): + for text, lineno, line, column in ( + ('ls\nshow \\x41 $nothing\n', 2, 'show \\x41 $nothing', 10), + ('ls\nshow \\x41 $nothing', 2, 'show \\x41 $nothing', 10), + ('ls\n\\tshow \\x41 $nothing\n', 2, '\\tshow \\x41 $nothing', 12), + ('\\t\nshow \\x41\\x42 $nothing\n', 2, 'show \\x41\\x42 $nothing', 14), + ): + with self.subTest(text=text): + with self.assertRaises(ExecuteError) as cm: + self.evaluate(text) + message = "Undefined variable 'nothing' in undefined:%d" % lineno + self.assertError(cm.exception, [line], ' ' * column + '^' * 8, message) + def testInvalidVariableName(self): with self.assertRaises(ExecuteError) as cm: self.evaluate('show $1x\n') diff --git a/tests/Exscript/interpreter/StringTest.py b/tests/Exscript/interpreter/StringTest.py index 39d4d25e..a7bc0671 100644 --- a/tests/Exscript/interpreter/StringTest.py +++ b/tests/Exscript/interpreter/StringTest.py @@ -54,6 +54,14 @@ def testUndefinedVariable(self): self.assertError(cm.exception, ['{x = "a $nothing b"}'], ' ^^^^^^^^', message) self.assertSent() + def testUndefinedVariableAfterEscapeSequences(self): + for text, column in ('{x = "\\x41\\t$nothing"}', 12), ('{x = "\\x41$nothing\\t"}', 10): + with self.subTest(text=text): + with self.assertRaises(ExecuteError) as cm: + self.evaluate(text) + message = "Undefined variable 'nothing' in undefined:1" + self.assertError(cm.exception, [text], ' ' * column + '^' * 8, message) + def testInvalidVariableName(self): with self.assertRaises(ExecuteError) as cm: self.evaluate('{x = "$1abc"}') @@ -103,6 +111,13 @@ def testVariables(self): self.assertMatches('"abc" matches /^a$p$/', 0, p='b') self.assertMatches('"a.c" matches /^a$p$/', 1, p='[.]c') + def testUndefinedVariableAfterAnEscapedSlash(self): + text = '{x = "a" matches /\\/$nothing/}' + with self.assertRaises(ExecuteError) as cm: + self.evaluate(text) + message = "Undefined variable 'nothing' in undefined:1" + self.assertError(cm.exception, [text], ' ' * 20 + '^' * 8, message) + def testInvalidModifier(self): with self.assertRaises(CompileError) as cm: self.evaluate('{x = /abc/x}') From 32e690f4eb86c7f657eb4ddbe3307e82c7b04b79 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 10 Nov 2026 17:33:55 +0100 Subject: [PATCH 103/250] Detect functions in strings with callable() The substitution of variables checked for the attribute func_name, which Python 3 functions do not have, so "$f" with a function f gave its repr instead of the intended ExecuteError. --- CHANGES.md | 4 ++++ src/Exscript/interpreter/string.py | 5 +++-- tests/Exscript/interpreter/CommandTest.py | 7 +++++++ tests/Exscript/interpreter/StringTest.py | 6 ++++++ 4 files changed, 20 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index d2866fbd..0db2d3ea 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -414,6 +414,10 @@ marked with **BREAKING**; a migration guide will accompany the release. sequences, e.g. to `$nothing` in `show \x41 $nothing`. Before, it pointed too far to the left, by the number of characters by which the escape sequences are longer than their text. +- **BREAKING - Functions in Strings:** A command or a string that refers to a + function like to a variable, e.g. `"$f"` with a function `f` that the caller + passes, raises an `ExecuteError`, as the interpreter intends. Before, it contained + the text of the function object, such as ``. ### Standard Library diff --git a/src/Exscript/interpreter/string.py b/src/Exscript/interpreter/string.py index aeb981dc..88b476f8 100644 --- a/src/Exscript/interpreter/string.py +++ b/src/Exscript/interpreter/string.py @@ -206,7 +206,8 @@ def variable_sub_cb(self, match: re.Match[str]) -> str: varstring_re refers to, for re.sub(); a list is joined with newlines. An escaped reference yields the reference without the backslash, and a lone dollar sign stays as is. Raises an ExecuteError if the name is - not a valid variable name, or if the variable is not defined. + not a valid variable name, or if the variable is not defined or is a + function. """ field_start = match.start(0) field = match.group(0) @@ -227,7 +228,7 @@ def variable_sub_cb(self, match: re.Match[str]) -> str: if value is None: msg = 'Undefined variable %s' % repr(varname) self._variable_error(field, field_start, msg) - elif hasattr(value, 'func_name'): + elif callable(value): msg = '%s is a function, not a variable name' % repr(varname) self._variable_error(field, field_start, msg) elif isinstance(value, list): diff --git a/tests/Exscript/interpreter/CommandTest.py b/tests/Exscript/interpreter/CommandTest.py index ac6b8949..efc167b3 100644 --- a/tests/Exscript/interpreter/CommandTest.py +++ b/tests/Exscript/interpreter/CommandTest.py @@ -110,6 +110,13 @@ def testInvalidVariableName(self): message = "'1x' is not a variable name in undefined:1" self.assertError(cm.exception, ['show $1x'], ' ^^^', message) + def testFunction(self): + with self.assertRaises(ExecuteError) as cm: + self.evaluate('show $f\n', f=lambda scope: ['b']) + message = "'f' is a function, not a variable name in undefined:1" + self.assertError(cm.exception, ['show $f'], ' ^^', message) + self.assertSent() + def testEscapeSequences(self): self.evaluate('show \\x41\\101\\u00e9\\U0001F600 \\t \\e \\a \\0 \\v \\f \\b\n') self.assertSent('show AA\u00e9\U0001f600 \t \x1b \x07 \x00 \x0b \x0c \x08') diff --git a/tests/Exscript/interpreter/StringTest.py b/tests/Exscript/interpreter/StringTest.py index a7bc0671..f753b5c7 100644 --- a/tests/Exscript/interpreter/StringTest.py +++ b/tests/Exscript/interpreter/StringTest.py @@ -68,6 +68,12 @@ def testInvalidVariableName(self): message = "'1abc' is not a variable name in undefined:1" self.assertError(cm.exception, ['{x = "$1abc"}'], ' ^^^^^', message) + def testFunction(self): + with self.assertRaises(ExecuteError) as cm: + self.evaluate('{x = "a $f"}', f=lambda scope: ['b']) + message = "'f' is a function, not a variable name in undefined:1" + self.assertError(cm.exception, ['{x = "a $f"}'], ' ^^', message) + def testLineBreak(self): with self.assertRaises(CompileError) as cm: self.evaluate('{x = "a\nb"}') From 632fab4caea6235ab1d53220f21cf11a4feec282 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 10 Nov 2026 17:35:00 +0100 Subject: [PATCH 104/250] Raise a CompileError for long unmatched input in the lexer If no expression matched, Lexer.match() showed the input up to the last blank within 50 characters, and raised an AttributeError if there was none. Show the next 50 characters then. The grammars of the template language were not affected. --- CHANGES.md | 5 +++++ src/Exscript/parselib/lexer.py | 12 +++++++----- tests/Exscript/LexerTest.py | 11 +++++++++-- 3 files changed, 21 insertions(+), 7 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 0db2d3ea..14c627df 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -418,6 +418,11 @@ marked with **BREAKING**; a migration guide will accompany the release. function like to a variable, e.g. `"$f"` with a function `f` that the caller passes, raises an `ExecuteError`, as the interpreter intends. Before, it contained the text of the function object, such as ``. +- **Syntax Errors of Custom Grammars:** If no expression of the grammar of + `Exscript.parselib.Lexer` matches, and the next 50 characters contain no blank and + not the end of the input, the lexer raises a `CompileError` that shows these 50 + characters. Before, it raised an `AttributeError`. The grammars of the template + language were not affected. ### Standard Library diff --git a/src/Exscript/parselib/lexer.py b/src/Exscript/parselib/lexer.py index e95be81a..b159a34e 100644 --- a/src/Exscript/parselib/lexer.py +++ b/src/Exscript/parselib/lexer.py @@ -105,12 +105,14 @@ def match(self) -> None: # sender object with marks to reuse the existing error handling code. sender = SimpleNamespace(start=self.current_char, end=self.current_char + 1) # Match 1..50 chars (eager) up to the last space, newline or EOF to - # prevent cutoffs in the middle of a word/line. + # prevent cutoffs in the middle of a word/line. Without a blank or the + # end of the input within 50 characters, show all 50. regex = re.compile(r'.{1,50}(?=[ \t\r\n]|$)', flags=re.S) - # The cast assumes a match. The pattern fails without a blank or the end - # of the input within 50 characters; the grammars of the interpreter fail - # only at a backslash before a line break or the end of the input. - context = cast(re.Match[str], regex.match(self.input, sender.start)).group(0) + match = regex.match(self.input, sender.start) + if match is None: + context = self.input[sender.start : sender.start + 50] + else: + context = match.group(0) error = 'Invalid syntax: %s' % repr(context) self.syntax_error(error, sender) diff --git a/tests/Exscript/LexerTest.py b/tests/Exscript/LexerTest.py index 985e05c7..58ccb1af 100644 --- a/tests/Exscript/LexerTest.py +++ b/tests/Exscript/LexerTest.py @@ -125,6 +125,13 @@ def testMatchShowsWholeWords(self): context = re.search(r"Invalid syntax: '(.*)'", str(cm.exception)).group(1) self.assertEqual(context, '#' + ' '.join(['word'] * 10)) + def testMatchWithoutABlank(self): + # Without a blank or the end of the input within 50 characters. + with self.assertRaises(CompileError) as cm: + Lexer(Tokens).parse('ab ' + '#' * 60) + message = "\nab %s\n ^\nInvalid syntax: '%s' in None:1" % ('#' * 60, '#' * 50) + self.assertEqual(str(cm.exception), message) + def testGetCurrentLineNumber(self): lexer = lexer_for('ab\ncd ef\n') self.assertEqual(lexer.get_current_line_number(), 1) @@ -285,9 +292,9 @@ def testTokensCoverTheInput(self, text): self.assertTrue(first != second or first == 'newline', tokens) # match() shows the input up to 50 characters behind an error, up to a - # blank or the end of the input; shorter inputs always have such an end. + # blank or the end of the input if there is one within them. @settings(max_examples=200, deadline=None) - @given(st.text(alphabet='ab1 \n#', max_size=40)) + @given(st.text(alphabet='ab1 \n#', max_size=120)) def testInvalidSyntaxShowsThePosition(self, text): if '#' not in text: Lexer(Tokens).parse(text) From 5427bf40e3f1208700fc729042e637078c50f0c7 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 10 Nov 2026 17:36:58 +0100 Subject: [PATCH 105/250] Raise the protocol errors of template functions unchanged FunctionCall turned every exception of a function into an ExecuteError, also a ProtocolException, so try did not ignore a command that the device rejected in connection.execline(). Other exceptions still raise an ExecuteError. --- CHANGES.md | 7 +++++++ docs/migration.rst | 4 ++++ docs/templates.rst | 6 ++++++ src/Exscript/interpreter/functioncall.py | 7 ++++++- tests/Exscript/interpreter/FailTest.py | 4 ++++ tests/Exscript/stdlib/connectionTest.py | 6 ++++++ 6 files changed, 33 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 14c627df..6e23db3e 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -423,6 +423,13 @@ marked with **BREAKING**; a migration guide will accompany the release. not the end of the input, the lexer raises a `CompileError` that shows these 50 characters. Before, it raised an `AttributeError`. The grammars of the template language were not affected. +- **BREAKING - Errors of the Device in Functions:** If a function of a template raises + a `ProtocolException`, e.g. `connection.execline()` an `InvalidCommandException` for + a command that the device rejects, the exception stops the template unchanged, like + the error of a command, and `try` ignores it. Other exceptions of functions raise an + `ExecuteError` that shows the position of the call. Before, all of them raised an + `ExecuteError`, which `try` does not ignore; Exscript 2.6 raised all of them + unchanged. ### Standard Library diff --git a/docs/migration.rst b/docs/migration.rst index d444a88e..8f31fb4f 100644 --- a/docs/migration.rst +++ b/docs/migration.rst @@ -217,3 +217,7 @@ their users should know: new names; see the :doc:`changelog`. - ``Exscript.protocols.telnetlib`` lost the methods that the Telnet adapter does not use, e.g. ``read_until()`` and ``expect()``. +- A function of a template that raises a ``ProtocolException``, e.g. + ``connection.execline()`` for a command that the device rejects, stops + the template with that exception instead of an ``ExecuteError``, as in + 2.x, and ``try`` ignores it. diff --git a/docs/templates.rst b/docs/templates.rst index 131689ee..6235fdd6 100644 --- a/docs/templates.rst +++ b/docs/templates.rst @@ -485,6 +485,12 @@ example, the following will also work:: {end} show ip int brief +The functions of the ``connection`` module that send commands, such as +``connection.execline()``, report errors in the same way, so a try block +ignores their errors, too:: + + {try}{connection.execline("ls -l")}{end} + Templates And Jinja2 -------------------- diff --git a/src/Exscript/interpreter/functioncall.py b/src/Exscript/interpreter/functioncall.py index 46abb1ef..dbc38d94 100644 --- a/src/Exscript/interpreter/functioncall.py +++ b/src/Exscript/interpreter/functioncall.py @@ -29,6 +29,7 @@ import Exscript.interpreter.expression from ..parselib import Lexer, Token +from ..protocols.exception import ProtocolException from .exception import PermissionError from .scope import Scope @@ -103,7 +104,8 @@ def value(self, context: Scope) -> Any: """ Calls the function with the scope and the values of the arguments. Raises an ExecuteError if the function is not defined, or if it - raises an exception. + raises an exception. A ProtocolException, such as an error of the + remote host, is raised unchanged, as for a command. :param context: The scope in which the template runs. :return: The result of the function. @@ -116,6 +118,9 @@ def value(self, context: Scope) -> Any: self.lexer.runtime_error('Undefined function %s' % repr(self.funcname), self) try: result = function(self.parent, *argument_values) + except ProtocolException: + # Stops the template like the error of a command, unless "try" ignores it. + raise except Exception as e: self.mark_end(distance=len(cast(str, self.funcname))) self.lexer.runtime_error( diff --git a/tests/Exscript/interpreter/FailTest.py b/tests/Exscript/interpreter/FailTest.py index a453fe6c..7530a3e9 100644 --- a/tests/Exscript/interpreter/FailTest.py +++ b/tests/Exscript/interpreter/FailTest.py @@ -101,6 +101,10 @@ def testOtherErrorsAreNotIgnored(self): self.evaluate('{try}{x = list.get(list.new(), 1)}{end}\nls\n') self.assertSent() + def testErrorOfAConnectionFunction(self): + self.evaluate('{try}{connection.execline("bad")}\nshow version\n{end}\nls\n') + self.assertSent('bad', 'ls') + def testErrorsOutsideAreNotIgnored(self): with self.assertRaises(InvalidCommandException): self.evaluate('{try}\nls\n{end}\nbad\nshow version\n') diff --git a/tests/Exscript/stdlib/connectionTest.py b/tests/Exscript/stdlib/connectionTest.py index 51794ca5..ac913ba3 100644 --- a/tests/Exscript/stdlib/connectionTest.py +++ b/tests/Exscript/stdlib/connectionTest.py @@ -136,6 +136,12 @@ def testExecline(self): variables['__response__'], ['Version 1.0', 'Uptime 5 days', 'file1', 'file2'] ) + def testExeclineWithAnErrorOfTheDevice(self): + # Like a command, the function stops the template with the error. + with self.assertRaises(InvalidCommandException): + self.evaluate('{x = connection.execline("bad")}\nls\n') + self.assertSent('bad') + def testExpect(self): variables = self.evaluate( '{connection.sendline("reload")}{connection.expect(/\\[confirm\\]/)}' From d01043b44f24bab7da8c26f8c8bd9cc0452ee66c Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 10 Nov 2026 18:13:40 +0100 Subject: [PATCH 106/250] Never turn on paramiko's packet dumps SSH2 turned them on at debug levels above 5, and they log every packet before encryption, with passwords in clear text. The documented debug levels end at 5. --- CHANGES.md | 2 ++ src/Exscript/protocols/ssh2.py | 3 +-- tests/Exscript/protocols/SSH2Test.py | 10 ++++++++++ 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 6e23db3e..89d3a0e2 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -80,6 +80,8 @@ marked with **BREAKING**; a migration guide will accompany the release. `host_key_policy='auto-add'` or `host_key_policy='warn'`. - **Passwords in Debug Logs:** Passwords and S/Key responses sent by the app-level login are logged as ``. Before, debug level 4 logged them in clear text. + Debug levels above 5 no longer turn on paramiko's packet dumps, which showed the + password of an SSH login in clear text. - **BREAKING - `InvalidCommandException` Message:** The message contains only the line that matched an error pattern, for example `Error in the response from the device: % Invalid input detected`. Before, it contained the full response, which diff --git a/src/Exscript/protocols/ssh2.py b/src/Exscript/protocols/ssh2.py index 53d53d08..f6b2f21f 100644 --- a/src/Exscript/protocols/ssh2.py +++ b/src/Exscript/protocols/ssh2.py @@ -206,8 +206,7 @@ def _paramiko_connect(self) -> paramiko.Transport: t.name = cast(str, self.host) # connect() checks that the host is set if self.debug < 5: t.logger.disabled = True - if self.debug > 5: - t.set_hexdump(True) + # No packet dumps (paramiko's set_hexdump()): they show passwords in clear text. # The known keys are loaded before the key exchange, so that the key is # checked before any credentials are sent. diff --git a/tests/Exscript/protocols/SSH2Test.py b/tests/Exscript/protocols/SSH2Test.py index 934d776c..8026f644 100644 --- a/tests/Exscript/protocols/SSH2Test.py +++ b/tests/Exscript/protocols/SSH2Test.py @@ -606,6 +606,16 @@ def testPasswordIsNotLogged(self): with open(logfile, encoding='utf-8') as fp: self.assertNotIn(PASSWORD, fp.read()) + def testNoPacketDumps(self): + # paramiko's packet dumps show passwords in clear text, also above level 5. + self.serve(allowed='password') + for debug in 5, 6, 10: + with self.subTest(debug=debug), self.assertLogs(level=logging.DEBUG) as logs: + conn = self.login(debug=debug) + self.assertFalse(conn._transport.get_hexdump()) + conn.close(force=True) + self.assertEqual([line for line in logs.output if PASSWORD in line], []) + def testKeyboardInteractiveWithAnEmptyRound(self): # Some servers send a round without prompts first. rounds = [[], [('Password: ', False)]] From 184aadd1ae51cb5ffbd72f9629b100cb69974868 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 10 Nov 2026 18:19:46 +0100 Subject: [PATCH 107/250] Raise LoginFailure for failed key logins A key login raised paramiko's exceptions or an OSError instead, so callers had to catch both, and autologin(attempts=...) did not retry key logins. The original error is the cause of the LoginFailure. --- CHANGES.md | 5 +++++ docs/migration.rst | 7 +++++++ src/Exscript/protocols/ssh2.py | 10 ++++++++-- tests/Exscript/protocols/SSH2Test.py | 18 ++++++++++++++---- 4 files changed, 34 insertions(+), 6 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 89d3a0e2..c0480e8e 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -163,6 +163,11 @@ marked with **BREAKING**; a migration guide will accompany the release. (`Account(..., key=...)`) is tried without the password of the account, so such a login fails with a `LoginFailure` that names the required methods. Thanks to u1735067 (pull request #203), whose rewrite of the authentication included this. +- **BREAKING - Key Logins Raise `LoginFailure`:** A key login that fails raises + `LoginFailure`, as a password login does, with the original error as its cause. + Before, it raised paramiko's `AuthenticationException` or `SSHException` when the + server rejected the key, and an `OSError` when the key file could not be read, so + `autologin(attempts=...)` did not retry key logins. - **BREAKING - Terminal Size:** SSH and Telnet sessions get a terminal of 511 columns and 1000 lines; the new options `term_width` and `term_height` (also `Host` options) choose another size. Before, Exscript used the size of the terminal that diff --git a/docs/migration.rst b/docs/migration.rst index 8f31fb4f..bdd3e22f 100644 --- a/docs/migration.rst +++ b/docs/migration.rst @@ -66,6 +66,13 @@ Error messages contains only the line with the error. The full response is still in ``conn.response``. +Key logins + A failed SSH login with a key raises + :class:`~Exscript.protocols.exception.LoginFailure`, like a failed + password login. 2.x raised paramiko's ``AuthenticationException`` or + ``SSHException``, or an ``OSError`` for an unreadable key file; the + original error is now the ``__cause__``. + Responses ``conn.get_response()`` returns the response without the echo of the command, control characters and terminal escape sequences, and with diff --git a/src/Exscript/protocols/ssh2.py b/src/Exscript/protocols/ssh2.py index f6b2f21f..b00d09e3 100644 --- a/src/Exscript/protocols/ssh2.py +++ b/src/Exscript/protocols/ssh2.py @@ -535,7 +535,8 @@ def _protocol_authenticate_by_key(self, user: str, key: PrivateKey) -> None: Authenticates with the user name and the key file of the given key, decrypted with the password of the key, and opens the shell channel. - :raises LoginFailure: If the server requires another authentication + :raises LoginFailure: If the server rejects the key, if the key file + cannot be read, or if the server requires another authentication method after the key, such as a password. """ # Allow multiple key files. @@ -549,7 +550,12 @@ def _protocol_authenticate_by_key(self, user: str, key: PrivateKey) -> None: keys = [] for file in key_file: keys.append((keymap[key.get_type()], file)) - allowed = self._paramiko_auth_key(user, keys, key.get_password()) + try: + allowed = self._paramiko_auth_key(user, keys, key.get_password()) + except (SSHException, OSError) as e: + msg = 'Login with the key failed: %s' % e + self._dbg(1, msg) + raise LoginFailure(msg) from e if allowed: msg = 'Login failed: after the key, the server requires one of: %s' raise LoginFailure(msg % ', '.join(allowed)) diff --git a/tests/Exscript/protocols/SSH2Test.py b/tests/Exscript/protocols/SSH2Test.py index 8026f644..048b74d6 100644 --- a/tests/Exscript/protocols/SSH2Test.py +++ b/tests/Exscript/protocols/SSH2Test.py @@ -45,10 +45,11 @@ def testGetRemoteVersion(self): self.assertEqual(self.protocol.get_remote_version(), sshd.local_version) def testLogin(self): - self.assertRaises(IOError, ProtocolTest.testLogin, self) + # The account of ProtocolTest has a key file that does not exist. + self.assertRaises(LoginFailure, ProtocolTest.testLogin, self) def testAuthenticate(self): - self.assertRaises(IOError, ProtocolTest.testAuthenticate, self) + self.assertRaises(LoginFailure, ProtocolTest.testAuthenticate, self) class KeyServer: @@ -657,18 +658,27 @@ def testRejectedKey(self): # Key logins do not fall back to the password. requests = self.serve(allowed='publickey,password') account = Account('admin', PASSWORD, key=PrivateKey.from_file(keyfile), needs_lock=False) - with self.assertRaises((paramiko.SSHException, LoginFailure)) as cm: + with self.assertRaises(LoginFailure) as cm: self.login(account) + self.assertIsInstance(cm.exception.__cause__, paramiko.AuthenticationException) self.assertNotIn(PASSWORD, str(cm.exception)) public = paramiko.RSAKey(filename=keyfile) self.assertEqual(requests, [('publickey', 'admin', public.get_base64())]) # A key without a file is no key. account = Account('admin', PASSWORD, key=PrivateKey(), needs_lock=False) - with self.assertRaises((paramiko.SSHException, LoginFailure)): + with self.assertRaises(LoginFailure): self.login(account) self.assertEqual(len(requests), 1) + def testUnreadableKeyFile(self): + self.serve(allowed='publickey') + key = PrivateKey.from_file(keyfile) + key.set_filename(os.path.join(self.home, 'missing')) + with self.assertRaises(LoginFailure) as cm: + self.login(Account('admin', key=key, needs_lock=False)) + self.assertIsInstance(cm.exception.__cause__, FileNotFoundError) + def testAgentKeys(self): accepted, other = paramiko.RSAKey(filename=keyfile), paramiko.RSAKey.generate(1024) paramiko.Agent.return_value.get_keys.return_value = [other, accepted] From 14750ab482fd398c4ed78223407c02c534e1f5f1 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 10 Nov 2026 18:20:45 +0100 Subject: [PATCH 108/250] Log the errors of the logout in close() close() ignored them silently, which is right, as many hosts close the connection on the logout command, but its documentation said that it raises them. Log them at debug level 1 and correct the documentation. --- CHANGES.md | 7 ++++--- src/Exscript/protocols/protocol.py | 11 ++++++----- tests/Exscript/protocols/TelnetTest.py | 13 +++++++++++++ 3 files changed, 23 insertions(+), 8 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index c0480e8e..980a7315 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -178,9 +178,10 @@ marked with **BREAKING**; a migration guide will accompany the release. 1000 lines. `interact()` still uses the size of the local terminal. For the previous behavior without a terminal, pass `term_width=80, term_height=25`. - **Reliable `close()`:** `close()` closes the connection and resets the object even - if the logout fails, and raises the error of the logout afterwards. Errors while - closing, e.g. because the host closed the connection first, are only logged; - before, `close(force=True)` could raise an `EOFError`. + if the logout fails. Errors of the logout and while closing, e.g. because the host + closed the connection first, are only logged (debug level 1); before, errors of + the logout were ignored silently, and `close(force=True)` could raise an + `EOFError`. ### Interactive Shell diff --git a/src/Exscript/protocols/protocol.py b/src/Exscript/protocols/protocol.py index df8fcf2b..840546d2 100644 --- a/src/Exscript/protocols/protocol.py +++ b/src/Exscript/protocols/protocol.py @@ -2236,14 +2236,15 @@ def restore_device_context(self) -> None: def _logout(self) -> None: """ - Logout from the remote host using the driver-specific command. + Logs out from the remote host using the driver-specific command. + Errors are only logged. """ driver = self.get_driver() self._dbg(4, 'Calling %s.logout().' % repr(driver)) try: driver.logout(self) - except Exception: - pass + except Exception as e: # e.g. the host closed the connection first + self._dbg(1, 'Error while logging out: %s' % repr(e)) def _close(self, force: bool = False) -> None: """ @@ -2260,8 +2261,8 @@ def close(self, force: bool = False) -> None: With `force=False` the driver-specific logout() method is called before the connection is closed. Dummy and Netconf adapters have a special static logout sequence. The connection is closed even if the logout - fails, and the error of the logout is raised afterwards; errors while - closing are only logged, because the host may have closed first. + fails. Errors of the logout and while closing are only logged (debug + level 1), because many hosts close the connection after the logout. .. WARNING:: Ensure the remote host has already left any previously used submode diff --git a/tests/Exscript/protocols/TelnetTest.py b/tests/Exscript/protocols/TelnetTest.py index f7ae8b33..5e106070 100644 --- a/tests/Exscript/protocols/TelnetTest.py +++ b/tests/Exscript/protocols/TelnetTest.py @@ -98,6 +98,19 @@ def testErrorWhileClosing(self): conn.close() self.assertFalse(conn.connected) + def testErrorWhileLoggingOut(self): + # E.g. the host closed the connection before the logout; the error is logged. + conn = self.login(debug=1) + with ( + mock.patch.object(conn.get_driver(), 'logout', side_effect=BrokenPipeError), + self.assertLogs('Exscript.protocols.telnet', logging.DEBUG) as logs, + ): + conn.close() + self.assertFalse(conn.connected) + self.assertTrue( + any('Error while logging out: BrokenPipeError' in line for line in logs.output) + ) + def testErrorWhileSending(self): conn = self.login(debug=1) with ( From dd5271c3f95b77d8396d7fec00cb10a1422154ae Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 10 Nov 2026 18:22:48 +0100 Subject: [PATCH 109/250] Start the test servers of ExpectTest without sleeping The servers take port 0 instead of a free port that another process could take in the meantime, and the tests wait for the raw mode of the terminal instead of sleeping. ExpectTest takes 1.3 seconds instead of about 6. --- tests/Exscript/protocols/ExpectTest.py | 32 +++++++++++++++----------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/tests/Exscript/protocols/ExpectTest.py b/tests/Exscript/protocols/ExpectTest.py index 1dda5570..183fbb4d 100644 --- a/tests/Exscript/protocols/ExpectTest.py +++ b/tests/Exscript/protocols/ExpectTest.py @@ -462,16 +462,13 @@ def testEndOfInput(self): def start_daemon(test, daemon_cls, device, **kwargs): - """Starts a test server for the device; it stops when the test ends.""" - with socket.socket() as sock: - sock.bind((HOST, 0)) - port = sock.getsockname()[1] - daemon = daemon_cls(HOST, port, device, **kwargs) - daemon.start() - time.sleep(0.2) + """Starts a test server for the device on a free port; it stops when the test ends.""" + daemon = daemon_cls(HOST, 0, device, **kwargs) + daemon.timeout = 0.05 # stops quickly when the test ends + daemon.start() # listens before it returns test.addCleanup(daemon.join) # cleanups run LIFO: exit() first test.addCleanup(daemon.exit) - return daemon, port + return daemon, daemon.port class ServerExpectTest(unittest.TestCase): @@ -523,18 +520,27 @@ def interact(self, daemon_cls, conn, **kwargs): received = [] conn.data_received_event.connect(received.append) + import tty # POSIX only + master, slave = os.openpty() # interact() closes the slave self.addCleanup(os.close, master) + raw = threading.Event() + setcbreak = tty.setcbreak + + def set_raw_mode(fd, *args): + setcbreak(fd, *args) + raw.set() def type_keys(): - time.sleep(0.3) - os.write(master, 'say grüße\r'.encode()) - time.sleep(0.3) - os.write(master, b'exit\r') + raw.wait(5) # the switch to raw mode discards the keys typed before + os.write(master, 'say grüße\rexit\r'.encode()) typist = threading.Thread(target=type_keys) typist.start() - with mock.patch('sys.stdin', mock.Mock(encoding='utf-8', fileno=lambda: slave)): + with ( + mock.patch('sys.stdin', mock.Mock(encoding='utf-8', fileno=lambda: slave)), + mock.patch('tty.setcbreak', set_raw_mode), + ): conn.interact(handle_window_size=False) typist.join() self.assertIn('grüße!', ''.join(received)) From c0e728e321acac12f93c09133d6baf9aaa0c4155 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 10 Nov 2026 18:26:50 +0100 Subject: [PATCH 110/250] Accept SSH keys that are not stored in a file A key from a secret store had to be written to a file first (upstream issue #209). Add PrivateKey.from_string(), which detects the type of the key like from_file(). The error for an unknown key never contains the key. --- CHANGES.md | 3 ++ docs/security.rst | 4 ++- src/Exscript/key.py | 44 ++++++++++++++++++++++++ src/Exscript/protocols/ssh2.py | 16 +++++++-- tests/Exscript/PrivateKeyTest.py | 51 ++++++++++++++++++++++++++++ tests/Exscript/protocols/SSH2Test.py | 16 +++++++++ 6 files changed, 130 insertions(+), 4 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 980a7315..68ba552b 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -168,6 +168,9 @@ marked with **BREAKING**; a migration guide will accompany the release. Before, it raised paramiko's `AuthenticationException` or `SSHException` when the server rejected the key, and an `OSError` when the key file could not be read, so `autologin(attempts=...)` did not retry key logins. +- **SSH Keys Without Files:** `PrivateKey.from_string()` creates a key from its text, in + the format of a key file, e.g. a key from a secret store, which then never needs to + be written to a file (upstream issue #209). The type is detected unless it is given. - **BREAKING - Terminal Size:** SSH and Telnet sessions get a terminal of 511 columns and 1000 lines; the new options `term_width` and `term_height` (also `Host` options) choose another size. Before, Exscript used the size of the terminal that diff --git a/docs/security.rst b/docs/security.rst index 179ddea5..1444b3cc 100644 --- a/docs/security.rst +++ b/docs/security.rst @@ -89,7 +89,9 @@ contains only the line with the error, not the whole response. Where the passwords come from is up to you. In order of preference: -- SSH keys: ``Account('admin', key=PrivateKey.from_file('/home/admin/.ssh/id_ed25519'))``. +- SSH keys: ``Account('admin', key=PrivateKey.from_file('/home/admin/.ssh/id_ed25519'))``, + or :meth:`~Exscript.PrivateKey.from_string` for a key from a secret store, + which then never needs to be written to a file. - A secret store or the environment of a service, read at run time. - A prompt: :func:`Exscript.util.interact.read_login`. - An account pool file (:func:`Exscript.util.file.get_accounts_from_file`, diff --git a/src/Exscript/key.py b/src/Exscript/key.py index 3178d296..27e36661 100644 --- a/src/Exscript/key.py +++ b/src/Exscript/key.py @@ -24,6 +24,8 @@ Represents a private key. """ +import io + from paramiko import ECDSAKey, Ed25519Key, RSAKey from paramiko.ssh_exception import SSHException @@ -49,6 +51,7 @@ class attribute. raise TypeError('unsupported key type: ' + repr(keytype)) self.keytype = keytype self.filename: str | None = None + self.data: str | None = None self.password: str | None = None @staticmethod @@ -88,6 +91,38 @@ def from_file( key.password = password return key + @staticmethod + def from_string( + data: str, password: str | None = '', keytype: str | None = None + ) -> 'PrivateKey': + """ + Returns a new PrivateKey with the given key, in the format of a key + file, e.g. "-----BEGIN OPENSSH PRIVATE KEY-----...". Use it for a key + that is not stored in a file, e.g. one that comes from a secret + store. If keytype is None, the type is detected. + + :param data: The key, in the format of a key file. + :param password: The key password, or None if the key is not encrypted. + :param keytype: The key type, e.g. 'rsa' or 'ed25519'. + :raises ValueError: If keytype is None and the key is not one of the + supported types, or cannot be decrypted with the password. + :return: The new key. + """ + if keytype is None: + for name, cls in (('rsa', RSAKey), ('ecdsa', ECDSAKey), ('ed25519', Ed25519Key)): + try: + cls.from_private_key(io.StringIO(data), password) + except SSHException: + continue + keytype = name + break + else: + raise ValueError('not a recognized private key') # never shows the key + key = PrivateKey(keytype) + key.data = data + key.password = password + return key + def get_type(self) -> str: """ Returns the type of the key, e.g. 'rsa' or 'ed25519'. @@ -115,6 +150,15 @@ def get_filename(self) -> str | None: """ return self.filename + def get_data(self) -> str | None: + """ + Returns the key, in the format of a key file, if it was given with + from_string(). + + :return: The key, or None if none was set. + """ + return self.data + def set_password(self, password: str | None) -> None: """ Defines the password used for decrypting the key. diff --git a/src/Exscript/protocols/ssh2.py b/src/Exscript/protocols/ssh2.py index b00d09e3..78f62231 100644 --- a/src/Exscript/protocols/ssh2.py +++ b/src/Exscript/protocols/ssh2.py @@ -25,6 +25,7 @@ """ import codecs +import io import logging import os import socket @@ -532,8 +533,9 @@ def _protocol_authenticate(self, user: str, password: str | None) -> None: def _protocol_authenticate_by_key(self, user: str, key: PrivateKey) -> None: """ - Authenticates with the user name and the key file of the given key, - decrypted with the password of the key, and opens the shell channel. + Authenticates with the user name and the given key, decrypted with + the password of the key, and opens the shell channel. The key is the + one given with PrivateKey.from_string(), if any, else the key file. :raises LoginFailure: If the server rejects the key, if the key file cannot be read, or if the server requires another authentication @@ -551,7 +553,15 @@ def _protocol_authenticate_by_key(self, user: str, key: PrivateKey) -> None: for file in key_file: keys.append((keymap[key.get_type()], file)) try: - allowed = self._paramiko_auth_key(user, keys, key.get_password()) + data = key.get_data() + if data is None: + allowed = self._paramiko_auth_key(user, keys, key.get_password()) + else: + pkey = keymap[key.get_type()].from_private_key( + io.StringIO(data), key.get_password() + ) + self._dbg(2, 'Trying key %s' % pkey.fingerprint) + allowed = cast(paramiko.Transport, self._transport).auth_publickey(user, pkey) except (SSHException, OSError) as e: msg = 'Login with the key failed: %s' % e self._dbg(1, msg) diff --git a/tests/Exscript/PrivateKeyTest.py b/tests/Exscript/PrivateKeyTest.py index 8bb3d4a8..bbad3339 100644 --- a/tests/Exscript/PrivateKeyTest.py +++ b/tests/Exscript/PrivateKeyTest.py @@ -1,7 +1,22 @@ +import io +import os import unittest +import paramiko +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric import ed25519 + from Exscript import PrivateKey +KEYFILE = os.path.join(os.path.dirname(__file__), 'protocols', 'id_rsa') + + +def key_text(key, password=None): + """Returns the given paramiko key in the format of a key file.""" + text = io.StringIO() + key.write_private_key(text, password=password) + return text.getvalue() + class PrivateKeyTest(unittest.TestCase): CORRELATE = PrivateKey @@ -24,6 +39,42 @@ def testFromFile(self): self.assertEqual(self.filename, self.key.get_filename()) self.assertEqual(self.password, self.key.get_password()) + def testFromString(self): + ed25519_text = ( + ed25519.Ed25519PrivateKey.generate() + .private_bytes( + serialization.Encoding.PEM, + serialization.PrivateFormat.OpenSSH, + serialization.NoEncryption(), + ) + .decode() + ) + for keytype, text in ( + ('rsa', key_text(paramiko.RSAKey(filename=KEYFILE))), + ('ecdsa', key_text(paramiko.ECDSAKey.generate())), + ('ed25519', ed25519_text), + ): + with self.subTest(keytype=keytype): + key = PrivateKey.from_string(text) + self.assertEqual(key.get_type(), keytype) + self.assertEqual(key.get_data(), text) + self.assertIsNone(key.get_filename()) + + def testFromStringEncrypted(self): + text = key_text(paramiko.ECDSAKey.generate(), password=self.password) + key = PrivateKey.from_string(text, self.password) + self.assertEqual((key.get_type(), key.get_password()), ('ecdsa', self.password)) + self.assertRaises(ValueError, PrivateKey.from_string, text, 'wrong') + + def testFromStringWithType(self): + key = PrivateKey.from_string('not checked', keytype='ed25519') + self.assertEqual((key.get_type(), key.get_data()), ('ed25519', 'not checked')) + + def testFromStringShowsNoKey(self): + with self.assertRaises(ValueError) as cm: + PrivateKey.from_string('-----BEGIN PRIVATE KEY-----\nc2VjcmV0\n') + self.assertNotIn('c2VjcmV0', str(cm.exception)) + def testGetType(self): self.testConstructor() self.assertEqual(self.key.get_type(), 'rsa') diff --git a/tests/Exscript/protocols/SSH2Test.py b/tests/Exscript/protocols/SSH2Test.py index 048b74d6..5b523885 100644 --- a/tests/Exscript/protocols/SSH2Test.py +++ b/tests/Exscript/protocols/SSH2Test.py @@ -654,6 +654,22 @@ def testEncryptedKeyFile(self): self.login(Account('admin', PASSWORD, key=key, needs_lock=False)) self.assertEqual(requests, [('publickey', 'admin', public.get_base64())]) + def testKeyFromString(self): + # E.g. a key from a secret store, which is never written to a file. + public = paramiko.RSAKey(filename=keyfile) + requests = self.serve(allowed='publickey', keys=(public,)) + with open(keyfile, encoding='utf-8') as fp: + key = PrivateKey.from_string(fp.read()) + conn = self.login(Account('admin', key=key, needs_lock=False)) + self.assertTrue(conn.is_protocol_authenticated()) + self.assertEqual(requests, [('publickey', 'admin', public.get_base64())]) + + other = io.StringIO() + paramiko.ECDSAKey.generate().write_private_key(other) + key = PrivateKey.from_string(other.getvalue()) + with self.assertRaises(LoginFailure): + self.login(Account('admin', key=key, needs_lock=False)) + def testRejectedKey(self): # Key logins do not fall back to the password. requests = self.serve(allowed='publickey,password') From 9ad402b3fbe7deacf0f627edb9234b368f8db2d3 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 10 Nov 2026 18:43:05 +0100 Subject: [PATCH 111/250] Document that a connection belongs to one thread The protocol adapters are not safe to share between threads, e.g. a connection that another thread closes while it waits for data may raise an exception. The queue gives each host its own connection. --- docs/platforms.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/platforms.rst b/docs/platforms.rst index f175a55e..46c78236 100644 --- a/docs/platforms.rst +++ b/docs/platforms.rst @@ -49,3 +49,11 @@ uses lxml, which does not declare support for free threading yet: when the first NETCONF session starts, Python enables the global interpreter lock and warns, unless it runs with ``PYTHON_GIL=0``. Exscript loads lxml only for NETCONF, so other protocols keep running without it. + +Threads and connections +----------------------- + +Use each connection from one thread at a time, as the queue does: it +gives each host its own connection. A connection that another thread +closes while it waits for data may raise an exception instead of +reporting the closed connection. From 5353a6487b3bc1e6f1214ef52ec60f25eccb91be Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 10 Nov 2026 18:16:46 +0100 Subject: [PATCH 112/250] Stop offering 'None' in prompt() without a default prompt() asked the history for str(default), so without a default and a stored value, it offered 'None' and returned it for an empty input. Pass the default unchanged, and offer no empty default either. --- CHANGES.md | 6 ++++++ src/Exscript/util/interact.py | 7 ++++--- tests/Exscript/util/interactTest.py | 18 +++++++++++++++++- 3 files changed, 27 insertions(+), 4 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 68ba552b..429ed26e 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -470,6 +470,12 @@ The names below refer to functions callable from templates, not protocol methods exist and asks again, as documented. It passed its check and history one position too early to `prompt()`, so it never checked the name, and failed with a `TypeError` when a history was given. +- **`Exscript.util.interact.prompt()` Without a Default:** Without a default value + and a value in the history, or with an empty one, `prompt()` and `get_filename()` + offer no default, as in `Host: `, and an empty input to `prompt()` returns an empty + string. Before, they showed an empty value as `Host []: `, and without one offered the + string `'None'`, as in `Host [None]: `, which `prompt()` returned and stored for an + empty input. - **`Exscript.util.mail.send()`:** Ends the SMTP session with `QUIT` and closes the connection, also if sending fails; before, it left the connection open. It connects only after the mail and its attachments are ready. diff --git a/src/Exscript/util/interact.py b/src/Exscript/util/interact.py index 81908fb3..11caf864 100644 --- a/src/Exscript/util/interact.py +++ b/src/Exscript/util/interact.py @@ -140,7 +140,8 @@ def prompt( If no default value is given, or if doverh is False, the value from the history is offered as the default; the given default value is only used - if the history has no value. + if the history has no value. If there is neither, or if the value is + empty, no default is offered, and an empty input returns an empty string. The strip argument specifies that the returned value should be stripped of whitespace (default). @@ -188,9 +189,9 @@ def validate(input): if history is None: history = InputHistory() if not doverh or default is None: - default = history.get(key, str(default)) + default = history.get(key, default) while True: - if default is None: + if not default: value = input('%s: ' % message) else: value = input('%s [%s]: ' % (message, default)) or default diff --git a/tests/Exscript/util/interactTest.py b/tests/Exscript/util/interactTest.py index db6fb4bd..3e3dd236 100644 --- a/tests/Exscript/util/interactTest.py +++ b/tests/Exscript/util/interactTest.py @@ -105,6 +105,21 @@ def testPromptHistoryDefaultForEmptyHistory(self): self.assertEqual(value, '22') self.assertEqual(prompts, ['Port [22]: ']) + def testPromptWithoutDefault(self): + # Without a default, or with an empty one, also from the history, none is offered. + prompt = Exscript.util.interact.prompt + prompts = self.typing('', '', '', '', 'admin', '') + self.assertEqual(prompt('host', 'Host', history=self.history), '') + self.assertEqual(self.history.get('host'), '') + self.assertEqual(prompt('host', 'Host', history=self.history), '') + self.assertEqual(prompt('port', 'Port', doverh=False, history=self.history), '') + self.assertEqual(prompt('path', 'Path', '', history=self.history), '') + self.assertEqual(prompt('user', 'User', history=self.history), 'admin') + self.assertEqual(prompt('user', 'User', history=self.history), 'admin') + self.assertEqual( + prompts, ['Host: ', 'Host: ', 'Port: ', 'Path: ', 'User: ', 'User [admin]: '] + ) + def testPromptCheck(self): def check(value): if len(value) < 4: @@ -139,12 +154,13 @@ def testGetFilename(self): pass history = InputHistory(os.path.join(self.tmpdir, 'history')) missing = os.path.join(self.tmpdir, 'missing.txt') - self.typing(missing, ' ' + existing + ' ') + prompts = self.typing(missing, ' ' + existing + ' ') with unittest.mock.patch('builtins.print') as output: filename = Exscript.util.interact.get_filename('hosts', 'Host file', history=history) self.assertEqual(filename, existing) output.assert_called_once_with('File not found. Please enter a filename.') self.assertEqual(history.get('hosts'), existing) + self.assertEqual(prompts, ['Host file: '] * 2) def testGetUser(self): get_user = Exscript.util.interact.get_user From 18ee4aab6a46d252761b237709b876e9ed8d4234 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 10 Nov 2026 18:17:41 +0100 Subject: [PATCH 113/250] Handle the OSError of getuser() in interact.get_user() Since Python 3.13, getpass.getuser() raises OSError instead of KeyError if it finds no user name, e.g. in a container without a passwd entry. Treat it like KeyError and ask for the user name without a default. --- CHANGES.md | 6 ++++++ src/Exscript/util/interact.py | 7 ++++--- tests/Exscript/util/interactTest.py | 14 ++++++++++++++ 3 files changed, 24 insertions(+), 3 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 429ed26e..768da808 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -476,6 +476,12 @@ The names below refer to functions callable from templates, not protocol methods string. Before, they showed an empty value as `Host []: `, and without one offered the string `'None'`, as in `Host [None]: `, which `prompt()` returned and stored for an empty input. +- **`Exscript.util.interact.get_user()` Without a User Name:** If the name of the + current user is unknown, e.g. in a container that runs as a user ID without a + passwd entry and without the `USER` and `LOGNAME` variables, `get_user()`, + `get_login()` and `read_login()` ask for the user name without offering a default. + Before, they failed with an `OSError` on Python 3.13 and newer, where + `getpass.getuser()` raises it instead of a `KeyError`. - **`Exscript.util.mail.send()`:** Ends the SMTP session with `QUIT` and closes the connection, also if sending fails; before, it left the connection open. It connects only after the mail and its attachments are ready. diff --git a/src/Exscript/util/interact.py b/src/Exscript/util/interact.py index 11caf864..6bd6b8b2 100644 --- a/src/Exscript/util/interact.py +++ b/src/Exscript/util/interact.py @@ -236,8 +236,9 @@ def _validate(string: str) -> str | None: def get_user(prompt: str | None = None) -> str: """ - Prompts the user for his login name, defaulting to the USER environment - variable. Returns a string containing the username. + Prompts the user for his login name, defaulting to the name of the + current user (see getpass.getuser()); if that is unknown, there is no + default. Returns a string containing the username. May throw an exception if EOF is given by the user. :type prompt: str|None @@ -248,7 +249,7 @@ def get_user(prompt: str | None = None) -> str: # Read username and password. try: env_user = getpass.getuser() - except KeyError: + except (KeyError, OSError): # no user name: OSError since Python 3.13 env_user = '' if prompt is None: prompt = "Please enter your user name" diff --git a/tests/Exscript/util/interactTest.py b/tests/Exscript/util/interactTest.py index 3e3dd236..af417a76 100644 --- a/tests/Exscript/util/interactTest.py +++ b/tests/Exscript/util/interactTest.py @@ -186,6 +186,20 @@ def testGetUserWithoutDefault(self): self.assertEqual(Exscript.util.interact.get_user(), 'dave') self.assertEqual(prompts, ['Please enter your user name: '] * 2) + def testGetUserWithoutUserName(self): + # getuser() raises OSError since Python 3.13, KeyError before, if it finds no name. + errors = OSError('No username set in the environment'), KeyError('uid not found: 1000') + for error in errors: + prompts = self.typing('carol', 'dave', 'erin') + with ( + unittest.mock.patch('getpass.getuser', side_effect=error), + unittest.mock.patch('getpass.getpass', return_value='S3cr3t'), + ): + self.assertEqual(Exscript.util.interact.get_user(), 'carol') + self.assertEqual(Exscript.util.interact.get_login(), ('dave', 'S3cr3t')) + self.assertEqual(Exscript.util.interact.read_login().get_name(), 'erin') + self.assertEqual(prompts, ['Please enter your user name: '] * 3) + def testGetLogin(self): prompts = self.typing('') with ( From 2b38e8abdf0f382176b80d068d1bb4c468f7fff9 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 10 Nov 2026 18:19:56 +0100 Subject: [PATCH 114/250] Handle an unknown user name in util.mail getpass.getuser() raises OSError (KeyError before Python 3.13) if the name is unknown, so every mail template failed. The variable user is then empty, a From line of the template sets the sender of Mail(), and Mail() without a sender raises ValueError. --- CHANGES.md | 6 ++++ src/Exscript/util/mail.py | 56 ++++++++++++++++++++++++++------- tests/Exscript/util/mailTest.py | 20 ++++++++++++ 3 files changed, 70 insertions(+), 12 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 768da808..f2655dac 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -485,6 +485,12 @@ The names below refer to functions callable from templates, not protocol methods - **`Exscript.util.mail.send()`:** Ends the SMTP session with `QUIT` and closes the connection, also if sending fails; before, it left the connection open. It connects only after the mail and its attachments are ready. +- **`Exscript.util.mail` Without a User Name:** If `getpass.getuser()` finds no name + for the current user, the template variable `user` is empty, so + `from_template_string()` and `from_template()` work if the template has a `From:` + line. `Mail()` without a sender, and a template without a `From:` line, raise a + `ValueError`. Before, all of them failed with the `OSError` of `getuser()`, or with + a `KeyError` before Python 3.13. - **Pseudo Protocol on Windows:** `pseudo://` URLs accept absolute Windows paths such as `pseudo://C:\devices\router.py`; the drive letter was taken for a port number. - **`Exscript.util.tty.get_terminal_size()`:** Uses `shutil.get_terminal_size()`: the diff --git a/src/Exscript/util/mail.py b/src/Exscript/util/mail.py index 3c8822ba..eba3569d 100644 --- a/src/Exscript/util/mail.py +++ b/src/Exscript/util/mail.py @@ -102,14 +102,25 @@ def parse(self, template: str, **kwargs: Any) -> str: return _string_re.sub(self._variable_sub_cb, output) +def _get_user() -> str: + """ + Returns the name of the current user, or an empty string if it is + unknown. + """ + try: + return getuser() + except (KeyError, OSError): # no user name: OSError since Python 3.13 + return '' + + def _render_template(string: str, **vars: Any) -> str: """ Returns the given mail template with the placeholders replaced by the given variables. The variables ``date`` (the current date, as - YYYY-MM-DD) and ``user`` (the name of the current user) are defined by - default. + YYYY-MM-DD) and ``user`` (the name of the current user, or an empty + string if it is unknown) are defined by default. """ - default = {'date': time.strftime('%Y-%m-%d'), 'user': getuser()} + default = {'date': time.strftime('%Y-%m-%d'), 'user': _get_user()} default.update(vars) parser = _TemplateParser() return parser.parse(string, **default) @@ -134,6 +145,20 @@ def _get_var_from_header_line(line: str) -> tuple[str, str]: return match.group(1).strip().lower(), match.group(2).strip() +def _get_sender_from_template(string: str) -> str | None: + """ + Returns the value of the first From header field of the given mail + template, or None if it has none. + """ + for line in string.split('\n'): + if not _is_header_line(line): + break + key, value = _get_var_from_header_line(line) + if key == 'from': + return value + return None + + def _cleanup_mail_addresses(recipients: str | list[str]) -> list[str]: """ Returns the given email addresses in lower case, sorted and without @@ -168,7 +193,8 @@ def __init__( """ Creates a new email with the given values. If the given sender is None or empty, the sender is the name of - the current user (see getpass.getuser()) at the local host. + the current user (see getpass.getuser()) at the local host; if the + name of the current user is unknown, a ValueError is raised. :type sender: string|None :param sender: The email address of the sender. @@ -193,8 +219,11 @@ def __init__( self.subject: str = cast(str, None) self.body: str = cast(str, None) if not sender: + user = _get_user() + if not user: + raise ValueError('a sender is required: the name of the current user is unknown') domain = socket.getfqdn('localhost') - sender = getuser() + '@' + domain + sender = user + '@' + domain self.set_sender(sender) self.set_to(to) self.set_cc(cc) @@ -446,12 +475,14 @@ def from_template_string(string: str, **kwargs: Any) -> Mail: Placeholders of the form ``{name}`` are replaced by the value of the variable with that name; the items of a list are put on separate lines. The variables ``date`` (the current date) and ``user`` (the name of the - current user) are defined by default. A line that ends with a space is - joined with the next line. The header fields From, To, Cc, Bcc and - Subject at the start of the template set the fields of the mail; the - rest of the template is the body. Raises an Exception if a placeholder - names an undefined variable, or if the template has another header - field. + current user, or an empty string if it is unknown) are defined by + default. A line that ends with a space is joined with the next line. The + header fields From, To, Cc, Bcc and Subject at the start of the template + set the fields of the mail; the rest of the template is the body. + Without a From header field, the sender is the default of :class:`Mail`, + so a ValueError is raised if the name of the current user is unknown. + Raises an Exception if a placeholder names an undefined variable, or if + the template has another header field. :type string: str :param string: The SMTP formatted template. @@ -461,7 +492,8 @@ def from_template_string(string: str, **kwargs: Any) -> Mail: :return: The resulting mail. """ tmpl = _render_template(string, **kwargs) - mail = Mail() + # Mail() needs the name of the current user only if the template names no sender. + mail = Mail(_get_sender_from_template(tmpl)) mail.set_from_template_string(tmpl) return mail diff --git a/tests/Exscript/util/mailTest.py b/tests/Exscript/util/mailTest.py index f7fec11b..dbcc2b5b 100644 --- a/tests/Exscript/util/mailTest.py +++ b/tests/Exscript/util/mailTest.py @@ -45,6 +45,14 @@ def testConstructor(self): user = getuser() self.assertTrue(mail.get_sender().startswith(user + '@')) + def testConstructorWithoutUserName(self): + # getuser() raises OSError since Python 3.13, KeyError before, if it finds no name. + for error in OSError('No username set in the environment'), KeyError('uid not found'): + with mock.patch('Exscript.util.mail.getuser', side_effect=error): + with self.assertRaisesRegex(ValueError, 'the name of the current user is unknown'): + Mail() + self.assertEqual(Mail(sender='me@example.com').get_sender(), 'me@example.com') + def testSetFromTemplateString(self): self.mail.set_from_template_string(template) tmpl = self.mail.get_smtp_mail().strip() @@ -164,6 +172,18 @@ def testFromTemplateString(self): mail = from_template_string(template, **vars) self.checkResult(mail) + def testFromTemplateStringWithoutUserName(self): + # The variable user is empty, and the template must name the sender. + from Exscript.util.mail import from_template_string + + for error in OSError('No username set in the environment'), KeyError('uid not found'): + with mock.patch('Exscript.util.mail.getuser', side_effect=error): + mail = from_template_string('From: me@example.com\n\nSent by "{user}"') + self.assertEqual(mail.get_sender(), 'me@example.com') + self.assertEqual(mail.get_body(), 'Sent by ""') + with self.assertRaisesRegex(ValueError, 'a sender is required'): + from_template_string(template, **vars) + def testFromTemplate(self): from Exscript.util.mail import from_template From 25f167a3ec2056f4928ac749200a97a8d66dde74 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 10 Nov 2026 18:24:37 +0100 Subject: [PATCH 115/250] Keep eof set if close() runs while Telnet.receive() waits If recv() still returned data, fill_rawq() set eof back to False, and the next receive() failed with an AttributeError, as the socket was gone. Never clear eof there, and return False from receive() if the socket is gone. --- CHANGES.md | 5 ++++ src/Exscript/protocols/telnetlib.py | 11 +++++---- tests/Exscript/protocols/telnetlibTest.py | 28 +++++++++++++++++++++++ 3 files changed, 40 insertions(+), 4 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index f2655dac..8d5dfb0a 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -274,6 +274,11 @@ marked with **BREAKING**; a migration guide will accompany the release. constants are integers instead of bytes, and the options of the constructor are keyword-only. Use `Exscript.protocols.Telnet`. If you need a drop-in replacement for the telnetlib module that Python 3.13 removed, use the package `standard-telnetlib`. +- **Closing While Receiving:** If `close()` of `Exscript.protocols.telnetlib.Telnet` + runs in another thread while `receive()` waits for data, the connection stays + closed, and `receive()` returns `False`. Before, data that arrived in the meantime + cleared the end-of-file flag, and the next `receive()` failed with + `AttributeError: 'NoneType' object has no attribute 'settimeout'`. ### Protocol Drivers diff --git a/src/Exscript/protocols/telnetlib.py b/src/Exscript/protocols/telnetlib.py index e7b8841d..34bd346c 100644 --- a/src/Exscript/protocols/telnetlib.py +++ b/src/Exscript/protocols/telnetlib.py @@ -281,11 +281,13 @@ def process_rawq(self) -> None: def fill_rawq(self) -> None: """ Reads once from the socket into the raw queue. Blocks if no data is - available, and sets eof when the connection was closed. + available, and sets eof when the connection was closed. Never clears + eof, which close() may set in another thread while it waits. """ data = cast(socket.socket, self.sock).recv(_READ_SIZE) self.msg('Recv: %r', data) - self.eof = not data + if not data: + self.eof = True self.rawq += data def receive(self, timeout: float | None) -> bool | None: @@ -300,9 +302,10 @@ def receive(self, timeout: float | None) -> bool | None: :return: True if data was received, None if the time ran out, False if the connection was closed. """ - if self.eof: + # close() in another thread may have removed the socket, but not set eof yet. + if self.eof or self.sock is None: return False - cast(socket.socket, self.sock).settimeout(timeout) + self.sock.settimeout(timeout) try: self.fill_rawq() except (TimeoutError, BlockingIOError): # BlockingIOError: timeout 0 diff --git a/tests/Exscript/protocols/telnetlibTest.py b/tests/Exscript/protocols/telnetlibTest.py index ffb475d4..a1c004c0 100644 --- a/tests/Exscript/protocols/telnetlibTest.py +++ b/tests/Exscript/protocols/telnetlibTest.py @@ -18,6 +18,7 @@ class FakeSock: def __init__(self, chunks): self.chunks = list(chunks) self.sent = [] + self.closed = False def recv(self, size): return self.chunks.pop(0) if self.chunks else b'' @@ -28,6 +29,9 @@ def sendall(self, data): def settimeout(self, timeout): pass + def close(self): + self.closed = True + def process(chunks, encoding='latin-1'): """Returns the text passed to the callback and the sent bytes.""" @@ -190,6 +194,30 @@ def testCloseTwice(self): self.assertEqual(local.fileno(), -1) self.assertFalse(tn.receive(1)) + def testCloseWhileReceiving(self): + # Another thread closes the connection while recv() waits, and recv() still returns data. + tn = Telnet() + tn.sock = sock = FakeSock([b'late', b'more']) + recv = sock.recv + + def recv_and_close(size): + tn.close() + return recv(size) + + sock.recv = recv_and_close + received = [] + tn.data_callback = received.append + self.assertFalse(tn.receive(None)) + self.assertTrue(tn.eof) + self.assertTrue(sock.closed) + self.assertFalse(tn.receive(None)) # the socket is None + self.assertEqual(received, ['late']) + + # In close() in another thread, after it removed the socket, before it set eof. + tn.eof = False + self.assertFalse(tn.receive(None)) + self.assertEqual(received, ['late']) + def testNawsWithoutSize(self): # The size is sent once it is known. tn = Telnet(termsize=None) From bbe2a71d11dfadcc569f2f44023068582096884c Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 10 Nov 2026 18:26:05 +0100 Subject: [PATCH 116/250] Fix deepcopy and pickling of OrderedDefaultDict __reduce__() returned the items view where pickle expects an iterator, and __deepcopy__() deep-copied that view, so both failed. Return an iterator, and deep-copy the keys and values with the memo of deepcopy(). --- CHANGES.md | 4 ++++ src/Exscript/util/collections.py | 8 ++++++-- tests/Exscript/util/collectionsTest.py | 27 ++++++++++++++++++++++++++ 3 files changed, 37 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 8d5dfb0a..37c8a4b6 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -502,6 +502,10 @@ The names below refer to functions callable from templates, not protocol methods `LINES` and `COLUMNS` environment variables, then the terminal of the standard output, then the defaults. It no longer asks standard input, standard error and the controlling terminal, and no longer runs `stty size`, which leaked a pipe. +- **`Exscript.util.collections.OrderedDefaultDict`:** Can be deep-copied and pickled, + and the copies keep the default factory and the order of the keys. Before, + `copy.deepcopy()` failed with `TypeError: cannot pickle 'odict_items' object`, and + `pickle.dumps()` with a `PicklingError`. - **BREAKING - Removed Modules:** `Exscript.util.daemonize`, `Exscript.util.pidutil` and `Exscript.util.sigintcatcher` are removed, and `daemonize()` failed on Python 3 with `ValueError: can't have unbuffered text I/O`. diff --git a/src/Exscript/util/collections.py b/src/Exscript/util/collections.py index 36c58bb9..6ad9426f 100644 --- a/src/Exscript/util/collections.py +++ b/src/Exscript/util/collections.py @@ -43,7 +43,7 @@ def __reduce__(self) -> tuple[Any, ...]: args: tuple[Callable[[], V], ...] = tuple() else: args = (self.default_factory,) - return type(self), args, None, None, self.items() + return type(self), args, None, None, iter(self.items()) def copy(self) -> Self: """ @@ -55,7 +55,11 @@ def __copy__(self) -> Self: return type(self)(self.default_factory, self) def __deepcopy__(self, memo: dict[int, Any]) -> Self: - return type(self)(self.default_factory, copy.deepcopy(self.items())) + result = type(self)(self.default_factory) + memo[id(self)] = result # for values that refer to this dictionary + for key, value in self.items(): + result[copy.deepcopy(key, memo)] = copy.deepcopy(value, memo) + return result def __repr__(self) -> str: return 'OrderedDefaultDict(%s, %s)' % (self.default_factory, OrderedDict.__repr__(self)) diff --git a/tests/Exscript/util/collectionsTest.py b/tests/Exscript/util/collectionsTest.py index d2c7613e..28ad4e12 100644 --- a/tests/Exscript/util/collectionsTest.py +++ b/tests/Exscript/util/collectionsTest.py @@ -1,4 +1,5 @@ import copy +import pickle import unittest from Exscript.util.collections import OrderedDefaultDict @@ -40,6 +41,32 @@ def testCopy(self): duplicate['edge'].append('r2') self.assertNotIn('edge', groups) + def testCopyAndPickle(self): + # The copies keep the default factory and the order. + for factory in list, None: + groups = OrderedDefaultDict(factory, [('edge', ['r2']), ('core', ['r1'])]) + copies = [copy.copy(groups), copy.deepcopy(groups)] + for protocol in range(pickle.HIGHEST_PROTOCOL + 1): + copies.append(pickle.loads(pickle.dumps(groups, protocol))) # noqa: S301 (own data) + for duplicate in copies: + self.assertIsInstance(duplicate, OrderedDefaultDict) + self.assertIs(duplicate.default_factory, factory) + self.assertEqual(list(duplicate.items()), [('edge', ['r2']), ('core', ['r1'])]) + + def testDeepCopy(self): + # Each value is copied once, also a value that refers to the dictionary. + shared = ['r1'] + groups = OrderedDefaultDict(list, core=shared, backup=shared) + groups['all'] = groups + duplicate = copy.deepcopy(groups) + self.assertEqual(list(duplicate), ['core', 'backup', 'all']) + self.assertIs(duplicate['backup'], duplicate['core']) + self.assertIs(duplicate['all'], duplicate) + duplicate['core'].append('r2') + duplicate['edge'].append('r3') + self.assertEqual(shared, ['r1']) + self.assertNotIn('edge', groups) + def testRepr(self): groups = OrderedDefaultDict(list, core=['r1']) self.assertEqual( From 24ca2ca30c7bade8e3922fca89f6a9c64c27c603 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 10 Nov 2026 18:47:02 +0100 Subject: [PATCH 117/250] Handle an unknown user name for --ssh-key in exscript Without the login prompt, the CLI adds the key to an account for the current user, and failed if getpass.getuser() found no name. Handle that like interact.get_user(), and add no account for the key then. --- CHANGES.md | 5 +++++ scripts/exscript | 5 ++++- tests/Exscript/ScriptsTest.py | 30 ++++++++++++++++++++++++++++-- 3 files changed, 37 insertions(+), 3 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 37c8a4b6..5008e8b7 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -487,6 +487,11 @@ The names below refer to functions callable from templates, not protocol methods `get_login()` and `read_login()` ask for the user name without offering a default. Before, they failed with an `OSError` on Python 3.13 and newer, where `getpass.getuser()` raises it instead of a `KeyError`. +- **`exscript` CLI Without a User Name:** Without the login prompt, i.e. with + `--non-interactive` or `--account-pool`, `--ssh-key` adds an account with the key + for the current user. If the name of the current user is unknown, it adds none, as + without a default user name. Before, the CLI failed with the `OSError` of + `getpass.getuser()`, or with a `KeyError` before Python 3.13. - **`Exscript.util.mail.send()`:** Ends the SMTP session with `QUIT` and closes the connection, also if sending fails; before, it left the connection open. It connects only after the mail and its attachments are ready. diff --git a/scripts/exscript b/scripts/exscript index 236662b1..6ea74000 100755 --- a/scripts/exscript +++ b/scripts/exscript @@ -292,7 +292,10 @@ def run(): except ValueError as e: parser.error(str(e)) if user is None: - user = getpass.getuser() + try: + user = getpass.getuser() + except (KeyError, OSError): # no user name: OSError since Python 3.13 + user = None else: key = None diff --git a/tests/Exscript/ScriptsTest.py b/tests/Exscript/ScriptsTest.py index 44a8ec7b..6bea8bc1 100644 --- a/tests/Exscript/ScriptsTest.py +++ b/tests/Exscript/ScriptsTest.py @@ -6,6 +6,22 @@ BASEDIR = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..')) SCRIPT = os.path.join(BASEDIR, 'scripts', 'exscript') +KEYFILE = os.path.join(BASEDIR, 'tests', 'Exscript', 'protocols', 'id_rsa') + +# Runs the script, but getpass.getuser() finds no user name, as for a user ID without a passwd +# entry: it raises OSError since Python 3.13, and KeyError before. +WITHOUT_USER_NAME = ''' +import getpass, runpy, sys + + +def getuser(): + raise %s('no user name') + + +getpass.getuser = getuser +del sys.argv[0] +runpy.run_path(sys.argv[0], run_name='__main__') +''' class ScriptsTest(unittest.TestCase): @@ -15,10 +31,10 @@ def setUp(self): self.tmpdir = tempfile.TemporaryDirectory() self.addCleanup(self.tmpdir.cleanup) - def run_exscript(self, *args): + def run_exscript(self, *args, python_args=()): env = dict(os.environ, HOME=self.tmpdir.name, USERPROFILE=self.tmpdir.name) proc = subprocess.run( # noqa: S603 - runs this repository's script with fixed arguments - [sys.executable, SCRIPT, *args], + [sys.executable, *python_args, SCRIPT, *args], capture_output=True, encoding='utf-8', env=env, @@ -55,6 +71,16 @@ def testSecureModeByDefault(self): code, output = self.run_exscript('--insecure', *args) self.assertEqual(code, 0, output) + def testSshKeyWithoutUserName(self): + # Without a login, the key is for the current user; without a name, it gets no account. + template = self.write_template('{sys.message("hello")}\n') + args = '--non-interactive', '--no-authentication', '--ssh-key', KEYFILE, template + for error in 'OSError', 'KeyError': + python_args = '-c', WITHOUT_USER_NAME % error + code, output = self.run_exscript(*args, 'dummy://r1', python_args=python_args) + self.assertEqual(code, 0, output) + self.assertIn('hello', output) + def suite(): return unittest.TestLoader().loadTestsFromTestCase(ScriptsTest) From ef0592a465b704a6c5a947c55ea78d98f21c59bc Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 10 Nov 2026 18:27:15 +0100 Subject: [PATCH 118/250] End ncclient's IO thread when a NETCONF connection closes SSH2._close() closed the paramiko channel before the transport, and closing the channel closes the pipe that the thread watches, so the thread ran until the process ended. Close the transport first, as ncclient does, and wait up to a second for the thread. --- CHANGES.md | 10 +++++++ src/Exscript/protocols/netconf.py | 36 ++++++++++++++----------- tests/Exscript/protocols/NetconfTest.py | 15 +++++++++++ 3 files changed, 46 insertions(+), 15 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 5008e8b7..fcbbeef7 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -280,6 +280,16 @@ marked with **BREAKING**; a migration guide will accompany the release. cleared the end-of-file flag, and the next `receive()` failed with `AttributeError: 'NoneType' object has no attribute 'settimeout'`. +### NETCONF + +The `Netconf` adapter is new in 3.0, so these fixes matter only to users of the 2.9 +development versions; "before" refers to them. + +- **`close()` Ends ncclient's Thread:** `close()` ends the thread in which ncclient reads + the session, also with `force=True` or if the server does not answer + ``. Before, the thread then kept running until the process ended, and + `close()` waited 0.3 seconds for it in vain. + ### Protocol Drivers - **Generic Prompt Pattern in Linear Time:** The prompt pattern of the generic driver, diff --git a/src/Exscript/protocols/netconf.py b/src/Exscript/protocols/netconf.py index e30328f7..37cdcdcd 100644 --- a/src/Exscript/protocols/netconf.py +++ b/src/Exscript/protocols/netconf.py @@ -30,6 +30,7 @@ import inspect import logging import re +import threading import time from collections.abc import Callable, Iterator, Mapping from typing import TYPE_CHECKING, Any, NoReturn, Self, cast @@ -635,16 +636,16 @@ def _logout(self) -> None: def _close(self, force: bool = False) -> None: """ - Tells ncclient's IO thread that the session is closed on purpose, and - closes the channel, the transport and the socket like SSH2._close(). - Then waits up to 0.3 seconds for the IO thread to end. + Tells ncclient's IO thread that the session is closed on purpose, + closes the transport, and waits up to a second for the IO thread to + end. Then closes the channel and the socket like SSH2._close(). """ # Clear the ncclient manager reference. if self._ncc_manager: self._ncc_manager = None # Signal ncclient's IO thread that this is an intentional close. - # Must happen before the channel is shut down so the thread treats the + # Must happen before the transport is closed so the thread treats the # subsequent empty recv() as expected (breaks cleanly, no SessionCloseError). ncc_session = self._ncc_session if self._ncc_session: @@ -657,15 +658,20 @@ def _close(self, force: bool = False) -> None: # Disable the replication wrappers. self._replicate = False - # SSH2 closes the paramiko channel, transport, and socket. - # Closing the channel causes channel.recv() to return b'', which — - # combined with _closing being set — causes the IO thread to exit. + # Like ncclient's SSHSession.close(), close the transport first: paramiko + # then marks the channel closed and makes the pipe that the IO thread + # selects on readable, so the thread reads b'' and ends. Closing the + # channel first, as SSH2._close() does, closes that pipe, so the thread + # would never wake up again. + if self._transport: + self._transport.close() + + # Wait for ncclient's IO thread to end, unless this is that thread, e.g. + # in a monitor callback. + if ncc_session and ncc_session is not threading.current_thread(): + ncc_session.join(timeout=1) + if ncc_session.is_alive(): + self._dbg(1, "ncclient's IO thread did not end within a second.") + + # SSH2 closes the paramiko channel, the transport, and the socket. super()._close(force=force) - - # Wait for ncclient's IO thread to fully terminate. - # Must be after super() so the channel is already closed when we join. - if ncc_session: - try: - ncc_session.join(timeout=0.3) - except Exception: - pass diff --git a/tests/Exscript/protocols/NetconfTest.py b/tests/Exscript/protocols/NetconfTest.py index 957cd643..9cd81396 100644 --- a/tests/Exscript/protocols/NetconfTest.py +++ b/tests/Exscript/protocols/NetconfTest.py @@ -173,6 +173,8 @@ class NetconfServer: :param piece_size: Writes the data to the channel in pieces of this many bytes. :param subsystem: Whether the netconf subsystem is available. :param channels: Whether the server opens session channels. + :param ignore_close_session: Whether close-session gets no reply and + leaves the session open, like on a server that hangs. """ def __init__( @@ -185,11 +187,13 @@ def __init__( piece_size=None, subsystem=True, channels=True, + ignore_close_session=False, ): self.reply = reply or (lambda rpc: '') self.capabilities = capabilities self.chunk_size = chunk_size self.piece_size = piece_size + self.ignore_close_session = ignore_close_session self.client_hello = None self.hello_received = threading.Event() self.rpcs = [] @@ -271,6 +275,8 @@ def read(chunked): while (rpc := read(self.chunked)) is not None: self.rpcs.append(rpc) closing = operation(rpc) == 'close-session' + if closing and self.ignore_close_session: + continue content = '' if closing else self.reply(rpc) if content is CLOSE: break @@ -453,6 +459,15 @@ def testCloseSession(self): self.assertIn('close-session/>', ''.join(sent)) # pretty-printed self.assertFalse(conn.connected) + def testCloseEndsTheIOThread(self): + # ncclient closes the session itself after the reply to close-session. + for ignore_close_session, force in (False, False), (True, False), (False, True): + with self.subTest(ignore_close_session=ignore_close_session, force=force): + conn = self.login(timeout=0.3, ignore_close_session=ignore_close_session) + thread = conn._ncc_session + conn.close(force=force) + self.assertFalse(thread.is_alive()) + def testSubsystemRefused(self): for options in {'subsystem': False}, {'channels': False}: with self.assertRaisesRegex(LoginFailure, 'any subsystem: netconf'): From 59608c21ee4e3d152ece98adc80101bb2fe297ab Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 10 Nov 2026 18:30:29 +0100 Subject: [PATCH 119/250] Reject NETCONF RPC bodies that are not one XML element A body of two elements or of text passed the validation within an element, and then raised the XMLSyntaxError of lxml. Raise the documented XMLValidationError instead. --- CHANGES.md | 3 +++ docs/migration.rst | 4 ++++ docs/netconf.rst | 2 +- src/Exscript/protocols/netconf.py | 11 ++++++++++- tests/Exscript/protocols/NetconfTest.py | 10 ++++++++++ 5 files changed, 28 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index fcbbeef7..dda84998 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -289,6 +289,9 @@ development versions; "before" refers to them. the session, also with `force=True` or if the server does not answer ``. Before, the thread then kept running until the process ended, and `close()` waited 0.3 seconds for it in vain. +- **RPC Body of One Element:** `execute()` and `send()` raise `XMLValidationError` + before sending if the RPC body is not one XML element, e.g. `` or + text. Before, they raised lxml's `XMLSyntaxError`. ### Protocol Drivers diff --git a/docs/migration.rst b/docs/migration.rst index bdd3e22f..7f6dacdf 100644 --- a/docs/migration.rst +++ b/docs/migration.rst @@ -228,3 +228,7 @@ their users should know: ``connection.execline()`` for a command that the device rejects, stops the template with that exception instead of an ``ExecuteError``, as in 2.x, and ``try`` ignores it. +- ``Netconf.execute()`` raises + :class:`~Exscript.util.xml.XMLValidationError` instead of lxml's + ``XMLSyntaxError`` if the RPC body is not one XML element, e.g. + ````. diff --git a/docs/netconf.rst b/docs/netconf.rst index faac83b0..d1ffdcf6 100644 --- a/docs/netconf.rst +++ b/docs/netconf.rst @@ -36,7 +36,7 @@ Sending RPCs ``execute()`` sends the content of an ```` element and waits for the reply; Exscript adds the ```` element with its ``message-id``. It raises :class:`~Exscript.util.xml.XMLValidationError` before sending if -the content is not well-formed XML:: +the content is not well-formed XML, or not one XML element:: conn.execute('') print(conn.get_response()) # the as text diff --git a/src/Exscript/protocols/netconf.py b/src/Exscript/protocols/netconf.py index 37cdcdcd..1ec76517 100644 --- a/src/Exscript/protocols/netconf.py +++ b/src/Exscript/protocols/netconf.py @@ -540,10 +540,12 @@ def _send(self, data: str) -> None: recv() and send() wrappers of the channel pass the data to the buffer and the events. - :raises XMLValidationError: If the RPC body is not valid XML. + :raises XMLValidationError: If the RPC body is not valid XML, or not + one XML element. :raises TimeoutException: If no reply arrives in time. :raises ProtocolException: On other errors of ncclient. """ + from lxml.etree import XMLSyntaxError from ncclient import NCClientError from ncclient.operations.errors import TimeoutExpiredError @@ -551,6 +553,13 @@ def _send(self, data: str) -> None: self._validate_rpc_body(data) try: element = _parse(data) + except XMLSyntaxError as exc: # valid within , so e.g. two elements or text + raise XMLValidationError( + 'XML validation of the RPC command has failed: the RPC body must be one ' + f'XML element. The error was caused by:\n{type(exc).__name__}: {exc}', + xmlstring=data, + ) + try: with _pretty(): self._ncc_response = self._ncc_manager.dispatch(element) # wrappers handle the self.data_sent_event diff --git a/tests/Exscript/protocols/NetconfTest.py b/tests/Exscript/protocols/NetconfTest.py index 9cd81396..79f25a84 100644 --- a/tests/Exscript/protocols/NetconfTest.py +++ b/tests/Exscript/protocols/NetconfTest.py @@ -437,6 +437,16 @@ def testInvalidRPCIsNotSent(self): self.assertEqual((cause.start, cause.end), (12, 13)) # the positions in the body self.assertEqual(self.server.rpcs, []) + def testRPCBodyIsOneElement(self): + conn = self.login() + for body in '', 'hello', 'hello', '': + with self.subTest(body=body): + with self.assertRaisesRegex(XMLValidationError, 'must be one XML element') as cm: + conn.execute(body) + self.assertEqual(cm.exception.xmlstring, body + '\r') + conn.execute('\n \n') # whitespace is no content + self.assertEqual([operation(rpc) for rpc in self.server.rpcs], ['get']) + def testTimeout(self): conn = self.login(lambda rpc: None if operation(rpc) == 'get' else '', timeout=0.3) self.assertRaises(TimeoutException, conn.execute, '') From 39fa73a0ea4a15a97383be940ac11d572415ae1b Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 10 Nov 2026 18:32:52 +0100 Subject: [PATCH 120/250] Raise the first error of a NETCONF reply, not a warning before it RPCError._from_ncclient() took the first rpc-error of a reply, also a warning, so the error was only in its errlist. Without an error, the first rpc-error is still raised. --- CHANGES.md | 3 +++ docs/migration.rst | 3 +++ docs/netconf.rst | 5 ++-- src/Exscript/protocols/exception.py | 5 ++-- tests/Exscript/protocols/NetconfTest.py | 31 ++++++++++++++++++++----- 5 files changed, 37 insertions(+), 10 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index dda84998..aeefee77 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -292,6 +292,9 @@ development versions; "before" refers to them. - **RPC Body of One Element:** `execute()` and `send()` raise `XMLValidationError` before sending if the RPC body is not one XML element, e.g. `` or text. Before, they raised lxml's `XMLSyntaxError`. +- **`RPCError` After Warnings:** The raised `RPCError` is the first `` of the + reply with the severity `error`; `errlist` holds all of them, warnings too, in their + order. Before, it was the first one, which could be a warning. ### Protocol Drivers diff --git a/docs/migration.rst b/docs/migration.rst index 7f6dacdf..578ce76e 100644 --- a/docs/migration.rst +++ b/docs/migration.rst @@ -232,3 +232,6 @@ their users should know: :class:`~Exscript.util.xml.XMLValidationError` instead of lxml's ``XMLSyntaxError`` if the RPC body is not one XML element, e.g. ````. +- If a reply has warnings before an error, the raised + :class:`~Exscript.protocols.exception.RPCError` is the first error, not + the first warning; its ``errlist`` still holds all of them. diff --git a/docs/netconf.rst b/docs/netconf.rst index d1ffdcf6..cb45a5eb 100644 --- a/docs/netconf.rst +++ b/docs/netconf.rst @@ -64,8 +64,9 @@ Errors If the reply contains an ```` with the severity ``error``, ``execute()`` raises :class:`~Exscript.protocols.exception.RPCError`. -Its attributes hold the first error, and ``errlist`` holds all errors of -the reply. Replies with only warnings do not raise:: +Its attributes hold the first error with the severity ``error``, and +``errlist`` holds all errors of the reply, warnings too, in their order. +Replies with only warnings do not raise:: from Exscript.protocols.exception import RPCError diff --git a/src/Exscript/protocols/exception.py b/src/Exscript/protocols/exception.py index 08807c68..5e97c64e 100644 --- a/src/Exscript/protocols/exception.py +++ b/src/Exscript/protocols/exception.py @@ -148,7 +148,8 @@ def _from_ncclient(cls, rpc_errors: Iterable[Any]) -> 'RPCError': Translates a list of ncclient `RPCError` objects (from RPCReply.errors) into Exscript :class:`RPCError` instances. - :return: The first error; its errlist holds all of them. + :return: The first error with the severity error, else the first one; + its errlist holds all of them, in their order. """ all_errors = [ cls( @@ -163,6 +164,6 @@ def _from_ncclient(cls, rpc_errors: Iterable[Any]) -> 'RPCError': ) for error in rpc_errors ] - primary = all_errors[0] + primary = next((e for e in all_errors if e.severity == 'error'), all_errors[0]) primary.errlist = all_errors return primary diff --git a/tests/Exscript/protocols/NetconfTest.py b/tests/Exscript/protocols/NetconfTest.py index 79f25a84..d7607a97 100644 --- a/tests/Exscript/protocols/NetconfTest.py +++ b/tests/Exscript/protocols/NetconfTest.py @@ -414,15 +414,19 @@ def testRPCErrors(self): def testWarningsAreNoErrors(self): # Without , like servers send warnings (RFC 6241, section 4.4). - warning = ( - 'applicationoperation-failed' - 'warningdeprecated' - '' - ) - conn = self.login(lambda rpc: warning) + conn = self.login(lambda rpc: WARNING) conn.execute('') self.assertIn('deprecated', conn.response) + def testErrorAfterAWarning(self): + conn = self.login(lambda rpc: WARNING + LOCK_ERROR) + with self.assertRaises(RPCError) as cm: + conn.execute(LOCK) + error = cm.exception + self.assertEqual(str(error), 'error: configuration database locked') + self.assertEqual([e.severity for e in error.errlist], ['warning', 'error']) + self.assertIs(error.errlist[1], error) + def testInvalidRPCIsNotSent(self): conn = self.login() with self.assertRaises(XMLValidationError) as cm: @@ -526,6 +530,21 @@ def testDeviceHandlerThatTransformsTheReply(self): 'error' 'configuration database locked' ) +WARNING = ( + 'applicationoperation-failed' + 'warningdeprecated' + '' +) + + +class RPCErrorTest(unittest.TestCase): + def testOnlyWarnings(self): + # The raised error is the first with the severity error, else the first one. + content = WARNING + WARNING.replace('deprecated', 'slow') + reply = RPCReply(f'{content}') + error = RPCError._from_ncclient(reply.errors) + self.assertEqual(error.message, 'deprecated') + self.assertEqual([e.message for e in error.errlist], ['deprecated', 'slow']) class ReplayedSessionTest(unittest.TestCase): From 7dbe264ce68ccb080ff8df0580d58d9e088617b7 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 10 Nov 2026 18:35:48 +0100 Subject: [PATCH 121/250] Use the current timeout for the RPCs of a NETCONF session ncclient's Manager keeps the timeout of the login, so set_timeout() after the login had no effect on RPCs. Set the timeout of the manager before each RPC. --- CHANGES.md | 3 +++ docs/migration.rst | 3 +++ src/Exscript/protocols/netconf.py | 2 ++ tests/Exscript/protocols/NetconfTest.py | 18 ++++++++++++++++++ 4 files changed, 26 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index aeefee77..2ccaee7f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -295,6 +295,9 @@ development versions; "before" refers to them. - **`RPCError` After Warnings:** The raised `RPCError` is the first `` of the reply with the severity `error`; `errlist` holds all of them, warnings too, in their order. Before, it was the first one, which could be a warning. +- **`set_timeout()` After the Login:** RPCs, also the `` of `close()`, + wait for their reply as long as the current timeout says. Before, they kept the + timeout of the login, so `set_timeout()` after `login()` did not change it. ### Protocol Drivers diff --git a/docs/migration.rst b/docs/migration.rst index 578ce76e..6b0e8cbc 100644 --- a/docs/migration.rst +++ b/docs/migration.rst @@ -235,3 +235,6 @@ their users should know: - If a reply has warnings before an error, the raised :class:`~Exscript.protocols.exception.RPCError` is the first error, not the first warning; its ``errlist`` still holds all of them. +- ``set_timeout()`` after the login also changes how long ``Netconf`` + waits for the reply to an RPC; before, the timeout of the login + applied. diff --git a/src/Exscript/protocols/netconf.py b/src/Exscript/protocols/netconf.py index 1ec76517..a1ec4265 100644 --- a/src/Exscript/protocols/netconf.py +++ b/src/Exscript/protocols/netconf.py @@ -560,6 +560,7 @@ def _send(self, data: str) -> None: xmlstring=data, ) try: + self._ncc_manager.timeout = self.get_timeout() # set_timeout() may have changed it with _pretty(): self._ncc_response = self._ncc_manager.dispatch(element) # wrappers handle the self.data_sent_event @@ -638,6 +639,7 @@ def _logout(self) -> None: ignored. """ try: + self._ncc_manager.timeout = self.get_timeout() with _pretty(): self._ncc_manager.close_session() except Exception: diff --git a/tests/Exscript/protocols/NetconfTest.py b/tests/Exscript/protocols/NetconfTest.py index d7607a97..99d48ce1 100644 --- a/tests/Exscript/protocols/NetconfTest.py +++ b/tests/Exscript/protocols/NetconfTest.py @@ -11,6 +11,7 @@ import sys import threading import unittest +from unittest import mock import ncclient.operations.rpc import ncclient.transport.session @@ -457,6 +458,23 @@ def testTimeout(self): self.assertIsNone(conn.get_response_element()) self.assertRaises(TimeoutException, conn.expect, 'never') + def testSetTimeoutAfterLogin(self): + # Each RPC waits as long as the current timeout says, also close-session. + timeouts = [] + request = RPC._request + + def record(rpc, op): + timeouts.append(rpc.timeout) + return request(rpc, op) + + conn = self.login() + with mock.patch.object(RPC, '_request', record): + conn.set_timeout(5) + conn.execute(GET_CONFIG) + conn.set_timeout(7) + conn.close() + self.assertEqual(timeouts, [5, 7]) + def testServerClosesTheSession(self): conn = self.login(lambda rpc: CLOSE, timeout=0.3) self.assertRaises(ProtocolException, conn.execute, '') From 2c58fb68bd753b8a536aeb068c8b4e47184db505 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 10 Nov 2026 18:38:10 +0100 Subject: [PATCH 122/250] Raise ConnectionClosedException when a NETCONF server closes Netconf raised a plain ProtocolException, unlike SSH2 and Telnet. ConnectionClosedException is a ProtocolException, too, so existing handlers still catch it. --- CHANGES.md | 4 ++++ docs/migration.rst | 4 ++++ src/Exscript/protocols/netconf.py | 6 ++++++ tests/Exscript/protocols/NetconfTest.py | 6 ++++-- 4 files changed, 18 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 2ccaee7f..fea060d3 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -298,6 +298,10 @@ development versions; "before" refers to them. - **`set_timeout()` After the Login:** RPCs, also the `` of `close()`, wait for their reply as long as the current timeout says. Before, they kept the timeout of the login, so `set_timeout()` after `login()` did not change it. +- **Closed Sessions:** If the server closes the session, also during an RPC, `execute()` + raises `ConnectionClosedException`, like SSH and Telnet. Before, it raised a plain + `ProtocolException` ("ncclient related error ..."); the new exception is a + `ProtocolException`, too. ### Protocol Drivers diff --git a/docs/migration.rst b/docs/migration.rst index 6b0e8cbc..22ba952c 100644 --- a/docs/migration.rst +++ b/docs/migration.rst @@ -238,3 +238,7 @@ their users should know: - ``set_timeout()`` after the login also changes how long ``Netconf`` waits for the reply to an RPC; before, the timeout of the login applied. +- If the server closes a NETCONF session, ``execute()`` raises + :class:`~Exscript.protocols.exception.ConnectionClosedException` + instead of a plain ``ProtocolException``. It is a ``ProtocolException`` + and an ``EOFError``. diff --git a/src/Exscript/protocols/netconf.py b/src/Exscript/protocols/netconf.py index a1ec4265..7227d4bf 100644 --- a/src/Exscript/protocols/netconf.py +++ b/src/Exscript/protocols/netconf.py @@ -543,11 +543,14 @@ def _send(self, data: str) -> None: :raises XMLValidationError: If the RPC body is not valid XML, or not one XML element. :raises TimeoutException: If no reply arrives in time. + :raises ConnectionClosedException: If the server closed the session, + also before the RPC. :raises ProtocolException: On other errors of ncclient. """ from lxml.etree import XMLSyntaxError from ncclient import NCClientError from ncclient.operations.errors import TimeoutExpiredError + from ncclient.transport.errors import SessionCloseError self._ncc_response = None self._validate_rpc_body(data) @@ -567,6 +570,9 @@ def _send(self, data: str) -> None: except TimeoutExpiredError: self._raise_on_response_timeout() except NCClientError as e: + # ncclient reads the end of the session, or refuses to send after it. + if isinstance(e, SessionCloseError) or not self._ncc_session.connected: + self._raise_on_response_eof() msg = 'ncclient related error (%s)' % str(e) raise ProtocolException(msg) except Exception: diff --git a/tests/Exscript/protocols/NetconfTest.py b/tests/Exscript/protocols/NetconfTest.py index 99d48ce1..e7008f2e 100644 --- a/tests/Exscript/protocols/NetconfTest.py +++ b/tests/Exscript/protocols/NetconfTest.py @@ -28,7 +28,6 @@ from Exscript.protocols.exception import ( ConnectionClosedException, LoginFailure, - ProtocolException, RPCError, TimeoutException, ) @@ -477,8 +476,11 @@ def record(rpc, op): def testServerClosesTheSession(self): conn = self.login(lambda rpc: CLOSE, timeout=0.3) - self.assertRaises(ProtocolException, conn.execute, '') + thread = conn._ncc_session + self.assertRaises(ConnectionClosedException, conn.execute, '') self.assertRaises(ConnectionClosedException, conn.expect_prompt) + thread.join(5) # then ncclient marks the session as closed + self.assertRaises(ConnectionClosedException, conn.execute, '') conn.close() # the logout fails silently self.assertFalse(conn.connected) From 39b5cd2b3f0cae18ef7edc80a2cb1e154f7292ee Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 10 Nov 2026 18:49:47 +0100 Subject: [PATCH 123/250] Use UTF-8 for NETCONF sessions RFC 6241 requires UTF-8, but Netconf decoded the session with the encoding of the connection, latin-1 by default. Always use UTF-8, with an incremental decoder in the channel wrappers, and count the chunk sizes of NETCONF 1.1 in bytes, as RFC 6242 does. --- CHANGES.md | 8 +- docs/migration.rst | 5 + docs/netconf.rst | 4 + src/Exscript/protocols/netconf.py | 132 +++++++++++++++--------- src/Exscript/protocols/protocol.py | 6 +- tests/Exscript/protocols/NetconfTest.py | 73 +++++++++++++ 6 files changed, 175 insertions(+), 53 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index fea060d3..882748b4 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -235,7 +235,7 @@ marked with **BREAKING**; a migration guide will accompany the release. 1 KiB reads take 0.015 seconds instead of 9.4. The positions in the match object that the callback gets refer to the searched text, not to the whole buffer. - **NETCONF 1.1 Chunks:** `Netconf.reassemble_chunks()` takes linear time: a 16 MiB - reply in 4 KiB chunks takes 0.013 seconds instead of 5.2. + reply in 4 KiB chunks takes 0.02 seconds instead of 5.2. - **NETCONF Threads:** ncclient's thread appends the received data to the buffer that the caller searches. The buffer is locked now; before, a search could move the position at which the thread wrote. Exscript still sends its `` and RPCs @@ -302,6 +302,12 @@ development versions; "before" refers to them. raises `ConnectionClosedException`, like SSH and Telnet. Before, it raised a plain `ProtocolException` ("ncclient related error ..."); the new exception is a `ProtocolException`, too. +- **UTF-8:** `Netconf` always uses UTF-8, which RFC 6241 requires for NETCONF messages, + whatever `encoding` is passed, and `reassemble_chunks()` counts the chunk sizes in + bytes, as RFC 6242 does. Bytes that are no UTF-8, also in the SSH banner, become + U+FFFD. Before, non-ASCII text showed as mojibake in `response`, the standard output + and the logs with the default encoding latin-1, and with `encoding='utf-8'`, + NETCONF 1.1 replies with non-ASCII text failed to reassemble. ### Protocol Drivers diff --git a/docs/migration.rst b/docs/migration.rst index 22ba952c..46d0f104 100644 --- a/docs/migration.rst +++ b/docs/migration.rst @@ -242,3 +242,8 @@ their users should know: :class:`~Exscript.protocols.exception.ConnectionClosedException` instead of a plain ``ProtocolException``. It is a ``ProtocolException`` and an ``EOFError``. +- ``Netconf`` always uses UTF-8, whatever ``encoding`` is passed. With + the default ``latin-1``, the 2.9 versions showed non-ASCII text as + mojibake, e.g. ``Zürich``, in ``response`` and the logs. Bytes that + are no UTF-8, e.g. in a latin-1 SSH banner, become U+FFFD. + ``Netconf.reassemble_chunks()`` counts the chunk sizes in bytes. diff --git a/docs/netconf.rst b/docs/netconf.rst index cb45a5eb..83870209 100644 --- a/docs/netconf.rst +++ b/docs/netconf.rst @@ -30,6 +30,10 @@ there is no login prompt. ``get_server_capabilities()``, ``get_client_capabilities()`` and ``get_session_id()`` return what was exchanged. +NETCONF messages are UTF-8 (:rfc:`6241`), and so are SSH banners +(:rfc:`4252`), so ``Netconf`` always uses UTF-8; the ``encoding`` argument +has no effect. + Sending RPCs ------------ diff --git a/src/Exscript/protocols/netconf.py b/src/Exscript/protocols/netconf.py index 7227d4bf..d37f67a7 100644 --- a/src/Exscript/protocols/netconf.py +++ b/src/Exscript/protocols/netconf.py @@ -24,6 +24,7 @@ SSH2 Netconf support, based on ncclient and paramiko. """ +import codecs import contextlib import contextvars import functools @@ -59,8 +60,8 @@ _logger.addHandler(logging.NullHandler()) # Matches a NETCONF 1.1 chunk header: LF HASH chunk-size LF (RFC 6242 §4.2). -_chunk_header_re = re.compile(r'\n#(\d+)\n') -_end_of_chunks_re = re.compile(r'\n##\n') +_chunk_header_re = re.compile(rb'\n#(\d+)\n') +_end_of_chunks_re = re.compile(rb'\n##\n') # Exscript sends its and RPCs pretty-printed, so that they are readable # in logs. ncclient serializes them with the to_xml() of two of its modules, in @@ -108,6 +109,8 @@ def _pretty() -> Iterator[None]: class Netconf(SSH2): """ The SSH Netconf protocol adapter, based on the SSH2 adapter and ncclient. + It always uses UTF-8, which RFC 6241 requires for all NETCONF messages, + whatever encoding is passed. """ def __init__(self, **kwargs: Any) -> None: @@ -115,6 +118,7 @@ def __init__(self, **kwargs: Any) -> None: Constructor. Takes the keyword arguments of :class:`Protocol`. """ SSH2.__init__(self, **kwargs) + self.encoding = 'utf-8' # Module-Level self._logger = _logger # Internals; the ncclient objects are Any, because ncclient is untyped @@ -192,6 +196,20 @@ def get_session_id(self) -> str | None: return None return str(self._ncc_manager.session_id) + def get_banner(self) -> str | None: + """ + Returns the authentication banner that the server sent, decoded as + UTF-8 like the session (RFC 4252); bytes that are no UTF-8 become + U+FFFD. Returns None if the server sent no banner or if the adapter + is not connected. + """ + if not self._transport: + return None + banner = self._transport.get_banner() + if isinstance(banner, bytes): + return banner.decode('utf-8', errors='replace') + return banner + def get_response_element(self) -> '_Element | None': """ Returns the parsed XML element tree of the latest RPC reply. This might @@ -309,34 +327,69 @@ def _init_ncclient(self) -> None: session._connected = True session._closing.clear() + self._wrap_channel(cast('Channel', self._channel)) + + # Starts IO thread, sends client , consumes server . + self._buffer.clear() # keeps the first self.response clean + with _pretty(): + session._post_connect(self.get_timeout()) + + # Selects the driver/device-specific XML parser for all subsequent RPCs. + session.parser = device_handler.get_xml_parser(session) + + # Finally the manager is in charge of the channel. + self._ncc_session = session + self._ncc_manager = Manager( + session, + device_handler, + timeout=self.get_timeout(), + raise_mode=RaiseMode.NONE, # raising custom RPCError later... + ) + self._ncc_manager.huge_tree = self.huge_tree # ensure synced flags + + # Exscript needs to finalize the initialization. + self._os_guesser.check_netconf_capabilities(self.get_server_capabilities()) + self._update_auto_driver() + self.expect_prompt() # consumes the server from the buffer + + def _wrap_channel(self, channel: 'Channel') -> None: + """ + Wraps recv() and send() of the given channel, which ncclient's IO + thread calls, so the data of the session also reaches the buffer, the + logs and the events. The data is decoded as UTF-8 with an incremental + decoder per direction, so a character that spans two reads or writes + is decoded, too; bytes that are no UTF-8 become U+FFFD. + """ # Controls the behavior of the replication wrappers. The replication is # needed as ncclient owns the channel within its IO thread. So we need # to somehow fetch this raw data for display, logging and raw response # buffering/matching. self._replicate = True + recv_decoder = codecs.getincrementaldecoder('utf-8')(errors='replace') + send_decoder = codecs.getincrementaldecoder('utf-8')(errors='replace') # Setup the channel receive wrapper. - recv = cast('Channel', self._channel).recv + recv = channel.recv @functools.wraps(recv) def recv_wrapper(*args: Any, **kwargs: Any) -> bytes: data = recv(*args, **kwargs) if not self._replicate: return data - if data: - string_data = data.decode(self.encoding) + string_data = recv_decoder.decode(data, final=not data) + if string_data: # empty at the end, or for a part of a character self._receive_cb(string_data) self._buffer.append(string_data) - else: + if not data: self._dbg(1, 'channel.recv() returned EOF') self._replicate = False return data - cast(Any, self._channel).recv = recv_wrapper # Any: replaces a method + cast(Any, channel).recv = recv_wrapper # Any: replaces a method # Setup the channel send wrapper. This is needed as Netconf does not # echo the sent data back to the client known from CLI shells. - send = cast('Channel', self._channel).send + send = channel.send param = next(iter(inspect.signature(send).parameters)) # first param @functools.wraps(send) @@ -345,41 +398,19 @@ def send_wrapper(*args: Any, **kwargs: Any) -> int: if not self._replicate: return count data = kwargs[param] if param in kwargs else args[0] + string_data = '' if count is None: - string_data = data.decode(self.encoding) - self._send_cb(string_data) + string_data = send_decoder.decode(data) elif count > 0: - string_data = data[:count].decode(self.encoding) - self._send_cb(string_data) + string_data = send_decoder.decode(data[:count]) else: self._dbg(1, 'channel.send() returned EOF') self._replicate = False + if string_data: # empty for a part of a character + self._send_cb(string_data) return count - cast(Any, self._channel).send = send_wrapper # Any: replaces a method - - # Starts IO thread, sends client , consumes server . - self._buffer.clear() # keeps the first self.response clean - with _pretty(): - session._post_connect(self.get_timeout()) - - # Selects the driver/device-specific XML parser for all subsequent RPCs. - session.parser = device_handler.get_xml_parser(session) - - # Finally the manager is in charge of the channel. - self._ncc_session = session - self._ncc_manager = Manager( - session, - device_handler, - timeout=self.get_timeout(), - raise_mode=RaiseMode.NONE, # raising custom RPCError later... - ) - self._ncc_manager.huge_tree = self.huge_tree # ensure synced flags - - # Exscript needs to finalize the initialization. - self._os_guesser.check_netconf_capabilities(self.get_server_capabilities()) - self._update_auto_driver() - self.expect_prompt() # consumes the server from the buffer + cast(Any, channel).send = send_wrapper # Any: replaces a method def login( self, @@ -471,7 +502,8 @@ def reassemble_chunks(response: str) -> str: Reassembles a NETCONF 1.1 chunked response into plain content. Each chunk is introduced by a header `LF HASH chunk-size LF` where the - size is the exact byte count of the following chunk (RFC 6242 §4.2). + size is the exact byte count of the following chunk (RFC 6242 §4.2), + in the UTF-8 encoding of the response. The end-of-chunks marker `LF HASH HASH LF` gets stripped finally. Useful to also reassemble chunks in received data from events. @@ -481,29 +513,31 @@ def reassemble_chunks(response: str) -> str: :rtype: str :return: The reassembled response. """ - match = _chunk_header_re.match(response) + if not response.startswith('\n#'): # spares encoding other responses + return response + data = response.encode('utf-8') + match = _chunk_header_re.match(data) if not match: return response parts = [] while match: start = match.end() # after the chunk header - end = start + int(match.group(1)) # exactly size chars of content - parts.append(response[start:end]) - match = _chunk_header_re.match(response, end) - assembled = ''.join(parts) - response = response[end:] - if response: + end = start + int(match.group(1)) # exactly size bytes of content + parts.append(data[start:end]) + match = _chunk_header_re.match(data, end) + data = data[end:] + if data: # Internally this would be already consumed as the prompt. - match = _end_of_chunks_re.match(response) + match = _end_of_chunks_re.match(data) if match: - response = response[match.end() :] # consume the end-of-chunks marker - if response: + data = data[match.end() :] # consume the end-of-chunks marker + if data: # This should only happen if we have internal issues with the prompt # or buffer handling before this reassembly is called. ncclient # should have caught buggy devices already at the transport layer. msg = 'Unexpected trailing bytes after chunk reassembly: %s' - raise ValueError(msg % repr(response[:50] + '...')) - return assembled + raise ValueError(msg % repr(data[:50].decode('utf-8', 'replace') + '...')) + return b''.join(parts).decode('utf-8') def _domatch(self, prompt: list[re.Pattern[str]], flush: bool) -> tuple[int, re.Match[str]]: """ diff --git a/src/Exscript/protocols/protocol.py b/src/Exscript/protocols/protocol.py index 840546d2..02b91a0e 100644 --- a/src/Exscript/protocols/protocol.py +++ b/src/Exscript/protocols/protocol.py @@ -153,9 +153,9 @@ def _ends_with_anchor(regex: re.Pattern[str]) -> bool: _connection_monitor_lock = threading.RLock() # NOTE: The default encoding is latin-1: it decodes any byte, and most devices -# send 7-bit ASCII, even in Netconf XML/RPC (usually UTF-8). For devices that -# send UTF-8, pass encoding='utf-8'; the adapters decode incrementally, so a -# character split across two reads is decoded correctly. +# send 7-bit ASCII. For devices that send UTF-8, pass encoding='utf-8'; the +# adapters decode incrementally, so a character split across two reads is +# decoded correctly. Netconf always uses UTF-8, as RFC 6241 requires. class Protocol: diff --git a/tests/Exscript/protocols/NetconfTest.py b/tests/Exscript/protocols/NetconfTest.py index e7008f2e..1453faff 100644 --- a/tests/Exscript/protocols/NetconfTest.py +++ b/tests/Exscript/protocols/NetconfTest.py @@ -75,6 +75,11 @@ def testManyChunks(self): framed = ''.join('\n#4096\n' + chunk for _ in range(1024)) + '\n##\n' self.assertEqual(Netconf.reassemble_chunks(framed), chunk * 1024) + def testMultibyteCharacters(self): + # The sizes count the bytes of the UTF-8 encoding (RFC 6242, section 4.2). + framed = '\n#6\nZü\n#15\nrich 20 €\n##\n' + self.assertEqual(Netconf.reassemble_chunks(framed), 'Zürich 20 €') + class NcclientContractTest(unittest.TestCase): """The ncclient internals that the adapter uses; an update of ncclient must keep them.""" @@ -157,6 +162,50 @@ def testAppLevelAuthenticationIsNotNeeded(self): self.conn.auto_app_authorize() self.assertTrue(self.conn.is_app_authorized()) + def testUTF8(self): + self.assertEqual(self.conn.encoding, 'utf-8') + self.assertEqual(Netconf(encoding='latin-1').encoding, 'utf-8') + + def testCharactersSplitAcrossReads(self): + received = [] + self.conn.data_received_event.connect(received.append) + # ü and € span reads, \xff is no UTF-8, and the data ends within a character. + channel = FakeChannel([b'Z\xc3', b'\xbcrich \xe2', b'\x82', b'\xac \xff \xe2\x82']) + self.conn._wrap_channel(channel) + while channel.recv(4096): + pass + text = 'Zürich € \ufffd \ufffd' + self.assertEqual(''.join(received), text) + self.assertEqual(str(self.conn._buffer), text) + self.assertEqual(self.conn.stdout.getvalue(), text) + + def testCharactersSplitAcrossWrites(self): + sent = [] + self.conn.data_sent_event.connect(sent.append) + channel = FakeChannel() + self.conn._wrap_channel(channel) + data = rest = 'Zürich €'.encode() + while rest: # like ncclient, which sends the rest after a short write + rest = rest[channel.send(rest) :] + self.assertEqual(channel.sent, data) + self.assertEqual(''.join(sent), 'Zürich €') + self.assertEqual(self.conn.stdout.getvalue(), 'Zürich €') + + +class FakeChannel: + """Returns the given pieces from recv(), then b''; send() writes two bytes at most.""" + + def __init__(self, pieces=()): + self.pieces = list(pieces) + self.sent = b'' + + def recv(self, nbytes): + return self.pieces.pop(0) if self.pieces else b'' + + def send(self, s): + self.sent += s[:2] + return len(s[:2]) + class NetconfServer: """ @@ -175,6 +224,7 @@ class NetconfServer: :param channels: Whether the server opens session channels. :param ignore_close_session: Whether close-session gets no reply and leaves the session open, like on a server that hangs. + :param banner: The authentication banner (bytes), if any. """ def __init__( @@ -188,12 +238,14 @@ def __init__( subsystem=True, channels=True, ignore_close_session=False, + banner=None, ): self.reply = reply or (lambda rpc: '') self.capabilities = capabilities self.chunk_size = chunk_size self.piece_size = piece_size self.ignore_close_session = ignore_close_session + self.banner = banner self.client_hello = None self.hello_received = threading.Event() self.rpcs = [] @@ -208,6 +260,9 @@ def _interface(self, subsystem, channels): fake = self class Interface(paramiko.ServerInterface): + def get_banner(self): + return fake.banner, 'en' + def get_allowed_auths(self, username): return 'password' @@ -359,6 +414,24 @@ def testFraming11(self): element = conn.get_response_element() self.assertEqual(element.findtext(f'.//{{{NS}}}location'), 'rack 7') + def testUTF8(self): + # NETCONF is UTF-8 (RFC 6241), whatever the encoding argument says. The reply arrives + # in one read: ncclient's parser fails if a character spans two. + content = 'Zürich – 3 €' + for capabilities in (BASE,), (BASE, BASE_11): + with self.subTest(capabilities=capabilities): + conn = self.login(lambda rpc: content, capabilities=capabilities) + conn.execute('Zürich') + self.assertIn('Zürich'.encode(), self.server.rpcs[0]) + self.assertTrue(conn.response.endswith(content + ''), conn.response) + self.assertIn('Zürich', conn.stdout.getvalue()) # sent + self.assertIn(content, conn.stdout.getvalue()) + + def testBannerThatIsNoUTF8(self): + # SSH banners are UTF-8, too (RFC 4252), but other bytes must not fail the login. + conn = self.login(banner='Grüße\n'.encode('latin-1')) + self.assertEqual(conn.get_banner(), 'Gr\ufffd\ufffde\n') + def testNotification(self): conn = self.login() conn.execute( From 60ef8894aa98b6b24d9ddc011d15f08c027086fd Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 10 Nov 2026 19:09:38 +0100 Subject: [PATCH 124/250] Decode SSH banners that are not valid in the encoding SSH2.get_banner() and get_remote_version() decoded strictly, so such a banner failed the login. Decode with errors='replace', and remove Netconf.get_banner(), which did that for NETCONF only. --- CHANGES.md | 4 ++++ src/Exscript/protocols/netconf.py | 14 -------------- src/Exscript/protocols/ssh2.py | 9 +++++---- tests/Exscript/protocols/SSH2Test.py | 6 ++++++ 4 files changed, 15 insertions(+), 18 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 882748b4..120f0dfb 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -148,6 +148,10 @@ marked with **BREAKING**; a migration guide will accompany the release. - **SSH Key Fingerprints in Debug Logs:** The keys that are tried for a login are logged with their SHA256 fingerprints, as `ssh-add -l` and `ssh-keygen -l` show them. Before, the MD5 fingerprint was logged as a Python bytes literal. +- **SSH Banners in Another Encoding:** An authentication banner or a version string + that is not valid in the encoding of the connection no longer fails the login; the + invalid bytes become U+FFFD. Before, e.g. a latin-1 banner with `encoding='utf-8'` + raised a `UnicodeDecodeError`. - **Keyboard-Interactive Logins:** Every prompt gets an answer: password prompts the password, S/Key challenges the one-time password, other prompts, e.g. for a verification code, an empty string. Before, other prompts got no answer, and diff --git a/src/Exscript/protocols/netconf.py b/src/Exscript/protocols/netconf.py index d37f67a7..9ab99c1b 100644 --- a/src/Exscript/protocols/netconf.py +++ b/src/Exscript/protocols/netconf.py @@ -196,20 +196,6 @@ def get_session_id(self) -> str | None: return None return str(self._ncc_manager.session_id) - def get_banner(self) -> str | None: - """ - Returns the authentication banner that the server sent, decoded as - UTF-8 like the session (RFC 4252); bytes that are no UTF-8 become - U+FFFD. Returns None if the server sent no banner or if the adapter - is not connected. - """ - if not self._transport: - return None - banner = self._transport.get_banner() - if isinstance(banner, bytes): - return banner.decode('utf-8', errors='replace') - return banner - def get_response_element(self) -> '_Element | None': """ Returns the parsed XML element tree of the latest RPC reply. This might diff --git a/src/Exscript/protocols/ssh2.py b/src/Exscript/protocols/ssh2.py index 78f62231..18bf599f 100644 --- a/src/Exscript/protocols/ssh2.py +++ b/src/Exscript/protocols/ssh2.py @@ -576,14 +576,15 @@ def _protocol_authenticate_by_key(self, user: str, key: PrivateKey) -> None: def get_banner(self) -> str | None: """ Returns the authentication banner that the server sent, decoded with - the encoding of this object. Returns None if the server sent no - banner or if the adapter is not connected. + the encoding of this object; bytes that are not valid in it become + U+FFFD. Returns None if the server sent no banner or if the adapter + is not connected. """ if not self._transport: return None banner = self._transport.get_banner() if isinstance(banner, bytes): - return banner.decode(self.encoding) + return banner.decode(self.encoding, errors='replace') # must not fail the login return banner def get_remote_version(self) -> str | None: @@ -595,7 +596,7 @@ def get_remote_version(self) -> str | None: return None version = self._transport.remote_version if isinstance(version, bytes): - return version.decode(self.encoding) + return version.decode(self.encoding, errors='replace') return version def _send(self, data: str) -> None: diff --git a/tests/Exscript/protocols/SSH2Test.py b/tests/Exscript/protocols/SSH2Test.py index 5b523885..0d4cf9b6 100644 --- a/tests/Exscript/protocols/SSH2Test.py +++ b/tests/Exscript/protocols/SSH2Test.py @@ -743,6 +743,12 @@ def testBanner(self): self.assertIn('Authorized users only!', conn.stdout.getvalue()) conn.expect('users only!') + def testBannerThatIsNotValidInTheEncoding(self): + # A banner in latin-1 must not fail a login that uses UTF-8. + self.serve(banner='Grüße\r\n'.encode('latin-1')) + conn = self.login(encoding='utf-8') + self.assertEqual(conn.get_banner(), 'Gr\ufffd\ufffde\r\n') + def testPreferredMethodsNotAllowed(self): requests = self.serve(allowed='password') with self.assertRaisesRegex(paramiko.BadAuthenticationType, 'invalid preferred auth'): From 4839f92d2f9e51228db5eb33906c80f9a7ea1aed Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 10 Nov 2026 19:39:27 +0100 Subject: [PATCH 125/250] Measure branch coverage and enforce a floor in CI Fail the tests on Ubuntu below 93 percent. Windows and macOS skip the tests that need a POSIX terminal, so the floor does not apply there. --- .github/workflows/ci.yml | 7 ++++++- pyproject.toml | 3 +++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 46ac62e7..0adce82d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -54,7 +54,12 @@ jobs: env: PYTHON: ${{ matrix.python }} PYTHON_GIL: ${{ matrix.gil }} # empty, i.e. ignored, except for free-threading - run: uv run --frozen --python "$PYTHON" pytest --cov=Exscript --cov-report=term-missing:skip-covered + # Line and branch coverage: 94.2 % on 2026-11-10. Raise the floor, never lower it. + # Windows and macOS skip the tests of the POSIX terminal, so it applies on Ubuntu only. + COV_FAIL_UNDER: ${{ matrix.os == 'ubuntu-latest' && '93' || '0' }} + run: >- + uv run --frozen --python "$PYTHON" pytest --cov=Exscript + --cov-report=term-missing:skip-covered --cov-fail-under="$COV_FAIL_UNDER" build: name: Build distributions diff --git a/pyproject.toml b/pyproject.toml index dc4be15f..c4fe4e35 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -165,6 +165,9 @@ ignore-nested-functions = true ignore-setters = true exclude = ["docs", "tests", "tools", "demos", "scripts"] +[tool.coverage.run] +branch = true + [tool.pytest.ini_options] testpaths = ["tests"] python_files = ["*Test.py"] From b8844fdc03780ec7e99eb0606cd3e8093e721ee3 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 10 Nov 2026 19:40:58 +0100 Subject: [PATCH 126/250] Test the reassembly of NETCONF chunks with any text --- tests/Exscript/protocols/NetconfTest.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/Exscript/protocols/NetconfTest.py b/tests/Exscript/protocols/NetconfTest.py index 1453faff..dfbfb68f 100644 --- a/tests/Exscript/protocols/NetconfTest.py +++ b/tests/Exscript/protocols/NetconfTest.py @@ -16,6 +16,8 @@ import ncclient.operations.rpc import ncclient.transport.session import paramiko +from hypothesis import given, settings +from hypothesis import strategies as st from lxml import etree from ncclient.manager import make_device_handler from ncclient.operations.rpc import RPC, RPCReply @@ -80,6 +82,20 @@ def testMultibyteCharacters(self): framed = '\n#6\nZü\n#15\nrich 20 €\n##\n' self.assertEqual(Netconf.reassemble_chunks(framed), 'Zürich 20 €') + @settings(max_examples=200, deadline=None) + @given(st.text(min_size=1, max_size=200), st.lists(st.integers(1, 40), max_size=12)) + def testAnyText(self, text, sizes): + # Chunks of any size, also with text that looks like a chunk header. A + # character that a device splits across two chunks needs the framing to + # be parsed before the decoding, so the chunks end at characters here. + chunks, pos = [], 0 + for size in sizes: + chunks.append(text[pos : pos + size]) + pos += size + chunks.append(text[pos:]) + framed = ''.join('\n#%d\n%s' % (len(c.encode()), c) for c in chunks if c) + '\n##\n' + self.assertEqual(Netconf.reassemble_chunks(framed), text) + class NcclientContractTest(unittest.TestCase): """The ncclient internals that the adapter uses; an update of ncclient must keep them.""" From b2ead54fe1c0e7b2d44a24eef0a9fedb89f74d19 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 10 Nov 2026 18:28:08 +0100 Subject: [PATCH 127/250] Point error positions before code blocks and RPCs to the variable Execute computes the position of a command back from the delimiter that the lexer consumed last, but the template had not consumed the bracket or the tag yet, so errors pointed too far to the left. Consume an empty delimiter first, as at the end of the input. --- CHANGES.md | 8 ++++---- src/Exscript/interpreter/execute.py | 9 +++++---- src/Exscript/interpreter/template.py | 7 ++++++- tests/Exscript/interpreter/CommandTest.py | 18 ++++++++++++++++++ 4 files changed, 33 insertions(+), 9 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 120f0dfb..58c39c15 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -458,11 +458,11 @@ development versions; "before" refers to them. operand of `matches`, e.g. in `/a/ matches /a/`, `/a/ . "b"`, `"a" is /a/` or `not /a/`, raises an `ExecuteError` that shows its position. Before, it raised an `UnboundLocalError` or a `TypeError` of Python. -- **Error Positions After Escape Sequences:** An error about a variable in a command, +- **Error Positions in Commands and Strings:** An error about a variable in a command, a string or a regular expression points to the variable also behind escape - sequences, e.g. to `$nothing` in `show \x41 $nothing`. Before, it pointed too far to - the left, by the number of characters by which the escape sequences are longer than - their text. + sequences, e.g. to `$nothing` in `show \x41 $nothing`, and in a command right before + a code block or an `` element, e.g. in `show $nothing{x = 1}`. Before, it + pointed too far to the left, in the latter case often to an earlier line. - **BREAKING - Functions in Strings:** A command or a string that refers to a function like to a variable, e.g. `"$f"` with a function `f` that the caller passes, raises an `ExecuteError`, as the interpreter intends. Before, it contained diff --git a/src/Exscript/interpreter/execute.py b/src/Exscript/interpreter/execute.py index c5ec4166..be46f1ff 100644 --- a/src/Exscript/interpreter/execute.py +++ b/src/Exscript/interpreter/execute.py @@ -85,10 +85,11 @@ def __init__( self.no_prompt = parser.no_prompt self.strip_command = parser.strip_command - # The lexer has parsed the command and consumed the trailing command delimiter - # (newline, open curly bracket, RPC tag). We want the debugger point to the - # beginning of the command. The offset is expected to be the length of the - # additionally consumed command delimiter before entering this Execute token. + # The lexer has parsed the command and consumed the trailing command delimiter: + # a newline or the closing RPC tag, and nothing before a code block, an RPC or + # the end of the input. We want the debugger point to the beginning of the + # command. The offset is expected to be the length of the additionally consumed + # command delimiter before entering this Execute token. # The escape sequences of the left-stripped command, which are longer in the # template than in the command, are also taken into account. stripped = len(command) - len(command.lstrip()) diff --git a/src/Exscript/interpreter/template.py b/src/Exscript/interpreter/template.py index 2b3b8cbf..537e0fcf 100644 --- a/src/Exscript/interpreter/template.py +++ b/src/Exscript/interpreter/template.py @@ -101,17 +101,22 @@ def __init__( break elif lexer.current_is('open_curly_bracket'): if buffer.strip() != '': + # The code block ends the command, but is not part of it, so + # the delimiter that Execute expects is empty, as at the end. + lexer.forward(0) self.add(Execute(lexer, parser, self, buffer, escapes)) buffer = '' escapes = [] if isinstance(parent, Code): # Need to consume the bracket if the Code token is not called # due to nesting in IfCondition, Loop and Try tokens. - next(lexer) + lexer.next_if('open_curly_bracket') break self.add(Code(lexer, parser, self)) elif lexer.current_is('open_rpc_tag'): if buffer.strip() != '': + # As for a code block. + lexer.forward(0) self.add(Execute(lexer, parser, self, buffer, escapes)) buffer = '' escapes = [] diff --git a/tests/Exscript/interpreter/CommandTest.py b/tests/Exscript/interpreter/CommandTest.py index efc167b3..87600996 100644 --- a/tests/Exscript/interpreter/CommandTest.py +++ b/tests/Exscript/interpreter/CommandTest.py @@ -104,6 +104,19 @@ def testUndefinedVariableAfterEscapeSequences(self): message = "Undefined variable 'nothing' in undefined:%d" % lineno self.assertError(cm.exception, [line], ' ' * column + '^' * 8, message) + def testUndefinedVariableBeforeACodeBlockOrAnRpc(self): + for text, lineno, line, column in ( + ('ls\nshow $nothing{x = 1}', 2, 'show $nothing{x = 1}', 5), + ('ls\n show \\x41 $nothing {x = 1}', 2, ' show \\x41 $nothing {x = 1}', 12), + ('ls\nshow $nothing', 2, 'show $nothing', 5), + ('{if 1}show $nothing{end}', 1, '{if 1}show $nothing{end}', 11), + ): + with self.subTest(text=text): + with self.assertRaises(ExecuteError) as cm: + self.evaluate(text) + message = "Undefined variable 'nothing' in undefined:%d" % lineno + self.assertError(cm.exception, [line], ' ' * column + '^' * 8, message) + def testInvalidVariableName(self): with self.assertRaises(ExecuteError) as cm: self.evaluate('show $1x\n') @@ -178,6 +191,11 @@ def testConnectionIsRequired(self): program.execute() message = "Undefined variable '__connection__' in None:1" self.assertError(cm.exception, ['ls'], '^^', message) + program = Parser().parse('{x = 1}\nshow version{y = 2}') + with self.assertRaises(ExecuteError) as cm: + program.execute() + message = "Undefined variable '__connection__' in None:2" + self.assertError(cm.exception, ['show version{y = 2}'], '^' * 12, message) class EnterTest(TemplateTestCase): From e19fe61e039cf7ee5d19817e1331fb3e11d63c6d Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 10 Nov 2026 18:30:06 +0100 Subject: [PATCH 128/250] Raise a CompileError for octal escapes above \377 Python evaluates \400 to \777 with a SyntaxWarning, as they will not work in the future. Raise a CompileError that shows the position, as for Unicode escapes above U+10FFFF. --- CHANGES.md | 8 +++++--- src/Exscript/interpreter/string.py | 9 ++++++++- src/Exscript/interpreter/template.py | 9 ++++++++- tests/Exscript/interpreter/CommandTest.py | 9 +++++++++ tests/Exscript/interpreter/ParserPropertyTest.py | 1 + tests/Exscript/interpreter/StringTest.py | 8 ++++++++ 6 files changed, 39 insertions(+), 5 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 58c39c15..172027da 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -414,9 +414,11 @@ development versions; "before" refers to them. - **BREAKING - Last Line Without a Line Break:** A template also sends its last line if the line does not end with a line break, e.g. `show version` in `template.eval(conn, 'ls\nshow version')`. Before, such a line was ignored. -- **Unicode Escape Sequences Above U+10FFFF:** A sequence such as `\UFFFFFFFF` in a - command or a string raises a `CompileError` that shows its position. Before, it - raised a `SyntaxError` of Python. +- **BREAKING - Escape Sequences Out of Range:** A Unicode sequence above U+10FFFF, + such as `\UFFFFFFFF`, and an octal sequence above `\377`, such as `\400`, in a + command or a string raise a `CompileError` that shows their position. Before, the + former raised a `SyntaxError` of Python, and the latter gave the characters U+0100 + to U+01FF with a `SyntaxWarning` of Python. - **BREAKING - `extract` Requires a Regular Expression:** `extract` raises a `CompileError` if no regular expression between slashes follows, e.g. in `{extract "(a)" as y from "a"}` or `{extract x as y}`. Before, the character after diff --git a/src/Exscript/interpreter/string.py b/src/Exscript/interpreter/string.py index 88b476f8..60f54e60 100644 --- a/src/Exscript/interpreter/string.py +++ b/src/Exscript/interpreter/string.py @@ -142,7 +142,8 @@ def eval_escape(self, ttype: str, token: str) -> str: evaluated. An escaped dollar sign stays escaped for the substitution of variables; any other escaped character is returned without the backslash. Raises a CompileError at the current position of the - lexer for a Unicode sequence above U+10FFFF. + lexer for a Unicode sequence above U+10FFFF or an octal sequence + above \\377. :param ttype: The token type of the escape sequence. :param token: The escape sequence. @@ -163,6 +164,12 @@ def eval_escape(self, ttype: str, token: str) -> str: self.mark_end(distance=len(token)) msg = 'Invalid escape sequence %s: above U+10FFFF' % repr(token) self.lexer.syntax_error(msg, self) + # Python warns about these, and will reject them. + if ttype == 'escaped_octal' and int(token[1:], 8) > 0o377: + self.mark_start() + self.mark_end(distance=len(token)) + msg = 'Invalid escape sequence %s: octal value above 377' % repr(token) + self.lexer.syntax_error(msg, self) return cast(str, ast.literal_eval('"' + token + '"')) if ttype == 'escaped_common_controls': return cast(str, ast.literal_eval('"' + token + '"')) diff --git a/src/Exscript/interpreter/template.py b/src/Exscript/interpreter/template.py index 537e0fcf..c41c8731 100644 --- a/src/Exscript/interpreter/template.py +++ b/src/Exscript/interpreter/template.py @@ -163,7 +163,8 @@ def eval_escape(self, ttype: str, token: str) -> str: unless the parser disables escaping. An escaped dollar sign stays escaped for the Execute token; any other escaped character is returned without the backslash. Raises a CompileError at the current - position of the lexer for a Unicode sequence above U+10FFFF. + position of the lexer for a Unicode sequence above U+10FFFF or an + octal sequence above \\377. :param ttype: The token type of the escape sequence. :param token: The escape sequence. @@ -185,6 +186,12 @@ def eval_escape(self, ttype: str, token: str) -> str: self.mark_end(distance=len(token)) msg = 'Invalid escape sequence %s: above U+10FFFF' % repr(token) self.lexer.syntax_error(msg, self) + # Python warns about these, and will reject them. + if ttype == 'escaped_octal' and int(token[1:], 8) > 0o377: + self.mark_start() + self.mark_end(distance=len(token)) + msg = 'Invalid escape sequence %s: octal value above 377' % repr(token) + self.lexer.syntax_error(msg, self) return cast(str, ast.literal_eval('"' + token + '"')) # Simple escape sequence with one character. if len(token) == 2: diff --git a/tests/Exscript/interpreter/CommandTest.py b/tests/Exscript/interpreter/CommandTest.py index 87600996..acb4c174 100644 --- a/tests/Exscript/interpreter/CommandTest.py +++ b/tests/Exscript/interpreter/CommandTest.py @@ -148,6 +148,15 @@ def testUnicodeEscapeSequenceAboveTheRange(self): self.evaluate('show \\UFFFFFFFF\n', no_escaping=True) self.assertSent('show UFFFFFFFF') + def testOctalEscapeSequenceAboveTheRange(self): + with self.assertRaises(CompileError) as cm: + self.evaluate('ls\nshow \\777\n') + message = "Invalid escape sequence '\\\\777': octal value above 377 in undefined:2" + self.assertError(cm.exception, ['show \\777'], ' ' + '^' * 4, message) + self.assertSent() + self.evaluate('show \\400\n', no_escaping=True) + self.assertSent('show 400') + def testEscapedSpecialCharacters(self): self.evaluate('show \\{a\\} \\\\ \\$x \\# \\"\n') self.assertSent('show {a} \\ $x # "') diff --git a/tests/Exscript/interpreter/ParserPropertyTest.py b/tests/Exscript/interpreter/ParserPropertyTest.py index 46cc22e2..023b208a 100644 --- a/tests/Exscript/interpreter/ParserPropertyTest.py +++ b/tests/Exscript/interpreter/ParserPropertyTest.py @@ -63,6 +63,7 @@ '\\u00e9', '\\U0010FFFF', '\\U00110000', + '\\777', 'if ', 'else', 'end', diff --git a/tests/Exscript/interpreter/StringTest.py b/tests/Exscript/interpreter/StringTest.py index f753b5c7..be83298d 100644 --- a/tests/Exscript/interpreter/StringTest.py +++ b/tests/Exscript/interpreter/StringTest.py @@ -32,6 +32,14 @@ def testUnicodeEscapeSequenceAboveTheRange(self): self.assertError(cm.exception, ['{x = "a\\U00110000"}'], ' ' * 7 + '^' * 10, message) self.assertString('\\U00110000', 'U00110000', no_escaping=True) + def testOctalEscapeSequenceAboveTheRange(self): + with self.assertRaises(CompileError) as cm: + self.evaluate('{x = "a\\400"}') + message = "Invalid escape sequence '\\\\400': octal value above 377 in undefined:1" + self.assertError(cm.exception, ['{x = "a\\400"}'], ' ' * 7 + '^' * 4, message) + self.assertString('\\377', '\xff') + self.assertString('\\777', '777', no_escaping=True) + def testVariables(self): variables = self.evaluate( '{a = "1"}{b = list.new("x", "y")}{n = 2 * 3}{e = list.new()}' From 46a111a3117cfe252270831a42d4867feeaaede8 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 10 Nov 2026 18:40:46 +0100 Subject: [PATCH 129/250] Use values that are not lists like lists in operators and statements Terms can have other values, e.g. True from a function or a tuple from the caller, which raised an UnboundLocalError or a TypeError there. The new as_list() takes a tuple as a list and other values as one item; functions and regular expressions raise an ExecuteError. --- CHANGES.md | 19 +++- docs/migration.rst | 5 + src/Exscript/interpreter/append.py | 6 +- src/Exscript/interpreter/expression.py | 92 ++++++++++++------- src/Exscript/interpreter/extract.py | 5 +- src/Exscript/interpreter/fail.py | 2 +- src/Exscript/interpreter/loop.py | 6 +- tests/Exscript/interpreter/AssignTest.py | 12 ++- tests/Exscript/interpreter/ExpressionTest.py | 31 +++++++ tests/Exscript/interpreter/ExtractTest.py | 6 ++ tests/Exscript/interpreter/FailTest.py | 5 + tests/Exscript/interpreter/IfConditionTest.py | 11 ++- tests/Exscript/interpreter/LoopTest.py | 16 ++++ tests/Exscript/interpreter/StringTest.py | 23 +++++ 14 files changed, 189 insertions(+), 50 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 172027da..f203ef88 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -456,10 +456,21 @@ development versions; "before" refers to them. matches numbers as text, as `matches` does, e.g. in `{extract /(\d)/ as d from 42}` (before: `TypeError`); and the bounds of `loop from ... to ...` may be strings of digits, like the operands of arithmetic operators. -- **Regular Expressions as Operands:** A regular expression that is not the right - operand of `matches`, e.g. in `/a/ matches /a/`, `/a/ . "b"`, `"a" is /a/` or - `not /a/`, raises an `ExecuteError` that shows its position. Before, it raised an - `UnboundLocalError` or a `TypeError` of Python. +- **BREAKING - Values That Are Not Lists:** Operators and statements use a value + that is not a list like a list: a tuple as a list, and any other value, e.g. the + `True` that a function returns without a result, a string or a dict from the + caller, or the connection, as a list with this one item; a function raises an + `ExecuteError`, as in a string. Before, such values raised an `UnboundLocalError` + or a `TypeError`, except that `in`, `fail`, `loop` and `extract` used the + characters of a string, and `in` and `extract` the keys of a dict, e.g. + `"err" in out()` was a substring test if the function `out` returned a string. +- **Regular Expressions as Operands and Values:** A regular expression that is not + the right operand of `matches`, e.g. in `/a/ matches /a/`, `/a/ . "b"`, + `"a" is /a/` or `not /a/`, raises an `ExecuteError` that shows its position, also + as the condition of `if`, `fail` or `loop`, as a bound or the list of `loop`, as + the source of `extract`, as the value of `append` and as the message of `fail`. + Before, it raised an `UnboundLocalError` or a `TypeError` of Python. Assignments + and functions still take regular expressions. - **Error Positions in Commands and Strings:** An error about a variable in a command, a string or a regular expression points to the variable also behind escape sequences, e.g. to `$nothing` in `show \x41 $nothing`, and in a command right before diff --git a/docs/migration.rst b/docs/migration.rst index 46d0f104..848835b3 100644 --- a/docs/migration.rst +++ b/docs/migration.rst @@ -131,6 +131,11 @@ Operators result, as documented. 2.x always evaluated both, e.g. called the functions in both. +Values of the caller + A string that a function of the caller returns, or a dict that the + caller passes, is one item: ``in``, ``loop`` and ``extract`` no longer + use its characters or keys. Return and pass lists instead. + Command line tool ----------------- diff --git a/src/Exscript/interpreter/append.py b/src/Exscript/interpreter/append.py index 977871c4..4d8542bc 100644 --- a/src/Exscript/interpreter/append.py +++ b/src/Exscript/interpreter/append.py @@ -27,7 +27,7 @@ from typing import TYPE_CHECKING from ..parselib import Lexer, Token -from .expression import Expression +from .expression import Expression, as_list from .scope import Scope if TYPE_CHECKING: @@ -83,8 +83,8 @@ def value(self, context: Scope) -> int: :param context: The scope in which the template runs. :return: 1. """ - existing = self.parent.get(self.varname) - args = {self.varname: existing + self.expr.value(context)} + existing = as_list(self.parent.get(self.varname), self, self.varname) + args = {self.varname: existing + self.expr.values(context)} self.parent.define(**args) return 1 diff --git a/src/Exscript/interpreter/expression.py b/src/Exscript/interpreter/expression.py index a0f13247..a6813d24 100644 --- a/src/Exscript/interpreter/expression.py +++ b/src/Exscript/interpreter/expression.py @@ -36,6 +36,36 @@ from .parser import Parser +# Any: the value of a term, an expression or a variable, and the items of +# the list. +def as_list(value: Any, token: Token, name: str | None = None) -> list[Any]: + """ + Returns the given value as a list, as the operators and statements use + it: a list as it is, a tuple as a list, and any other value, such as the + True that a function returns without a result, as a list with this one + item. Raises an ExecuteError at the given token if the value is a + regular expression or a function. + + :param value: The value. + :param token: The term or the expression whose value it is, or the + statement that reads it from a variable. + :param name: The name of the variable for the error message about a + function; by default the text of the token. + """ + if isinstance(value, list): + return value + if isinstance(value, tuple): + return list(value) + if isinstance(value, re.Pattern): + msg = "A regular expression is only allowed as the right operand of 'matches'" + token.lexer.runtime_error(msg, token) + if callable(value): + if name is None: + name = token.lexer.input[token.start : token.end] + token.lexer.runtime_error('%s is a function, not a variable name' % repr(name), token) + return [value] + + class ExpressionNode(Token): """ A node of an expression. The constructor parses a chain of nodes: a @@ -125,25 +155,16 @@ def priority(self) -> int: else: raise Exception('Invalid operator: ' + repr(self.op)) - # Any: the value of an operand. - def _reject_regex(self, value: Any, operand: Token) -> None: - """ - Raises an ExecuteError at the given operand if its given value is a - regular expression. Only "matches" accepts one, as its right operand. - """ - if isinstance(value, re.Pattern): - msg = "A regular expression is only allowed as the right operand of 'matches'" - self.lexer.runtime_error(msg, operand) - # Any: the value of a term, or a list with the result of the operator. def value(self, context: Scope) -> Any: """ - Evaluates the node. A list operand stands for its first item, or '' - if it is empty. "and" and "or" evaluate the right operand only if - the left one does not decide the result. Arithmetic operators and - ge, gt, le and lt raise an ExecuteError if an operand is not a - number, and / and % if the right operand is 0. A regular expression - raises an ExecuteError unless it is the right operand of "matches". + Evaluates the node. An operand is used as a list, see as_list(), and + stands for its first item, or '' if it is empty. "and" and "or" + evaluate the right operand only if the left one does not decide the + result. Arithmetic operators and ge, gt, le and lt raise an + ExecuteError if an operand is not a number, and / and % if the right + operand is 0. A regular expression raises an ExecuteError unless it + is the right operand of "matches". :param context: The scope in which the template runs. :return: The value of the term if the node has no operator, or a @@ -156,31 +177,22 @@ def value(self, context: Scope) -> Any: return cast(Token, self.lft).value(context) elif self.op == 'not': rgt_lst = cast(Token, self.rgt).value(context) - self._reject_regex(rgt_lst, cast(Token, self.rgt)) + rgt_lst = as_list(rgt_lst, cast(Token, self.rgt)) return [not (rgt_lst[0] if len(rgt_lst) > 0 else '')] - # There are only two types of values: Regular expressions and lists. - # We also have to make sure that empty lists do not cause an error. - # The first item of a list can be a string, a number or anything else - # that a function returned, hence Any. - lft_lst = cast(Token, self.lft).value(context) - self._reject_regex(lft_lst, cast(Token, self.lft)) - if type(lft_lst) == type([]): - if len(lft_lst) > 0: - lft: Any = lft_lst[0] - else: - lft = '' + # The operands are lists, except the regular expression on the right of + # "matches". We also have to make sure that empty lists do not cause an + # error. The first item of a list can be a string, a number or anything + # else that a function returned, hence Any. + lft_lst = as_list(cast(Token, self.lft).value(context), cast(Token, self.lft)) + lft: Any = lft_lst[0] if len(lft_lst) > 0 else '' # The left operand of "and" and "or" may decide the result alone. if (self.op == 'and' and not lft) or (self.op == 'or' and lft): return [lft] rgt_lst = cast(Token, self.rgt).value(context) if self.op != 'matches': - self._reject_regex(rgt_lst, cast(Token, self.rgt)) - if type(rgt_lst) == type([]): - if len(rgt_lst) > 0: - rgt: Any = rgt_lst[0] - else: - rgt = '' + rgt_lst = as_list(rgt_lst, cast(Token, self.rgt)) + rgt: Any = rgt_lst[0] if isinstance(rgt_lst, list) and len(rgt_lst) > 0 else '' if self.op in ('*', '/', '%', '+', '-', 'ge', 'gt', 'le', 'lt'): error = 'Operand for %s is not a number' % (self.op) @@ -337,6 +349,16 @@ def value(self, context: Scope) -> Any: """ return self.root.value(context) + # Any: the items of the value. + def values(self, context: Scope) -> list[Any]: + """ + Evaluates the expression, and returns its value as a list, see + as_list(). + + :param context: The scope in which the template runs. + """ + return as_list(self.value(context), self.root) + def is_true(self, context: Scope) -> bool: """ Evaluates the expression as a condition, which is true if the first @@ -344,7 +366,7 @@ def is_true(self, context: Scope) -> bool: :param context: The scope in which the template runs. """ - values = self.value(context) + values = self.values(context) return len(values) > 0 and bool(values[0]) def dump(self, indent: int = 0) -> None: diff --git a/src/Exscript/interpreter/extract.py b/src/Exscript/interpreter/extract.py index 2a8e1376..a42f3400 100644 --- a/src/Exscript/interpreter/extract.py +++ b/src/Exscript/interpreter/extract.py @@ -27,6 +27,7 @@ from typing import TYPE_CHECKING from ..parselib import Lexer, Token +from .expression import as_list from .regex import Regex from .scope import Scope from .term import Term @@ -154,7 +155,7 @@ def extract(self, context: Scope) -> None: if self.source is None: buffer = self.parent.get('__response__') else: - buffer = self.source.value(context) + buffer = as_list(self.source.value(context), self.source) # print("Buffer contains", buffer) # Walk through all lines, matching each one against the regular @@ -192,7 +193,7 @@ def value(self, context: Scope) -> int: self.parent.define(**self.variables) else: for key in self.variables: - existing = self.parent.get(key) + existing = as_list(self.parent.get(key), self, key) self.parent.define(**{key: existing + self.variables[key]}) return 1 diff --git a/src/Exscript/interpreter/fail.py b/src/Exscript/interpreter/fail.py index 505f0664..eb55fd37 100644 --- a/src/Exscript/interpreter/fail.py +++ b/src/Exscript/interpreter/fail.py @@ -83,7 +83,7 @@ def value(self, context: Scope) -> int: :return: 1 if the condition is false. """ if self.expression is None or self.expression.is_true(context): - msg = self.msg.value(context) + msg = self.msg.values(context) raise FailException(msg[0] if len(msg) > 0 else '') return 1 diff --git a/src/Exscript/interpreter/loop.py b/src/Exscript/interpreter/loop.py index 399ab1dd..20c2da6e 100644 --- a/src/Exscript/interpreter/loop.py +++ b/src/Exscript/interpreter/loop.py @@ -29,7 +29,7 @@ import Exscript.interpreter.code from ..parselib import Lexer, Token -from .expression import Expression +from .expression import Expression, as_list from .scope import Scope from .term import Term @@ -171,14 +171,14 @@ def value(self, context: Scope) -> int: bounds = [] # The constructor sets theto together with thefrom. for bound in self.thefrom, cast(Expression, self.theto): - values = bound.value(context) + values = bound.values(context) try: bounds.append(int(values[0] if len(values) > 0 else '')) except (TypeError, ValueError): self.lexer.runtime_error('Loop bound is not a number', bound.root) lists = [list(range(*bounds))] else: - lists = [var.value(context) for var in self.list_variables] + lists = [as_list(var.value(context), var) for var in self.list_variables] varnames = self.iter_varnames # Make sure that all lists have the same length. diff --git a/tests/Exscript/interpreter/AssignTest.py b/tests/Exscript/interpreter/AssignTest.py index 07536945..5b9417e4 100644 --- a/tests/Exscript/interpreter/AssignTest.py +++ b/tests/Exscript/interpreter/AssignTest.py @@ -1,6 +1,6 @@ import unittest -from Exscript.parselib.exception import CompileError +from Exscript.parselib.exception import CompileError, ExecuteError from .helpers import TemplateTestCase @@ -77,6 +77,16 @@ def testAppendToAVariableOfTheCaller(self): variables = self.evaluate('{append "b" to x}', x=['a']) self.assertEqual(variables['x'], ['a', 'b']) + def testValuesThatAreNotLists(self): + variables = self.evaluate( + '{append t to x}{append yes() to x}', t=('b', 'c'), x=('a',), yes=lambda scope: True + ) + self.assertEqual(variables['x'], ['a', 'b', 'c', True]) + with self.assertRaises(ExecuteError) as cm: + self.evaluate('{append 1 to f}', f=lambda scope: [1]) + message = "'f' is a function, not a variable name in undefined:1" + self.assertError(cm.exception, ['{append 1 to f}'], ' ' + '^' * 13, message) + def testAppendInALoop(self): variables = self.evaluate('{loop from 0 to 3 as i}{append i * 2 to doubles}{end}') self.assertEqual(variables['doubles'], [0, 2, 4]) diff --git a/tests/Exscript/interpreter/ExpressionTest.py b/tests/Exscript/interpreter/ExpressionTest.py index 38215f15..c06c3dcd 100644 --- a/tests/Exscript/interpreter/ExpressionTest.py +++ b/tests/Exscript/interpreter/ExpressionTest.py @@ -128,6 +128,37 @@ def testRegularExpressionAsAnotherOperand(self): ) self.assertError(cm.exception, [text], marker, message) + def testOperandsThatAreNotLists(self): + # Functions return True without a result, the caller may pass a tuple, + # and the connection is an object; a string is one item. + text = ( + '{a = yes() and "x"}{b = t . "c"}{c = "b" in t}{d = not t}' + '{e = __connection__ is __connection__}{f = not __connection__}' + '{g = two() * 3}{h = "a" in word()}{i = word() matches /b/}' + ) + variables = self.evaluate( + text, + yes=lambda scope: True, + t=('a', 'b'), + two=lambda scope: 2, + word=lambda scope: 'ab', + ) + actual = [variables[name] for name in 'abcdefghi'] + expected = [['x'], ['ac'], [True], [False], [True], [False], [6], [False], [1]] + self.assertEqual(actual, expected) + + def testFunctionAsAnOperand(self): + for text, marker in ( + ('{x = f . "a"}', ' ^'), + ('{x = "a" is f}', ' ' * 12 + '^'), + ('{x = not f}', ' ' * 9 + '^'), + ): + with self.subTest(text=text): + with self.assertRaises(ExecuteError) as cm: + self.evaluate(text, f=lambda scope: ['b']) + message = "'f' is a function, not a variable name in undefined:1" + self.assertError(cm.exception, [text], marker, message) + def testLogicalOperators(self): text = '{a = 1 and 0}{b = 0 or 1}{c = not 0}{d = not 1}{e = 1 and 1 and 0}' self.assertValues(text, a=[0], b=[1], c=[True], d=[False], e=[0]) diff --git a/tests/Exscript/interpreter/ExtractTest.py b/tests/Exscript/interpreter/ExtractTest.py index f1e22101..eed1933c 100644 --- a/tests/Exscript/interpreter/ExtractTest.py +++ b/tests/Exscript/interpreter/ExtractTest.py @@ -61,6 +61,12 @@ def testExtractFromNumbers(self): ) self.assertEqual((variables['a'], variables['b']), (['4'], ['7', '3'])) + def testValuesThatAreNotLists(self): + # A string is one item, and the variable of the caller may be a tuple. + text = '{extract /(\\d)/ as a from two()}{extract /(.)/ into t from word()}' + variables = self.evaluate(text, two=lambda scope: 2, word=lambda scope: 'ab', t=('x',)) + self.assertEqual((variables['a'], variables['t']), (['2'], ['x', 'a'])) + def testNoMatch(self): variables = self.evaluate('ls{extract /(nomatch)/ as found}\n') self.assertEqual(variables['found'], []) diff --git a/tests/Exscript/interpreter/FailTest.py b/tests/Exscript/interpreter/FailTest.py index 7530a3e9..84023bc0 100644 --- a/tests/Exscript/interpreter/FailTest.py +++ b/tests/Exscript/interpreter/FailTest.py @@ -40,6 +40,11 @@ def testEmptyList(self): self.evaluate('{fail list.new()}') self.assertEqual(str(cm.exception), '') + def testValuesThatAreNotLists(self): + with self.assertRaises(FailException) as cm: + self.evaluate('{fail word() if yes()}', word=lambda scope: 'ab', yes=lambda scope: True) + self.assertEqual(str(cm.exception), 'ab') + def testInABlock(self): with self.assertRaises(FailException) as cm: self.evaluate('{loop from 0 to 5 as i}\nls $i\n{fail "stop at " . i if i is 2}{end}') diff --git a/tests/Exscript/interpreter/IfConditionTest.py b/tests/Exscript/interpreter/IfConditionTest.py index 5bd7935d..8ca94720 100644 --- a/tests/Exscript/interpreter/IfConditionTest.py +++ b/tests/Exscript/interpreter/IfConditionTest.py @@ -1,6 +1,6 @@ import unittest -from Exscript.parselib.exception import CompileError +from Exscript.parselib.exception import CompileError, ExecuteError from .helpers import TemplateTestCase @@ -45,6 +45,15 @@ def testEmptyListIsFalse(self): self.evaluate(text) self.assertSent('show two') + def testConditionsThatAreNotLists(self): + text = '{if yes()}\nshow one\n{end}{if __connection__}\nshow two\n{end}{if t}\nls\n{end}' + self.evaluate(text, yes=lambda scope: True, t=('a',)) + self.assertSent('show one', 'show two', 'ls') + with self.assertRaises(ExecuteError) as cm: + self.evaluate('{if f}ls{end}', f=lambda scope: [1]) + message = "'f' is a function, not a variable name in undefined:1" + self.assertError(cm.exception, ['{if f}ls{end}'], ' ^', message) + def testOnOneLine(self): self.evaluate('{if 1}ls{end}\n{if 0}show version{else}show other{end}\n') self.assertSent('ls', 'show other') diff --git a/tests/Exscript/interpreter/LoopTest.py b/tests/Exscript/interpreter/LoopTest.py index 8b31221f..c9f46ebe 100644 --- a/tests/Exscript/interpreter/LoopTest.py +++ b/tests/Exscript/interpreter/LoopTest.py @@ -92,6 +92,22 @@ def testConditionsOnAnEmptyList(self): ) self.assertEqual(self.evaluate(text)['n'], [202]) + def testValuesThatAreNotLists(self): + text = ( + '{loop t as i}{append i to l}{end}' + '{loop yes() as i}{append i to l}{end}' + '{loop from 1 to two() as i}{append i to l}{end}' + '{loop while yes() and n lt 2}{n = n + 1}{end}' + ) + variables = self.evaluate( + text, t=('a', 'b'), yes=lambda scope: True, two=lambda scope: 2, n=0 + ) + self.assertEqual((variables['l'], variables['n']), (['a', 'b', True, 1], [2])) + with self.assertRaises(ExecuteError) as cm: + self.evaluate('{loop f as i}{end}', f=lambda scope: [1]) + message = "'f' is a function, not a variable name in undefined:1" + self.assertError(cm.exception, ['{loop f as i}{end}'], ' ^', message) + def testLoopWhile(self): variables = self.evaluate('{n = 0}{loop while n lt 3}{n = n + 1}\nls $n\n{end}') self.assertSent('ls 1', 'ls 2', 'ls 3') diff --git a/tests/Exscript/interpreter/StringTest.py b/tests/Exscript/interpreter/StringTest.py index be83298d..7bde0327 100644 --- a/tests/Exscript/interpreter/StringTest.py +++ b/tests/Exscript/interpreter/StringTest.py @@ -132,6 +132,29 @@ def testUndefinedVariableAfterAnEscapedSlash(self): message = "Undefined variable 'nothing' in undefined:1" self.assertError(cm.exception, [text], ' ' * 20 + '^' * 8, message) + def testRegularExpressionAsTheValueOfAStatement(self): + for text, column in ( + ('{if /a/}ls{end}', 4), + ('{fail "x" if /a/}', 13), + ('{fail /a/}', 6), + ('{loop while /a/}{end}', 12), + ('{loop until /a/}{end}', 12), + ('{loop from /a/ to 3}{end}', 11), + ('{loop from 1 to /a/}{end}', 16), + ('{loop /a/ as x}{end}', 6), + ('{extract /(a)/ as x from /a/}', 25), + ('{append /a/ to x}', 8), + ): + with self.subTest(text=text): + with self.assertRaises(ExecuteError) as cm: + self.evaluate(text) + message = ( + "A regular expression is only allowed as the right operand of 'matches'" + ' in undefined:1' + ) + self.assertError(cm.exception, [text], ' ' * column + '^^^', message) + self.assertSent() + def testInvalidModifier(self): with self.assertRaises(CompileError) as cm: self.evaluate('{x = /abc/x}') From ddd5ef7703308a8bd7f6566f66cc146e6c0790e8 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 10 Nov 2026 18:44:47 +0100 Subject: [PATCH 130/250] Accept several blanks between the words of "is not" and "not in" The grammar accepts any whitespace there, but the operator kept it, so two blanks or a line break between the words matched no operator. Replace the whitespace with one blank. --- CHANGES.md | 3 +++ src/Exscript/interpreter/expression.py | 2 ++ tests/Exscript/interpreter/ExpressionTest.py | 7 +++++++ 3 files changed, 12 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index f203ef88..75a9c917 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -442,6 +442,9 @@ development versions; "before" refers to them. `or` not if the left one is true; e.g. `{x = 0 and list.get(list.new(), 1)}` sets `x` to 0. Before, both operands were always evaluated, so the right one could raise an error or call a function, e.g. one that sends a command. +- **`is not` and `not in` With Several Blanks:** `is not` and `not in` with more + than one blank or a line break between the words mean the same as `is not` and + `not in`. Before, they gave `None`, and as a condition a `TypeError` of Python. - **BREAKING - Lines With Only Whitespace:** A line with only whitespace is ignored, also if the whitespace is not a space or a tab, e.g. a no-break space. Before, such whitespace was sent at the start of the command in the next line. diff --git a/src/Exscript/interpreter/expression.py b/src/Exscript/interpreter/expression.py index a6813d24..6144d5b8 100644 --- a/src/Exscript/interpreter/expression.py +++ b/src/Exscript/interpreter/expression.py @@ -124,6 +124,8 @@ def __init__(self, lexer: Lexer, parser: 'Parser', parent: Scope) -> None: self.mark_end(distance=len(self.op)) msg = 'Expected operator but got %s %s' % (self.op_type, repr(self.op)) lexer.syntax_error(msg, self) + # The words of "is not" and "not in" may be separated by any whitespace. + self.op = ' '.join(self.op.split()) # Expect the second term. self.rgt = ExpressionNode(lexer, parser, parent) diff --git a/tests/Exscript/interpreter/ExpressionTest.py b/tests/Exscript/interpreter/ExpressionTest.py index c06c3dcd..44a2c061 100644 --- a/tests/Exscript/interpreter/ExpressionTest.py +++ b/tests/Exscript/interpreter/ExpressionTest.py @@ -98,6 +98,13 @@ def testIn(self): ) self.assertValues(text, a=[True], b=[False], c=[True], d=[False]) + def testIsNotAndNotInWithSeveralBlanks(self): + text = ( + '{l = list.new("a")}' + '{a = 1 is not 2}{b = 1 is\n not 1}{c = "c" not \t in l}{d = "a" not\nin l}' + ) + self.assertValues(text, a=[True], b=[False], c=[True], d=[False]) + def testMatches(self): text = ( '{a = "abc" matches /b/}{b = "abc" matches /^b/}{c = "ABC" matches /b/i}' From 48a912a6d3542295ca9980b567f49f9bfa1b4085 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 10 Nov 2026 18:54:22 +0100 Subject: [PATCH 131/250] Start no comment behind escaped characters The Template started a comment behind an escaped "}", and Execute removed lines that started with an escaped hash, e.g. "\# x". The lookbehind of line_comment now excludes "\}", and Execute removes comments only in the body of an RPC. --- CHANGES.md | 6 ++++++ docs/templates.rst | 4 ++++ src/Exscript/interpreter/execute.py | 24 ++++++++++++++--------- src/Exscript/interpreter/template.py | 4 +++- tests/Exscript/interpreter/CommandTest.py | 21 ++++++++++++++++++++ tests/Exscript/interpreter/RpcTest.py | 6 ++++++ 6 files changed, 55 insertions(+), 10 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 75a9c917..b58353d7 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -448,6 +448,12 @@ development versions; "before" refers to them. - **BREAKING - Lines With Only Whitespace:** A line with only whitespace is ignored, also if the whitespace is not a space or a tab, e.g. a no-break space. Before, such whitespace was sent at the start of the command in the next line. +- **BREAKING - Escaped Characters Start No Comment:** An escaped hash, or a hash + behind an escaped character, is part of the command: `\# x` sends `# x`, + `show \}# x` sends `show }# x`, and `show a\n# b` sends `# b` in its second line. + Before, the hash started a comment there, so `\# x` sent an empty line, and + `show \}# x` took the next line into the command. A hash behind an `` element + is part of the command as well; before, it sent an empty line. - **Errors at Runtime:** A division by zero (`1 / 0`, `1 % 0`), an operand of `/`, `ge`, `gt`, `le` or `lt` that is not a number (`"a" / 2`), an operand of an arithmetic operator without a value, such as a group that `extract` did not match, diff --git a/docs/templates.rst b/docs/templates.rst index 6235fdd6..5dafd3e0 100644 --- a/docs/templates.rst +++ b/docs/templates.rst @@ -23,6 +23,10 @@ ignored. For example:: 3. # ...and so is this one. 4. {end} +A hash behind a code block also starts a comment that ends with the line, +e.g. in ``{x = 1} # set x``. Escaped characters are text: ``\# text`` +sends the command ``# text``, and ``show \}# x`` sends ``show }# x``. + Using Variables --------------- diff --git a/src/Exscript/interpreter/execute.py b/src/Exscript/interpreter/execute.py index be46f1ff..bf33daab 100644 --- a/src/Exscript/interpreter/execute.py +++ b/src/Exscript/interpreter/execute.py @@ -44,6 +44,7 @@ } leading_whitespace_re = re.compile(r'^[ \t\r\n]*') line_comment_re = re.compile(grammar['line_comment']) +blank_lines_re = re.compile(grammar['leading_blank_lines'] + '|' + grammar['trailing_blank_lines']) strip_re = re.compile('|'.join(grammar.values())) @@ -84,6 +85,10 @@ def __init__( self.string = command # Original command will be evaluated at runtime. self.no_prompt = parser.no_prompt self.strip_command = parser.strip_command + # Only the body of an RPC contains line comments. The Template skips + # those of the other commands, so a hash in them, e.g. behind an + # escaped line break, is part of the command. + self.in_rpc = type(parent).__name__ == 'RPCBody' # The lexer has parsed the command and consumed the trailing command delimiter: # a newline or the closing RPC tag, and nothing before a code block, an RPC or @@ -104,7 +109,8 @@ def __init__( # NOTE: The substitution test might also fire in line comments with variables. # This is prevented by making the line comments "invisible" while preserving # the overall command length/structure for proper error tracking. - command = line_comment_re.sub(self.invisible_sub_cb, command) + if self.in_rpc: + command = line_comment_re.sub(self.invisible_sub_cb, command) varstring_re.sub(self.variable_test_cb, command) # Validate XML for RPCBody tokens. We need to use the original command @@ -112,7 +118,7 @@ def __init__( # Additionally we need to replace all remaining variable chars with "_" # to create an XML tree as valid as possible. The XML body will be # tested once more at runtime by the protocol adapter (evaluated vars). - if type(parent).__name__ == 'RPCBody': + if self.in_rpc: from lxml.etree import XMLSyntaxError # only templates with RPCs need lxml # The start mark points to the first non-whitespace character of the @@ -168,12 +174,12 @@ def __init__( def value(self, context: Scope) -> int: """ - Sends the command, without line comments, surrounding blank lines - and common indentation, and with the variables replaced by their - values. Waits for the prompt unless the parser was created with - no_prompt. Stores the response in the __response__ variable, one - line per item. Raises an ExecuteError if the __connection__ variable - is not defined. + Sends the command, without surrounding blank lines, common + indentation and, in an element, line comments, and with the + variables replaced by their values. Waits for the prompt unless the + parser was created with no_prompt. Stores the response in the + __response__ variable, one line per item. Raises an ExecuteError if + the __connection__ variable is not defined. :param context: The scope in which the template runs. :return: 1. @@ -186,7 +192,7 @@ def value(self, context: Scope) -> int: # Prepare the command for execution by stripping unwanted substrings. # The dedent of common indentations is useful for multiline commands. - command = strip_re.sub('', self.string) + command = (strip_re if self.in_rpc else blank_lines_re).sub('', self.string) command = textwrap.dedent(command) # Substitute variables in the command for values. diff --git a/src/Exscript/interpreter/template.py b/src/Exscript/interpreter/template.py index c41c8731..31619207 100644 --- a/src/Exscript/interpreter/template.py +++ b/src/Exscript/interpreter/template.py @@ -49,7 +49,9 @@ grammar = escaped_grammar + [ ('escaped_data', r'\\.'), - ('line_comment', r'(?:^|(?<=[\r\n}]))[ \t]*#[^\r\n]*(?:[\r\n]|$)'), + # A hash starts a comment at the start of a line and behind a code block, + # but not behind an escaped closing curly bracket. + ('line_comment', r'(?:^|(?<=[\r\n])|(?<=})(?'), diff --git a/tests/Exscript/interpreter/CommandTest.py b/tests/Exscript/interpreter/CommandTest.py index acb4c174..ae3086fc 100644 --- a/tests/Exscript/interpreter/CommandTest.py +++ b/tests/Exscript/interpreter/CommandTest.py @@ -38,6 +38,27 @@ def testCommentsAreNotSent(self): self.evaluate(text) self.assertSent('ls', 'show version # not a comment') + def testEscapedCharactersStartNoComment(self): + # A hash behind an escaped character is part of the command, also + # behind an escaped closing curly bracket or line break. + text = ( + 'show \\}# x\n' + 'ls\n' + '\\# not a comment\n' + ' \\# $n\n' + 'show a\\n# b\n' + '{x = 1} # a comment \\\n' + 'ls\n' + ) + self.evaluate(text, n='1') + sent = ('show }# x', 'ls', '# not a comment', '# 1', 'show a\n# b', 'ls') + self.assertSent(*sent) + with self.assertRaises(ExecuteError) as cm: + self.evaluate('ls\n\\# $nothing\n') + message = "Undefined variable 'nothing' in undefined:2" + self.assertError(cm.exception, ['\\# $nothing'], ' ' + '^' * 8, message) + self.assertSent(*sent) + def testLastLineWithoutLineBreak(self): variables = self.evaluate('ls\nshow version') self.assertSent('ls', 'show version') diff --git a/tests/Exscript/interpreter/RpcTest.py b/tests/Exscript/interpreter/RpcTest.py index 32e303b1..e119066d 100644 --- a/tests/Exscript/interpreter/RpcTest.py +++ b/tests/Exscript/interpreter/RpcTest.py @@ -35,6 +35,12 @@ def testCommentsAreNotSent(self): self.evaluate('\n # a comment\n \n # another comment\n \n\n') self.assertSent('\n') + def testHashBehindAnRpc(self): + # Only lines starting with a hash, and the text behind a code block, + # are comments. + self.evaluate(' # not a comment\nls\n') + self.assertSent('', '# not a comment', 'ls') + def testVariables(self): text = '<$source/>\n' self.evaluate(text, source='candidate') From ff4a195d087cf7d8a82e4885a81e14e059b73253 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 10 Nov 2026 19:30:45 +0100 Subject: [PATCH 132/250] Start a comment behind an RPC as behind a code block A hash behind an element was sent as part of a command. The lookbehind of line_comment in the Template now also accepts "". --- CHANGES.md | 2 +- docs/templates.rst | 7 ++++--- src/Exscript/interpreter/template.py | 6 +++--- tests/Exscript/interpreter/RpcTest.py | 9 ++++----- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index b58353d7..c5a20d31 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -453,7 +453,7 @@ development versions; "before" refers to them. `show \}# x` sends `show }# x`, and `show a\n# b` sends `# b` in its second line. Before, the hash started a comment there, so `\# x` sent an empty line, and `show \}# x` took the next line into the command. A hash behind an `` element - is part of the command as well; before, it sent an empty line. + starts a comment, as behind a code block; before, it sent an empty line. - **Errors at Runtime:** A division by zero (`1 / 0`, `1 % 0`), an operand of `/`, `ge`, `gt`, `le` or `lt` that is not a number (`"a" / 2`), an operand of an arithmetic operator without a value, such as a group that `extract` did not match, diff --git a/docs/templates.rst b/docs/templates.rst index 5dafd3e0..e192b574 100644 --- a/docs/templates.rst +++ b/docs/templates.rst @@ -23,9 +23,10 @@ ignored. For example:: 3. # ...and so is this one. 4. {end} -A hash behind a code block also starts a comment that ends with the line, -e.g. in ``{x = 1} # set x``. Escaped characters are text: ``\# text`` -sends the command ``# text``, and ``show \}# x`` sends ``show }# x``. +A hash behind a code block or an ```` element also starts a comment +that ends with the line, e.g. in ``{x = 1} # set x``. Escaped characters +are text: ``\# text`` sends the command ``# text``, and ``show \}# x`` +sends ``show }# x``. Using Variables --------------- diff --git a/src/Exscript/interpreter/template.py b/src/Exscript/interpreter/template.py index 31619207..4a81af09 100644 --- a/src/Exscript/interpreter/template.py +++ b/src/Exscript/interpreter/template.py @@ -49,9 +49,9 @@ grammar = escaped_grammar + [ ('escaped_data', r'\\.'), - # A hash starts a comment at the start of a line and behind a code block, - # but not behind an escaped closing curly bracket. - ('line_comment', r'(?:^|(?<=[\r\n])|(?<=})(? element, but not behind an escaped closing curly bracket. + ('line_comment', r'(?:^|(?<=[\r\n])|(?<=})(?))[ \t]*#[^\r\n]*(?:[\r\n]|$)'), ('open_curly_bracket', '{'), ('close_curly_bracket', '}'), ('open_rpc_tag', r''), diff --git a/tests/Exscript/interpreter/RpcTest.py b/tests/Exscript/interpreter/RpcTest.py index e119066d..4fb5a6e0 100644 --- a/tests/Exscript/interpreter/RpcTest.py +++ b/tests/Exscript/interpreter/RpcTest.py @@ -35,11 +35,10 @@ def testCommentsAreNotSent(self): self.evaluate('\n # a comment\n \n # another comment\n \n\n') self.assertSent('\n') - def testHashBehindAnRpc(self): - # Only lines starting with a hash, and the text behind a code block, - # are comments. - self.evaluate(' # not a comment\nls\n') - self.assertSent('', '# not a comment', 'ls') + def testCommentBehindAnRpc(self): + # As behind a code block. + self.evaluate(' # a comment\nls\n# a comment') + self.assertSent('', 'ls', '') def testVariables(self): text = '<$source/>\n' From 1eacddbbbb61ef24130df92c2d0e4f4da1f022be Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 10 Nov 2026 19:34:53 +0100 Subject: [PATCH 133/250] Store tuples as lists in variables Every variable of a template is a list, as docs/templates.rst says, but variables kept tuples, e.g. those of the caller, so "$t" gave the repr of the tuple. The new to_list() of the scope module converts them when a variable is defined. --- CHANGES.md | 4 ++++ docs/migration.rst | 5 ++++- src/Exscript/interpreter/parser.py | 10 +++++---- src/Exscript/interpreter/program.py | 13 +++++------- src/Exscript/interpreter/scope.py | 22 +++++++++++++++----- tests/Exscript/interpreter/AssignTest.py | 8 +++++++ tests/Exscript/interpreter/CommandTest.py | 5 +++++ tests/Exscript/interpreter/ExpressionTest.py | 11 +++++----- tests/Exscript/interpreter/LoopTest.py | 7 +++++++ tests/Exscript/interpreter/ParserTest.py | 15 ++++++++++--- tests/Exscript/interpreter/StringTest.py | 6 ++++++ 11 files changed, 80 insertions(+), 26 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index c5a20d31..fc3c712d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -473,6 +473,10 @@ development versions; "before" refers to them. or a `TypeError`, except that `in`, `fail`, `loop` and `extract` used the characters of a string, and `in` and `extract` the keys of a dict, e.g. `"err" in out()` was a substring test if the function `out` returned a string. + Variables store a tuple as a list, e.g. a tuple that the caller passes or that a + function returns to an assignment, so `$t` in a command or a string gives its + items, one per line, and `{x = t}` copies them. Before, `$t` gave the text of the + tuple, such as `('a', 'b')`, and `x` became a list with the tuple as its one item. - **Regular Expressions as Operands and Values:** A regular expression that is not the right operand of `matches`, e.g. in `/a/ matches /a/`, `/a/ . "b"`, `"a" is /a/` or `not /a/`, raises an `ExecuteError` that shows its position, also diff --git a/docs/migration.rst b/docs/migration.rst index 848835b3..1c817488 100644 --- a/docs/migration.rst +++ b/docs/migration.rst @@ -134,7 +134,10 @@ Operators Values of the caller A string that a function of the caller returns, or a dict that the caller passes, is one item: ``in``, ``loop`` and ``extract`` no longer - use its characters or keys. Return and pass lists instead. + use its characters or keys. Return and pass lists instead. A tuple is + a list: ``$t`` gives its items, one per line, instead of the text of + the tuple, and ``{x = t}`` copies the items instead of storing the + tuple as one item. Command line tool ----------------- diff --git a/src/Exscript/interpreter/parser.py b/src/Exscript/interpreter/parser.py index 732f7f76..c41105f9 100644 --- a/src/Exscript/interpreter/parser.py +++ b/src/Exscript/interpreter/parser.py @@ -30,6 +30,7 @@ from ..parselib import Lexer from .program import Program +from .scope import to_list class Parser: @@ -58,19 +59,20 @@ def __init__(self, **kwargs: Any) -> None: def define(self, **kwargs: Any) -> None: """ - Defines the given variables. A value that is a string or not - iterable is wrapped in a list, unless it is callable. + Defines the given variables. A value that is callable, or iterable + but not a string or a tuple, is defined as it is; any other value as + to_list() of the scope module returns it, so a tuple becomes a list. """ for key, value in list(kwargs.items()): if ( - not isinstance(value, str) + not isinstance(value, (str, tuple)) and isinstance(value, Iterable) # mypy takes Callable for a special form, not for the ABC it is. or isinstance(value, cast(type, Callable)) ): self.variables[key] = value else: - self.variables[key] = [value] + self.variables[key] = to_list(value) def define_object(self, **kwargs: Any) -> None: """ diff --git a/src/Exscript/interpreter/program.py b/src/Exscript/interpreter/program.py index 8a9fb2c5..4702d4b0 100644 --- a/src/Exscript/interpreter/program.py +++ b/src/Exscript/interpreter/program.py @@ -28,7 +28,7 @@ from typing import TYPE_CHECKING, Any, cast from ..parselib import Lexer -from .scope import Scope +from .scope import Scope, to_list from .template import Template if TYPE_CHECKING: @@ -60,17 +60,14 @@ def __init__( def init(self, *args: Any, **kwargs: Any) -> None: """ - Sets the initial values of the given variables. A value that is not - a list is wrapped in a list. Names that contain a dot or start with - an underscore are ignored. + Sets the initial values of the given variables, as to_list() returns + them. Names that contain a dot or start with an underscore are + ignored. """ for key in kwargs: if key.find('.') >= 0 or key.startswith('_'): continue - if type(kwargs[key]) == type([]): - self.init_variables[key] = kwargs[key] - else: - self.init_variables[key] = [kwargs[key]] + self.init_variables[key] = to_list(kwargs[key]) def execute(self, *args: Any, **kwargs: Any) -> dict[str, Any]: """ diff --git a/src/Exscript/interpreter/scope.py b/src/Exscript/interpreter/scope.py index f6bc2288..55f3b4b9 100644 --- a/src/Exscript/interpreter/scope.py +++ b/src/Exscript/interpreter/scope.py @@ -33,6 +33,18 @@ from .parser import Parser +def to_list(value: Any) -> list[Any]: + """ + Returns the given value as the value of a variable: a list as it is, a + tuple as a list, and any other value as a list with this one item. + """ + if type(value) is list: + return value + if isinstance(value, tuple): + return list(value) + return [value] + + class Scope(Token): """ A token that holds variables and evaluates its children in order. @@ -82,17 +94,17 @@ def exit_request(self) -> None: def define(self, **kwargs: Any) -> None: """ - Defines the given variables in the root scope. A value that is not - a list is wrapped in a list, unless the name contains a dot (as the - names of functions do) or starts with an underscore. + Defines the given variables in the root scope, with the values as + to_list() returns them, unless the name contains a dot (as the names + of functions do) or starts with an underscore. """ if self.parent is not None: return self.parent.define(**kwargs) for key in kwargs: - if key.find('.') >= 0 or key.startswith('_') or type(kwargs[key]) is type([]): + if key.find('.') >= 0 or key.startswith('_'): self.variables[key] = kwargs[key] else: - self.variables[key] = [kwargs[key]] + self.variables[key] = to_list(kwargs[key]) def define_object(self, **kwargs: Any) -> None: """ diff --git a/tests/Exscript/interpreter/AssignTest.py b/tests/Exscript/interpreter/AssignTest.py index 5b9417e4..e621e7e1 100644 --- a/tests/Exscript/interpreter/AssignTest.py +++ b/tests/Exscript/interpreter/AssignTest.py @@ -31,6 +31,14 @@ def testVariableOfTheCallerBeforeTheAssignment(self): variables = self.evaluate('{if 0}{x = "new"}{end}', x='hi') self.assertEqual(variables['x'], ['hi']) + def testAssignATuple(self): + # A tuple is a list, also one that a function returns. + variables = self.evaluate( + '{x = t}{y = pair()}', t=('a', 'b'), pair=lambda scope: ('c', 'd') + ) + actual = (variables['t'], variables['x'], variables['y']) + self.assertEqual(actual, (['a', 'b'], ['a', 'b'], ['c', 'd'])) + def testInternalVariables(self): with self.assertRaises(CompileError) as cm: self.evaluate('{__hostname__ = "r2"}') diff --git a/tests/Exscript/interpreter/CommandTest.py b/tests/Exscript/interpreter/CommandTest.py index ae3086fc..f0020621 100644 --- a/tests/Exscript/interpreter/CommandTest.py +++ b/tests/Exscript/interpreter/CommandTest.py @@ -95,6 +95,11 @@ def testVariablesOfTheCaller(self): self.evaluate('show interface $name\nshow vlan $vlan\n', name='eth0', vlan=10) self.assertSent('show interface eth0', 'show vlan 10') + def testTupleVariable(self): + # A tuple is a list; the items of a list are joined with line breaks. + self.evaluate('show $t\nshow $u\n', t=('version',), u=('a', 'b')) + self.assertSent('show version', 'show a\nb') + def testDollarSign(self): self.evaluate('{name = "eth0"}\nshow \\$name $ $\n') self.assertSent('show $name $ $') diff --git a/tests/Exscript/interpreter/ExpressionTest.py b/tests/Exscript/interpreter/ExpressionTest.py index 44a2c061..ae39c6d7 100644 --- a/tests/Exscript/interpreter/ExpressionTest.py +++ b/tests/Exscript/interpreter/ExpressionTest.py @@ -136,12 +136,12 @@ def testRegularExpressionAsAnotherOperand(self): self.assertError(cm.exception, [text], marker, message) def testOperandsThatAreNotLists(self): - # Functions return True without a result, the caller may pass a tuple, - # and the connection is an object; a string is one item. + # Functions return True without a result, or a tuple, the caller may + # pass a tuple, and the connection is an object; a string is one item. text = ( '{a = yes() and "x"}{b = t . "c"}{c = "b" in t}{d = not t}' '{e = __connection__ is __connection__}{f = not __connection__}' - '{g = two() * 3}{h = "a" in word()}{i = word() matches /b/}' + '{g = two() * 3}{h = "a" in word()}{i = word() matches /b/}{j = "b" in pair()}' ) variables = self.evaluate( text, @@ -149,9 +149,10 @@ def testOperandsThatAreNotLists(self): t=('a', 'b'), two=lambda scope: 2, word=lambda scope: 'ab', + pair=lambda scope: ('a', 'b'), ) - actual = [variables[name] for name in 'abcdefghi'] - expected = [['x'], ['ac'], [True], [False], [True], [False], [6], [False], [1]] + actual = [variables[name] for name in 'abcdefghij'] + expected = [['x'], ['ac'], [True], [False], [True], [False], [6], [False], [1], [True]] self.assertEqual(actual, expected) def testFunctionAsAnOperand(self): diff --git a/tests/Exscript/interpreter/LoopTest.py b/tests/Exscript/interpreter/LoopTest.py index c9f46ebe..7069e95d 100644 --- a/tests/Exscript/interpreter/LoopTest.py +++ b/tests/Exscript/interpreter/LoopTest.py @@ -26,6 +26,13 @@ def testLoopOverAFunctionResult(self): self.evaluate('{loop string.split("a,b", ",") as x}\nshow $x\n{end}') self.assertSent('show a', 'show b') + def testLoopOverATupleInAnotherVariable(self): + text = ( + '{x = t}{y = pair()}{loop x as i}{append i to l}{end}{loop y as i}{append i to l}{end}' + ) + variables = self.evaluate(text, t=('a', 'b'), pair=lambda scope: ('c', 'd')) + self.assertEqual(variables['l'], ['a', 'b', 'c', 'd']) + def testLoopOverSeveralLists(self): text = '{loop modes,\n directories\n as mode,\n directory}\nls $mode $directory\n{end}' self.evaluate(text, modes=['drwx', '-rw'], directories=['a', 'b']) diff --git a/tests/Exscript/interpreter/ParserTest.py b/tests/Exscript/interpreter/ParserTest.py index 0478a8e3..098d238b 100644 --- a/tests/Exscript/interpreter/ParserTest.py +++ b/tests/Exscript/interpreter/ParserTest.py @@ -44,7 +44,7 @@ def function(scope): parser = Parser() parser.define(a='x', b=1, c=['y', 'z'], d=('t',), e=function) self.assertEqual( - parser.variables, {'a': ['x'], 'b': [1], 'c': ['y', 'z'], 'd': ('t',), 'e': function} + parser.variables, {'a': ['x'], 'b': [1], 'c': ['y', 'z'], 'd': ['t'], 'e': function} ) variables = parser.parse('{x = a . b . c . e()}').execute() self.assertEqual(variables['x'], ['x1yTrue']) @@ -109,6 +109,8 @@ def testInit(self): self.assertNotIn('_h', variables) self.program.init(x=['c', 'd']) self.assertEqual(self.program.execute()['y'], ['c!']) + self.program.init(x=('e',)) + self.assertEqual(self.program.execute()['y'], ['e!']) def testVariableThatIsUndefinedAtRuntime(self): with self.assertRaises(ExecuteError) as cm: @@ -153,10 +155,17 @@ def testConstructor(self): Scope('Root', lexer, None, variables={'f.g': 'a', '_h': 1, 'i': []}) def testDefine(self): - self.child.define(b='x', c=['y'], **{'f.g': 'function', '_h': 'private'}) + self.child.define(b='x', c=['y'], d=('z', 'w'), **{'f.g': 'function', '_h': 'private'}) self.assertEqual( self.root.variables, - {'a': [1], 'b': ['x'], 'c': ['y'], 'f.g': 'function', '_h': 'private'}, + { + 'a': [1], + 'b': ['x'], + 'c': ['y'], + 'd': ['z', 'w'], + 'f.g': 'function', + '_h': 'private', + }, ) self.assertEqual(self.child.variables, {}) diff --git a/tests/Exscript/interpreter/StringTest.py b/tests/Exscript/interpreter/StringTest.py index 7bde0327..bdb5c039 100644 --- a/tests/Exscript/interpreter/StringTest.py +++ b/tests/Exscript/interpreter/StringTest.py @@ -47,6 +47,12 @@ def testVariables(self): ) self.assertEqual(variables['x'], ['1-x\ny-6-r1-[]']) + def testTupleVariable(self): + # A tuple is a list, also one that a function returns. + text = '{x = "$t!"}{p = pair()}{y = "$p"}' + variables = self.evaluate(text, t=('a', 'b'), pair=lambda scope: ('c', 'd')) + self.assertEqual((variables['x'], variables['y']), (['a\nb!'], ['c\nd'])) + def testVariableNames(self): variables = self.evaluate('{a = "1"}{a_b = "2"}{x = "$a.b $a_b $a_b2"}', a_b2='3') self.assertEqual(variables['x'], ['1.b 2 3']) From 870c9a54a52135c19679dc0b94bbe72200a63154 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 10 Nov 2026 20:05:23 +0100 Subject: [PATCH 134/250] List the removed internals of the template interpreter Add the attributes that the new ordering of operators removed to CHANGES.md and the migration guide. --- CHANGES.md | 4 +++- docs/migration.rst | 6 +++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index fc3c712d..2b63f8f0 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -39,7 +39,9 @@ marked with **BREAKING**; a migration guide will accompany the release. `test(secure=True)`; `InputHistory.file`, now `filename`; the `debug` argument of `WorkQueue`, whose `set_debug()` argument is now `level`; the `type` argument of `parselib.Lexer.expect()` and `current_is()`, now `types`; and the `dump1()` methods - of the template interpreter. + of the template interpreter, `ExpressionNode.parent_node`, the `prio` argument of + `Expression.prioritize()`, and `lft`, `rgt`, `op` and `priority()` of `Term`, which + only the old ordering of operators used. - **OpenTelemetry Tracing (optional):** Install the `otel` extra (`Exscript[otel]`) and call `Exscript.contrib.otel.ExscriptInstrumentor().instrument()`. Every call of `connect()`, `login()`, `authenticate()`, `send()`, `execute()` and `close()` creates diff --git a/docs/migration.rst b/docs/migration.rst index 1c817488..f3f05c51 100644 --- a/docs/migration.rst +++ b/docs/migration.rst @@ -201,7 +201,11 @@ Other renamed or removed names: ``level``. Pass the other arguments by keyword. - ``parselib.Lexer.expect()`` and ``current_is()`` take ``types`` instead of ``type``. -- The ``dump1()`` methods of the template interpreter are removed. +- The ``dump1()`` methods of the template interpreter are removed, and so + are the attributes that only its old ordering of operators used: + ``ExpressionNode.parent_node``, the ``prio`` argument of + ``Expression.prioritize()``, and ``lft``, ``rgt``, ``op`` and + ``priority()`` of ``Term``. - Adapters that subclass ``Protocol`` must pass its arguments by keyword; new ones were inserted. From 0070c551b0bd4bfae073345081851b4569b3fcb8 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 10 Nov 2026 21:52:30 +0100 Subject: [PATCH 135/250] Raise for unknown driver names in the Protocol constructor An unknown driver name only logged a debug message, and the driver was guessed, so a typo went unnoticed until a device behaved unexpectedly. Check the name like set_driver(), which raises a TypeError. --- CHANGES.md | 7 +++++ docs/drivers.rst | 4 +-- docs/migration.rst | 6 ++++ src/Exscript/protocols/protocol.py | 43 +++++++++++++------------- tests/Exscript/protocols/ExpectTest.py | 9 ++++++ 5 files changed, 46 insertions(+), 23 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 2b63f8f0..c566b02c 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -317,6 +317,13 @@ development versions; "before" refers to them. ### Protocol Drivers +- **BREAKING - Unknown Driver Names Raise:** The constructor of the protocol adapters, + e.g. `SSH2(driver='iosxr')`, raises a `TypeError` for the name of a driver that does + not exist, as `set_driver()` does. This also applies to the `driver` option of a + `Host` and to the `--use-driver` option of the command line tool. Before, the + constructor ignored such a name silently and guessed the driver, so a typo, e.g. in + an inventory, went unnoticed. To migrate, correct the name; `drivers.driver_map` + lists the names. - **Generic Prompt Pattern in Linear Time:** The prompt pattern of the generic driver, which other drivers inherit, backtracked catastrophically on long lines that are no prompt: a line with a run of 100 word characters took 1.3 seconds to check, 200 diff --git a/docs/drivers.rst b/docs/drivers.rst index 1c6cf79f..7f2b14ac 100644 --- a/docs/drivers.rst +++ b/docs/drivers.rst @@ -19,8 +19,8 @@ name of its driver. This is more reliable than a guess:: conn.set_driver('ios_xr') # the same, also after the login ``set_driver(None)`` turns the guessing back on. A name that does not -exist makes ``set_driver()`` raise a ``TypeError``; the constructor -ignores it and guesses. +exist raises a ``TypeError``, in the constructor as in ``set_driver()``, +so a typo, e.g. in an inventory, shows at once. For :class:`~Exscript.Host` objects, set the option ``driver``; the command line tool has ``--use-driver``:: diff --git a/docs/migration.rst b/docs/migration.rst index f3f05c51..49d2864a 100644 --- a/docs/migration.rst +++ b/docs/migration.rst @@ -89,6 +89,12 @@ OS detection driver. Pass ``driver=`` if a device needs a specific driver. See :doc:`drivers`. +Driver names + A driver name that does not exist raises a ``TypeError`` in the + constructor, e.g. ``SSH2(driver='iosxr')``, also through the ``driver`` + option of a ``Host`` and ``--use-driver``. 2.x ignored it and guessed + the driver. + Templates --------- diff --git a/src/Exscript/protocols/protocol.py b/src/Exscript/protocols/protocol.py index 02b91a0e..a1b6acdb 100644 --- a/src/Exscript/protocols/protocol.py +++ b/src/Exscript/protocols/protocol.py @@ -158,6 +158,22 @@ def _ends_with_anchor(regex: re.Pattern[str]) -> bool: # decoded correctly. Netconf always uses UTF-8, as RFC 6241 requires. +def _lookup_driver(driver: Driver | str) -> Driver: + """ + Returns the given driver, or the driver with the given name. + + :raises TypeError: If no driver has the given name, or if the argument + is neither a Driver nor a string. + """ + if isinstance(driver, str): + if driver not in driver_map: + raise TypeError('no such driver: ' + repr(driver)) + return driver_map[driver] + if isinstance(driver, Driver): + return driver + raise TypeError('unsupported argument type: ' + type(driver).__name__) + + class Protocol: """ This is the base class for all protocols; it defines the common portions @@ -260,7 +276,8 @@ def __init__( one-time-password to be entered. :type driver: Driver()|str|None - :keyword driver: The manually picked device driver. + :keyword driver: The manually picked device driver: a Driver, or the + name of one; see set_driver(). An unknown name raises a TypeError. :keyword stdout: Where to write the device response: any object with write() and flush(). Defaults to an in-memory buffer. :keyword stderr: The stream that the stderr property returns; @@ -372,17 +389,8 @@ def __init__( self.host_key_policy = host_key_policy self.known_hosts = known_hosts - # Set manual_driver: if driver is not None: - if isinstance(driver, str): - if driver in driver_map: - self._manual_driver = driver_map[driver] - else: - self._dbg(1, 'Invalid driver string given. Ignoring.') - elif isinstance(driver, Driver): - self._manual_driver = driver - else: - self._dbg(1, 'Invalid driver given. Ignoring.') + self._manual_driver = _lookup_driver(driver) def __copy__(self) -> Self: """ @@ -656,19 +664,12 @@ def set_driver(self, driver: Driver | str | None = None) -> None: :type driver: Driver()|str|None :param driver: The driver to be pinned (disables auto-detection). + :raises TypeError: If no driver has the given name, or if the + argument is neither a Driver nor a string. """ old_driver = self.get_driver() - if driver is None: - self._manual_driver = None - elif isinstance(driver, str): - if driver not in driver_map: - raise TypeError('no such driver:' + repr(driver)) - self._manual_driver = driver_map[driver] - elif isinstance(driver, Driver): - self._manual_driver = driver - else: - raise TypeError('unsupported argument type: ' + type(driver).__name__) + self._manual_driver = None if driver is None else _lookup_driver(driver) new_driver = self.get_driver() if old_driver != new_driver: diff --git a/tests/Exscript/protocols/ExpectTest.py b/tests/Exscript/protocols/ExpectTest.py index 183fbb4d..7f4c15e3 100644 --- a/tests/Exscript/protocols/ExpectTest.py +++ b/tests/Exscript/protocols/ExpectTest.py @@ -252,6 +252,15 @@ def testSetDriver(self): self.assertRaises(TypeError, conn.set_driver, 42) self.assertIs(conn.get_driver(), driver) + def testDriverOfTheConstructor(self): + # An unknown name, e.g. a typo, raises instead of guessing the driver silently. + self.assertEqual(Dummy(driver='ios').get_driver().name, 'ios') + driver = drivers.Driver('custom') + self.assertIs(Dummy(driver=driver).get_driver(), driver) + with self.assertRaisesRegex(TypeError, "no such driver: 'iosxr'"): + Dummy(driver='iosxr') + self.assertRaises(TypeError, Dummy, driver=42) + class LoginTest(unittest.TestCase): """App-level logins: S/Key, bailout, an account for the device, and a closed connection.""" From 21ca5c93f40e068d34dce8b7f150584f69b162d0 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 10 Nov 2026 22:05:44 +0100 Subject: [PATCH 136/250] Frame NETCONF messages on bytes, and buffer whole messages ncclient 0.7.1 decodes data before a message is complete, so a character split across two reads or chunks ended the session. Install a parser that frames on bytes, and put whole messages without framing in the buffer of expect(), where chunk headers could break matches. --- CHANGES.md | 10 ++ docs/netconf.rst | 17 +++ src/Exscript/protocols/netconf.py | 184 ++++++++++++++++++++---- tests/Exscript/protocols/NetconfTest.py | 146 ++++++++++++++++++- 4 files changed, 322 insertions(+), 35 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index c566b02c..fde0e12d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -308,6 +308,16 @@ development versions; "before" refers to them. raises `ConnectionClosedException`, like SSH and Telnet. Before, it raised a plain `ProtocolException` ("ncclient related error ..."); the new exception is a `ProtocolException`, too. +- **Messages Instead of Received Data:** Exscript splits the session into its messages + itself, as RFC 6242 frames them, and decodes each message as UTF-8 when it is + complete. `response` and the patterns of `expect()` and `waitfor()` see these + messages, each followed by `]]>]]>`, also with NETCONF 1.1. Before, they saw the data + as received: with NETCONF 1.1, a chunk header could split the text that a pattern + looks for, e.g. ``, and text like `\n##\n` behind a tag ended a + message early; and a character whose bytes arrived in two reads, or that a device + split across two chunks, ended the session with a `UnicodeDecodeError` in ncclient + 0.7.1. The logs, the standard output and `data_received_event` still show the data + as received. - **UTF-8:** `Netconf` always uses UTF-8, which RFC 6241 requires for NETCONF messages, whatever `encoding` is passed, and `reassemble_chunks()` counts the chunk sizes in bytes, as RFC 6242 does. Bytes that are no UTF-8, also in the SSH banner, become diff --git a/docs/netconf.rst b/docs/netconf.rst index 83870209..9d7cc586 100644 --- a/docs/netconf.rst +++ b/docs/netconf.rst @@ -63,6 +63,23 @@ Filters work as with any NETCONF client:: ``close()`` sends ```` and closes the connection. +Messages and notifications +-------------------------- + +``response`` and the patterns of ``expect()`` and ``waitfor()`` see the +messages of the session, decoded and without the framing of :rfc:`6242`; +each message is followed by ``]]>]]>``, also with NETCONF 1.1. To wait for +a notification after ````:: + + conn.execute('') + conn.expect('') + print(conn.response) + +The logs, the standard output and ``data_received_event`` show the data +as it was received, with the chunk headers of NETCONF 1.1; +``Netconf.reassemble_chunks()`` removes them. + Errors ------ diff --git a/src/Exscript/protocols/netconf.py b/src/Exscript/protocols/netconf.py index 9ab99c1b..bb949c4f 100644 --- a/src/Exscript/protocols/netconf.py +++ b/src/Exscript/protocols/netconf.py @@ -63,6 +63,125 @@ _chunk_header_re = re.compile(rb'\n#(\d+)\n') _end_of_chunks_re = re.compile(rb'\n##\n') +# The framing of NETCONF messages (RFC 6242): 1.0 ends each message with +# ]]>]]> (section 4.3); 1.1 sends chunks, each with a header of its size, +# and ends the message with \n##\n (section 4.2). +_EOM = b']]>]]>' +_frame_re = re.compile(rb'\n#([1-9][0-9]{0,9})\n|\n##\n') +_MAX_FRAME_HEADER = len(b'\n#4294967295\n') +_MAX_CHUNK_SIZE = 4294967295 + + +class _Framer: + """ + Splits the data of a NETCONF session into its messages, framed as RFC + 6242 describes, and passes each message to ncclient as text, decoded as + UTF-8 when it is complete; bytes that are no UTF-8 become U+FFFD. + ncclient's IO thread calls parse() with the data of each read, as with + its own parsers. + + Workaround for ncclient 0.7.1 (the latest release): its DefaultXMLParser + decodes the data before the message is complete, from the position of + the last search for ]]>]]> (NETCONF 1.0) and chunk by chunk (NETCONF + 1.1). A character whose bytes arrive in two reads, or that a device + splits across two chunks, raises a UnicodeDecodeError in the IO thread, + which ends the session. Remove this class when ncclient frames the + messages on bytes, and let _init_ncclient() keep the parser of ncclient + again (session.parser = device_handler.get_xml_parser(session)). + """ + + def __init__(self, session: Any) -> None: + """ + Constructor. + + :param session: ncclient's SSHSession, which gets the messages. + """ + from ncclient.transport.session import NetconfBase + + self._session = session + self._base_11 = NetconfBase.BASE_11 + self._data = bytearray() + self._chunks: list[bytes] = [] # of the NETCONF 1.1 message being received + self._searched = 0 # NETCONF 1.0: the data before this has no ]]>]]> + + def parse(self, data: bytes) -> None: + """ + Takes the data of a read, and passes the messages that are complete + to ncclient, in their order. + + :raises NetconfFramingError: (of ncclient) If the data breaks the + chunked framing of NETCONF 1.1. + """ + self._data += data + while (message := self._next_message()) is not None: + text = message.decode('utf-8', errors='replace').strip() + if text: + self._session._dispatch_message(text) + + def _next_message(self) -> bytes | None: + """ + Removes the next complete message from the data and returns it + without its framing, or None if no message is complete. Uses the + framing of NETCONF 1.1 once ncclient selects it after the + messages. + """ + if self._session._base == self._base_11: + return self._next_chunked_message() + end = self._data.find(_EOM, self._searched) + if end < 0: + self._searched = max(0, len(self._data) - len(_EOM) + 1) + return None + message = bytes(self._data[:end]) + del self._data[: end + len(_EOM)] + self._searched = 0 + return message + + def _next_chunked_message(self) -> bytes | None: + """ + Like _next_message(), for the chunked framing of NETCONF 1.1. Skips + whitespace between messages, e.g. behind the ]]>]]> of the + of a server. + """ + data = self._data + pos = 0 + if not self._chunks: + while pos + 1 < len(data) and data[pos] in b' \t\r\n' and data[pos + 1] != ord('#'): + pos += 1 + try: + while True: + match = _frame_re.match(data, pos) + if match is None: + rest = bytes(data[pos : pos + _MAX_FRAME_HEADER]) + if len(rest) == _MAX_FRAME_HEADER or b'\n' in rest[1:]: + self._framing_error(rest) + return None # the header is not complete + if match.group(1) is None: # \n##\n ends the message + if not self._chunks: + self._framing_error(match.group()) + pos = match.end() + message = b''.join(self._chunks) + self._chunks = [] + return message + size = int(match.group(1)) + if size > _MAX_CHUNK_SIZE: + self._framing_error(match.group()) + if match.end() + size > len(data): + return None # the chunk is not complete + self._chunks.append(bytes(data[match.end() : match.end() + size])) + pos = match.end() + size + finally: + del data[:pos] + + def _framing_error(self, data: bytes) -> NoReturn: + """ + Raises ncclient's NetconfFramingError for the given data, which is + not a chunk header where one must be. + """ + from ncclient.transport.errors import NetconfFramingError + + raise NetconfFramingError('invalid NETCONF 1.1 chunk header: %r' % data) + + # Exscript sends its and RPCs pretty-printed, so that they are readable # in logs. ncclient serializes them with the to_xml() of two of its modules, in # the thread that sends; _install_to_xml() replaces it with a function that @@ -281,13 +400,12 @@ def _paramiko_channel(self) -> None: def _init_ncclient(self) -> None: """ - Starts the NETCONF session with ncclient on the authenticated - transport and the open channel: exchanges the `` messages, - starts the IO thread of ncclient and creates its manager. recv() and - send() of the channel are wrapped, so the data of the session also - reaches the buffer, the logs and the events. Then updates the driver - from the server capabilities and consumes the server `` from - the buffer. Does nothing if the manager exists already. + Starts the NETCONF session with ncclient on the open channel, unless + the manager exists already: exchanges the `` messages, starts + ncclient's IO thread and manager, and passes the data of the session + to the logs and the events, and its messages to the buffer. Then + updates the driver from the server capabilities and consumes the + server `` from the buffer. """ # Bypasses ncclient's default SSH connection flow by injecting the # already-authenticated paramiko transport and the open NETCONF channel @@ -297,6 +415,7 @@ def _init_ncclient(self) -> None: return from ncclient.manager import Manager from ncclient.operations.rpc import RaiseMode + from ncclient.transport.session import SessionListener from ncclient.transport.ssh import SSHSession _install_to_xml() @@ -312,6 +431,15 @@ def _init_ncclient(self) -> None: session._channel_name = cast('Channel', self._channel).get_name() session._connected = True session._closing.clear() + session.parser = _Framer(session) # see _Framer: a workaround for ncclient + + # Every message that ncclient receives also reaches the buffer. An + # ncclient listener must be a SessionListener; its methods are replaced + # here, so this module imports ncclient only when a session starts. + listener = SessionListener() + listener.callback = lambda root, raw: self._receive_message(raw) + listener.errback = lambda error: None # the end of the channel ends expect() + session.add_listener(listener) self._wrap_channel(cast('Channel', self._channel)) @@ -320,9 +448,6 @@ def _init_ncclient(self) -> None: with _pretty(): session._post_connect(self.get_timeout()) - # Selects the driver/device-specific XML parser for all subsequent RPCs. - session.parser = device_handler.get_xml_parser(session) - # Finally the manager is in charge of the channel. self._ncc_session = session self._ncc_manager = Manager( @@ -341,15 +466,16 @@ def _init_ncclient(self) -> None: def _wrap_channel(self, channel: 'Channel') -> None: """ Wraps recv() and send() of the given channel, which ncclient's IO - thread calls, so the data of the session also reaches the buffer, the - logs and the events. The data is decoded as UTF-8 with an incremental - decoder per direction, so a character that spans two reads or writes - is decoded, too; bytes that are no UTF-8 become U+FFFD. + thread calls, so the data of the session also reaches the logs, the + standard output and the events, as it is sent and received, with its + framing. The data is decoded as UTF-8 with an incremental decoder per + direction, so a character that spans two reads or writes is decoded, + too; bytes that are no UTF-8 become U+FFFD. """ # Controls the behavior of the replication wrappers. The replication is # needed as ncclient owns the channel within its IO thread. So we need - # to somehow fetch this raw data for display, logging and raw response - # buffering/matching. + # to somehow fetch this raw data for display and logging. The end of the + # received data also ends the replication of the messages to the buffer. self._replicate = True recv_decoder = codecs.getincrementaldecoder('utf-8')(errors='replace') send_decoder = codecs.getincrementaldecoder('utf-8')(errors='replace') @@ -365,7 +491,6 @@ def recv_wrapper(*args: Any, **kwargs: Any) -> bytes: string_data = recv_decoder.decode(data, final=not data) if string_data: # empty at the end, or for a part of a character self._receive_cb(string_data) - self._buffer.append(string_data) if not data: self._dbg(1, 'channel.recv() returned EOF') self._replicate = False @@ -398,6 +523,16 @@ def send_wrapper(*args: Any, **kwargs: Any) -> int: cast(Any, channel).send = send_wrapper # Any: replaces a method + def _receive_message(self, message: str) -> None: + """ + Appends a message that ncclient's IO thread received, without its + framing, to the buffer, followed by the ]]>]]> that the prompt + matches; so the response has no framing, and a prompt never matches + within a message. Does nothing after the end of the received data. + """ + if self._replicate: + self._buffer.append(message + ']]>]]>') + def login( self, account: 'Account | AccountProxy | _Context | None' = None, @@ -490,8 +625,9 @@ def reassemble_chunks(response: str) -> str: Each chunk is introduced by a header `LF HASH chunk-size LF` where the size is the exact byte count of the following chunk (RFC 6242 §4.2), in the UTF-8 encoding of the response. - The end-of-chunks marker `LF HASH HASH LF` gets stripped finally. Useful - to also reassemble chunks in received data from events. + The end-of-chunks marker `LF HASH HASH LF` gets stripped finally. The + response and the buffer of the adapter have no framing; this is + useful for the received data of the logs and data_received_event. Raises ValueError on unexpected trailing bytes after the last chunk. @@ -525,16 +661,6 @@ def reassemble_chunks(response: str) -> str: raise ValueError(msg % repr(data[:50].decode('utf-8', 'replace') + '...')) return b''.join(parts).decode('utf-8') - def _domatch(self, prompt: list[re.Pattern[str]], flush: bool) -> tuple[int, re.Match[str]]: - """ - Like Protocol._domatch(), but reassembles a chunked response (NETCONF - 1.1) into plain content. - """ - n, match = super()._domatch(prompt, flush) - # super()._domatch() sets the response. - self.response = self.reassemble_chunks(cast(str, self.response)) - return n, match - def _fill_buffer(self, timeout: float | None) -> bool | None: """ Waits until ncclient's IO thread has appended data to the buffer, for diff --git a/tests/Exscript/protocols/NetconfTest.py b/tests/Exscript/protocols/NetconfTest.py index dfbfb68f..24607220 100644 --- a/tests/Exscript/protocols/NetconfTest.py +++ b/tests/Exscript/protocols/NetconfTest.py @@ -5,11 +5,13 @@ import inspect import io +import itertools import os import re import subprocess import sys import threading +import time import unittest from unittest import mock @@ -21,7 +23,8 @@ from lxml import etree from ncclient.manager import make_device_handler from ncclient.operations.rpc import RPC, RPCReply -from ncclient.transport.session import HelloHandler +from ncclient.transport.errors import NetconfFramingError +from ncclient.transport.session import HelloHandler, NetconfBase from ncclient.transport.ssh import SSHSession from ncclient.xml_ import NCElement, new_ele @@ -97,6 +100,99 @@ def testAnyText(self, text, sizes): self.assertEqual(Netconf.reassemble_chunks(framed), text) +class FakeSession: + """Records the messages that the framer dispatches, like ncclient's session.""" + + def __init__(self, base=NetconfBase.BASE_10): + self._base = base + self.messages = [] + + def _dispatch_message(self, raw): + self.messages.append(raw) + + +class FramerTest(unittest.TestCase): + """The framing of NETCONF messages (RFC 6242) on bytes; see netconf._Framer.""" + + def feed(self, data, read_sizes=(1,), base=NetconfBase.BASE_10): + """Feeds the data to a framer in reads of the given sizes, in turn.""" + session = FakeSession(base) + framer = netconf._Framer(session) + pos = 0 + for size in itertools.cycle(read_sizes): + if pos >= len(data): + break + framer.parse(data[pos : pos + size]) + pos += size + return session.messages + + def testFraming10(self): + # Characters and the ]]>]]> span reads; whitespace around a message is no part of it. + data = 'Zürich €]]>]]>\n\n]]>]]>'.encode() + self.assertEqual(self.feed(data), ['Zürich €', '']) + self.assertEqual(self.feed(data, [len(data)]), ['Zürich €', '']) + + def testIncompleteMessage(self): + self.assertEqual(self.feed(b']]>]]'), []) + self.assertEqual(self.feed(b'\n#4\n\n#', base=NetconfBase.BASE_11), []) + + def testNoUTF8(self): + self.assertEqual(self.feed(b'\xff]]>]]>'), ['\ufffd']) + + def testFraming11(self): + # A device splits the ü across two chunks; each byte arrives in its own read. + message = 'Zürich'.encode() + data = encode_message(message, True, [5, 0]) + encode_message(b'', True) + self.assertEqual(data[:10], b'\n#5\nZ\xc3\n') + self.assertEqual(self.feed(data, base=NetconfBase.BASE_11), ['Zürich', '']) + + def testEndOfChunksInAChunk(self): + # Text like the end of the chunks within a chunk is text. + message = b'\n##\n' + data = encode_message(message, True, [7, 0]) + self.assertEqual(data[:11], b'\n#7\n\n##\n') + for sizes in (1,), (11,), (len(data),): + messages = self.feed(data, sizes, base=NetconfBase.BASE_11) + self.assertEqual(messages, ['\n##\n'], sizes) + + def testWhitespaceBetweenMessages(self): + # E.g. a newline behind the ]]>]]> of the of a server. + data = b'\r\n \n' + encode_message(b'', True) + self.assertEqual(self.feed(data, base=NetconfBase.BASE_11), ['']) + + def testSwitchToFraming11(self): + # ncclient selects NETCONF 1.1 after the messages. + session = FakeSession() + framer = netconf._Framer(session) + framer.parse(b']]>]]>') + session._base = NetconfBase.BASE_11 + framer.parse(encode_message(b'', True)) + self.assertEqual(session.messages, ['', '']) + + def testFramingErrors(self): + for data in ( + b'\n#x\n\n##\n', # no size + b'\n#0\n\n##\n', # chunks are not empty + b'\n##\n', # a message has chunks + b'\n#4294967296\n', # too large + b']]>]]>' + b' ' * 20, # 1.0 framing + ): + with self.subTest(data=data), self.assertRaises(NetconfFramingError): + self.feed(data, [len(data)], base=NetconfBase.BASE_11) + + @settings(max_examples=200, deadline=None) + @given( + st.lists(st.text(max_size=40), min_size=1, max_size=4), + st.lists(st.integers(1, 12), max_size=6), + st.lists(st.integers(1, 30), min_size=1, max_size=6), + ) + def testAnyMessages(self, texts, chunk_sizes, read_sizes): + # Any text, also like a chunk header, in chunks of any size in bytes, read in any pieces. + messages = [f'{text}' for text in texts] + data = b''.join(encode_message(m.encode(), True, chunk_sizes) for m in messages) + self.assertEqual(self.feed(data, read_sizes, base=NetconfBase.BASE_11), messages) + + class NcclientContractTest(unittest.TestCase): """The ncclient internals that the adapter uses; an update of ncclient must keep them.""" @@ -192,8 +288,8 @@ def testCharactersSplitAcrossReads(self): pass text = 'Zürich € \ufffd \ufffd' self.assertEqual(''.join(received), text) - self.assertEqual(str(self.conn._buffer), text) self.assertEqual(self.conn.stdout.getvalue(), text) + self.assertEqual(str(self.conn._buffer), '') # it gets the messages from ncclient def testCharactersSplitAcrossWrites(self): sent = [] @@ -241,6 +337,9 @@ class NetconfServer: :param ignore_close_session: Whether close-session gets no reply and leaves the session open, like on a server that hangs. :param banner: The authentication banner (bytes), if any. + :param pause_after: Bytes behind which the server pauses for 0.2 + seconds when it writes them the first time, like a device that + stalls in a reply. """ def __init__( @@ -255,6 +354,7 @@ def __init__( channels=True, ignore_close_session=False, banner=None, + pause_after=None, ): self.reply = reply or (lambda rpc: '') self.capabilities = capabilities @@ -262,6 +362,7 @@ def __init__( self.piece_size = piece_size self.ignore_close_session = ignore_close_session self.banner = banner + self.pause_after = pause_after self.client_hello = None self.hello_received = threading.Event() self.rpcs = [] @@ -312,6 +413,12 @@ def send(self, message): self._write(encode_message(message, self.chunked, sizes)) def _write(self, data): + if self.pause_after and self.pause_after in data: + end = data.index(self.pause_after) + len(self.pause_after) + self.pause_after = None + self._write(data[:end]) + time.sleep(0.2) # a stalling device, not a wait for something + data = data[end:] size = self.piece_size or len(data) for pos in range(0, len(data), size): self.channel.sendall(data[pos : pos + size]) @@ -431,17 +538,44 @@ def testFraming11(self): self.assertEqual(element.findtext(f'.//{{{NS}}}location'), 'rack 7') def testUTF8(self): - # NETCONF is UTF-8 (RFC 6241), whatever the encoding argument says. The reply arrives - # in one read: ncclient's parser fails if a character spans two. + # NETCONF is UTF-8 (RFC 6241), whatever the encoding argument says. The server writes + # three bytes at a time, and with NETCONF 1.1 each byte in its own chunk, so characters + # span reads and chunks. content = 'Zürich – 3 €' for capabilities in (BASE,), (BASE, BASE_11): with self.subTest(capabilities=capabilities): - conn = self.login(lambda rpc: content, capabilities=capabilities) + conn = self.login( + lambda rpc: content, capabilities=capabilities, chunk_size=1, piece_size=3 + ) conn.execute('Zürich') self.assertIn('Zürich'.encode(), self.server.rpcs[0]) self.assertTrue(conn.response.endswith(content + '
'), conn.response) self.assertIn('Zürich', conn.stdout.getvalue()) # sent - self.assertIn(content, conn.stdout.getvalue()) + if capabilities == (BASE,): # the output shows the chunks as received + self.assertIn(content, conn.stdout.getvalue()) + + def subscribe(self, **kwargs): + conn = self.login(capabilities=(BASE, BASE_11), **kwargs) + conn.execute( + '' + ) + return conn + + def testPatternsSpanChunks(self): + # The patterns of expect() match the messages, not the chunks of NETCONF 1.1. + conn = self.subscribe(chunk_size=4) + self.server.send(NOTIFICATION) + conn.expect('') + self.assertIn('link down', conn.response) + + def testEndOfChunksInAChunk(self): + # Text like the end of the chunks of NETCONF 1.1 behind a tag does not end a message, + # also if the device stalls behind it. + conn = self.subscribe(pause_after=b'\n##\n') + notification = NOTIFICATION.replace('', '\n##\n') + self.server.send(notification) + conn.expect_prompt() + self.assertEqual(conn.response, notification) def testBannerThatIsNoUTF8(self): # SSH banners are UTF-8, too (RFC 4252), but other bytes must not fail the login. From 2278385f1ece7560cd986b3f97b3dfd1e12973e4 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 10 Nov 2026 22:45:15 +0100 Subject: [PATCH 137/250] Run the jobs of the queue in threads only The process mode relied on fork, which can deadlock a process with threads (upstream issue #230) and is the default only on Linux up to Python 3.13. Its process-shared locks also hung the queue under gevent. Queue(mode='multiprocessing') now raises a ValueError. --- CHANGES.md | 13 + docs/migration.rst | 4 + docs/platforms.rst | 16 +- src/Exscript/account.py | 236 +---------------- src/Exscript/logger.py | 69 +---- src/Exscript/protocols/drivers/ace.py | 4 +- src/Exscript/protocols/drivers/adtran.py | 4 +- src/Exscript/protocols/drivers/aruba.py | 4 +- src/Exscript/protocols/drivers/bigip.py | 4 +- src/Exscript/protocols/drivers/brocade.py | 4 +- src/Exscript/protocols/drivers/driver.py | 4 +- src/Exscript/protocols/drivers/eos.py | 4 +- src/Exscript/protocols/drivers/ios.py | 4 +- src/Exscript/protocols/drivers/junos_erx.py | 4 +- src/Exscript/protocols/drivers/nxos.py | 4 +- src/Exscript/protocols/drivers/one_os.py | 4 +- src/Exscript/protocols/drivers/rios.py | 4 +- .../protocols/drivers/smart_edge_os.py | 4 +- src/Exscript/protocols/drivers/sp_vxoa.py | 4 +- src/Exscript/protocols/drivers/vxworks.py | 4 +- src/Exscript/protocols/drivers/zte.py | 4 +- src/Exscript/protocols/netconf.py | 16 +- src/Exscript/protocols/protocol.py | 28 +- src/Exscript/queue.py | 242 ++++++------------ src/Exscript/util/impl.py | 5 +- src/Exscript/workqueue/job.py | 77 +++--- src/Exscript/workqueue/mainloop.py | 4 - src/Exscript/workqueue/pipeline.py | 2 +- src/Exscript/workqueue/workqueue.py | 30 ++- tests/Exscript/QueueTest.py | 65 ++--- tests/Exscript/util/implTest.py | 2 +- tests/Exscript/workqueue/JobTest.py | 35 ++- tests/Exscript/workqueue/MainLoopTest.py | 4 +- 33 files changed, 245 insertions(+), 667 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index fde0e12d..a3c9f775 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -596,6 +596,19 @@ The names below refer to functions callable from templates, not protocol methods ## Multi-Threading +- **BREAKING - Threads Only:** The queue runs its jobs in threads only. + `Queue(mode='multiprocessing')` and `WorkQueue(mode='multiprocessing')` raise a + `ValueError`. The mode needed the `fork` start method, which is the default only on + Linux up to Python 3.13, and a process with threads that forks can deadlock (upstream + issue #230). For several CPU cores, use the free-threaded build of Python, or start + processes yourself and create a queue in each. +- **BREAKING - No Pipes and Proxies:** Jobs take their accounts from the account manager + and write to their loggers directly, without a pipe and a thread per job. Removed: + `Exscript.account.AccountProxy`, `Exscript.logger.LoggerProxy`, the duplicate + `Exscript.account.LoggerProxy`, `Exscript.workqueue.job.Process` and + `Queue._create_pipe()`. The queue, accounts and account pools use the locks of + `threading` instead of those of `multiprocessing`. + ## Demos - Updated the demo scripts to match the latest Exscript version. diff --git a/docs/migration.rst b/docs/migration.rst index 49d2864a..90cd4017 100644 --- a/docs/migration.rst +++ b/docs/migration.rst @@ -205,6 +205,10 @@ Other renamed or removed names: under a service manager such as systemd. - ``WorkQueue`` has no ``debug`` argument, and ``set_debug()`` takes ``level``. Pass the other arguments by keyword. +- ``Queue(mode='multiprocessing')`` raises a ``ValueError``: jobs run in + threads only. For several CPU cores, see :doc:`platforms`. +- ``Exscript.account.AccountProxy`` and ``Exscript.logger.LoggerProxy`` are + removed; jobs use the accounts and the loggers directly. - ``parselib.Lexer.expect()`` and ``current_is()`` take ``types`` instead of ``type``. - The ``dump1()`` methods of the template interpreter are removed, and so diff --git a/docs/platforms.rst b/docs/platforms.rst index 46c78236..9aea2113 100644 --- a/docs/platforms.rst +++ b/docs/platforms.rst @@ -31,14 +31,14 @@ Windows - The default known_hosts file is ``%USERPROFILE%\.ssh\known_hosts`` (:doc:`security`). -The queue's process mode ------------------------- - -``Queue(mode='multiprocessing')`` needs the ``fork`` start method of -:mod:`multiprocessing`, which is the default only on Linux with Python -3.12 and 3.13. On other platforms and versions, use the default thread -mode, which is also what :func:`Exscript.util.start.start` uses. The -process mode warns that it is not recommended. +Threads and processes +--------------------- + +The queue runs each job in a thread; Exscript 3.0 removed its process +mode. To use several CPU cores, run the free-threaded build (below), or +start several processes yourself, for example with a pool of +:mod:`concurrent.futures` or the prefork pool of Celery, and create the +queue or the connections in the child processes. Free-threaded Python -------------------- diff --git a/src/Exscript/account.py b/src/Exscript/account.py index 092147e4..4b981469 100644 --- a/src/Exscript/account.py +++ b/src/Exscript/account.py @@ -24,7 +24,7 @@ Manages user accounts. """ -import multiprocessing +import threading from collections import defaultdict, deque from collections.abc import Callable, Hashable, Iterable from types import TracebackType @@ -37,10 +37,6 @@ from .util.impl import Context if TYPE_CHECKING: - # _ConnectionBase is the common base class of the pipe connections on - # POSIX (Connection) and on Windows (PipeConnection). - from multiprocessing.connection import _ConnectionBase - from .host import Host @@ -97,8 +93,8 @@ def __init__( self.password = password self.authorization_password = password2 self.key = key - self.synclock = multiprocessing.Condition(multiprocessing.Lock()) - self.lock = multiprocessing.Lock() + self.synclock = threading.Condition(threading.Lock()) + self.lock = threading.Lock() self.needs_lock = needs_lock def __repr__(self) -> str: @@ -228,230 +224,6 @@ def get_key(self) -> PrivateKey | None: return self.key -class AccountProxy: - """ - An object that has a 1:1 relation to an account object in another - process. - """ - - def __init__(self, parent: '_ConnectionBase') -> None: - """ - Constructor. - - :type parent: multiprocessing.Connection - :param parent: A pipe to the associated account manager. - """ - self.parent = parent - self.account_hash: int | None = None - self.host: Host | None = None - self.user: str | None = None - self.password: str | None = None - self.authorization_password: str | None = None - self.key: PrivateKey | None = None - - @staticmethod - def for_host(parent: '_ConnectionBase', host: 'Host') -> 'AccountProxy | None': - """ - Returns a new AccountProxy that has an account acquired. The - account is chosen based on what the connected AccountManager - selects for the given host. Returns None if the account manager - returns no account. - """ - account = AccountProxy(parent) - account.host = host - if account.acquire(): - return account - return None - - @staticmethod - def for_account_hash(parent: '_ConnectionBase', account_hash: int) -> 'AccountProxy | None': - """ - Returns a new AccountProxy that acquires the account with the - given hash, if such an account is known to the account manager. - Returns None if the account manager does not have such an - account. - """ - account = AccountProxy(parent) - account.account_hash = account_hash - if account.acquire(): - return account - return None - - @staticmethod - def for_random_account(parent: '_ConnectionBase') -> 'AccountProxy | None': - """ - Returns a new AccountProxy that has an account acquired. The - account is chosen by the connected AccountManager. Returns None - if the account manager returns no account. - """ - account = AccountProxy(parent) - if account.acquire(): - return account - return None - - def __hash__(self) -> int: - """ - Returns the hash of the currently acquired account. Until an - account was acquired, the hash is None, and hash() raises a - TypeError. - """ - # account_hash is None until an account is acquired; the cast keeps - # the int return type that hash() requires. - return cast(int, self.account_hash) - - def __enter__(self) -> bool: - """ - Like :class:`acquire()`. - """ - return self.acquire() - - def __exit__( - self, - thetype: type[BaseException] | None, - value: BaseException | None, - traceback: TracebackType | None, - ) -> None: - """ - Like :class:`release()`. - """ - return self.release() - - def context(self) -> Context: - """ - When you need a 'with' context for an already-acquired account. - """ - return Context(self) - - def acquire(self) -> bool: - """ - Locks the account. Returns True on success, False if the account - is thread-local and must not be locked. - """ - if self.host: - self.parent.send(('acquire-account-for-host', self.host)) - elif self.account_hash: - self.parent.send(('acquire-account-from-hash', self.account_hash)) - else: - self.parent.send('acquire-account') - - response = self.parent.recv() - if isinstance(response, Exception): - raise response - if response is None: - return False - - self.account_hash, self.user, self.password, self.authorization_password, self.key = ( - response - ) - return True - - def release(self) -> None: - """ - Unlocks the account. - """ - self.parent.send(('release-account', self.account_hash)) - - response = self.parent.recv() - if isinstance(response, Exception): - raise response - - if response != 'ok': - raise ValueError('unexpected response: ' + repr(response)) - - def get_name(self) -> str | None: - """ - Returns the name of the account. - - :rtype: string|None - :return: The account name, or None if no account was acquired. - """ - return self.user - - def get_password(self) -> str | None: - """ - Returns the password of the account. - - :rtype: string|None - :return: The account password, or None if it is unknown. - """ - return self.password - - def get_authorization_password(self) -> str | None: - """ - Returns the authorization password of the account, or the - authentication password if no authorization password is set. - - :rtype: string|None - :return: The account password. - """ - return self.authorization_password or self.password - - def get_key(self) -> PrivateKey | None: - """ - Returns the key of the account, if any. - - :rtype: Exscript.PrivateKey|None - :return: A key object. - """ - return self.key - - -class LoggerProxy: - """ - An object that has a 1:1 relation to a Logger object in another - process. - """ - - def __init__(self, parent: '_ConnectionBase', logger_id: int) -> None: - """ - Constructor. - - :type parent: multiprocessing.Connection - :param parent: A pipe to the associated pipe handler. - :param logger_id: The id() of the Logger in the other process. - """ - self.parent = parent - self.logger_id = logger_id - - def add_log(self, job_id: int, name: str, attempt: int) -> Log | None: - """ - Asks the logger in the other process to create a log for a new - attempt of the given job. - - :return: A copy of the new log, or None if the logger no longer - exists. - """ - self.parent.send(('log-add', (self.logger_id, job_id, name, attempt))) - response = self.parent.recv() - if isinstance(response, Exception): - raise response - # The pipe returns unpickled objects without a static type. - return cast(Log | None, response) - - def log(self, job_id: int, message: str) -> None: - """ - Sends the message to the log of the given job in the other process. - """ - self.parent.send(('log-message', (self.logger_id, job_id, message))) - - def log_aborted( - self, job_id: int, exc_info: tuple[type[BaseException], BaseException, str] - ) -> None: - """ - Tells the logger in the other process that the given job failed. - - :param exc_info: The exception, as returned by - :func:`Exscript.util.impl.serializeable_sys_exc_info()`. - """ - self.parent.send(('log-aborted', (self.logger_id, job_id, exc_info))) - - def log_succeeded(self, job_id: int) -> None: - """ - Tells the logger in the other process that the given job succeeded. - """ - self.parent.send(('log-succeeded', (self.logger_id, job_id))) - - class AccountPool: """ This class manages a collection of available accounts. @@ -468,7 +240,7 @@ def __init__(self, accounts: Account | Iterable[Account] | None = None) -> None: self.unlocked_accounts: deque[Account] = deque() self.owner2account: defaultdict[Hashable, list[Account]] = defaultdict(list) self.account2owner: dict[Account, Hashable] = dict() - self.unlock_cond = multiprocessing.Condition(multiprocessing.RLock()) + self.unlock_cond = threading.Condition(threading.RLock()) if accounts: self.add_account(accounts) diff --git a/src/Exscript/logger.py b/src/Exscript/logger.py index aa2fdd63..3e99baa5 100644 --- a/src/Exscript/logger.py +++ b/src/Exscript/logger.py @@ -39,11 +39,6 @@ from .util.impl import format_exception -if TYPE_CHECKING: - # _ConnectionBase is the common base class of the pipe connections on - # POSIX (Connection) and on Windows (PipeConnection). - from multiprocessing.connection import _ConnectionBase - # ----------------------------------------------------------------------------- # Python Logging @@ -424,8 +419,7 @@ def log(self, job_id: int, message: str) -> None: """ Appends the message to the log of the last attempt of the given job. """ - # This method is called whenever a sub thread sends a log message - # via a pipe. (See LoggerProxy and Queue.PipeHandler) + # The job threads of a queue call this for the data of their sessions. log = self._get_log(job_id) log.write(message) @@ -451,67 +445,6 @@ def log_succeeded(self, job_id: int) -> None: self.success += 1 -class LoggerProxy: - """ - An object that has a 1:1 relation to a Logger object in another - process. - """ - - def __init__(self, parent: '_ConnectionBase', logger_id: int) -> None: - """ - Constructor. - - :type parent: multiprocessing.Connection - :param parent: A pipe to the associated pipe handler. - :param logger_id: The id() of the Logger in the other process. - """ - self.parent = parent - self.logger_id = logger_id - self._lock = threading.Lock() - - def add_log(self, job_id: int, name: str, attempt: int) -> Log | None: - """ - Asks the logger in the other process to create a log for a new - attempt of the given job. - - :return: A copy of the new log, or None if the logger no longer - exists. - """ - with self._lock: - self.parent.send(('log-add', (self.logger_id, job_id, name, attempt))) - response = self.parent.recv() - if isinstance(response, Exception): - raise response - # The pipe returns unpickled objects without a static type. - return cast(Log | None, response) - - def log(self, job_id: int, message: str) -> None: - """ - Sends the message to the log of the given job in the other process. - """ - with self._lock: - self.parent.send(('log-message', (self.logger_id, job_id, message))) - - def log_aborted( - self, job_id: int, exc_info: tuple[type[BaseException], BaseException, str] - ) -> None: - """ - Tells the logger in the other process that the given job failed. - - :param exc_info: The exception, as returned by - :func:`Exscript.util.impl.serializeable_sys_exc_info()`. - """ - with self._lock: - self.parent.send(('log-aborted', (self.logger_id, job_id, exc_info))) - - def log_succeeded(self, job_id: int) -> None: - """ - Tells the logger in the other process that the given job succeeded. - """ - with self._lock: - self.parent.send(('log-succeeded', (self.logger_id, job_id))) - - class FileLogger(Logger): """ A Logger that stores logs into files. diff --git a/src/Exscript/protocols/drivers/ace.py b/src/Exscript/protocols/drivers/ace.py index 6848ed3d..94e58f33 100644 --- a/src/Exscript/protocols/drivers/ace.py +++ b/src/Exscript/protocols/drivers/ace.py @@ -30,7 +30,7 @@ from .driver import Driver if TYPE_CHECKING: - from ...account import Account, AccountProxy + from ...account import Account from ...util.impl import _Context from ..protocol import Protocol @@ -82,7 +82,7 @@ def init_terminal(self, conn: 'Protocol') -> None: def auto_authorize( self, conn: 'Protocol', - account: 'Account | AccountProxy | _Context | None', + account: 'Account | _Context | None', flush: bool, bailout: bool, ) -> None: diff --git a/src/Exscript/protocols/drivers/adtran.py b/src/Exscript/protocols/drivers/adtran.py index 292050ae..e9ef0788 100644 --- a/src/Exscript/protocols/drivers/adtran.py +++ b/src/Exscript/protocols/drivers/adtran.py @@ -32,7 +32,7 @@ from .driver import Driver if TYPE_CHECKING: - from ...account import Account, AccountProxy + from ...account import Account from ...util.impl import _Context from ..protocol import Protocol @@ -60,7 +60,7 @@ def init_terminal(self, conn: 'Protocol') -> None: def auto_authorize( self, conn: 'Protocol', - account: 'Account | AccountProxy | _Context | None', + account: 'Account | _Context | None', flush: bool, bailout: bool, ) -> None: diff --git a/src/Exscript/protocols/drivers/aruba.py b/src/Exscript/protocols/drivers/aruba.py index 8d1e7e2f..3e7d97b6 100644 --- a/src/Exscript/protocols/drivers/aruba.py +++ b/src/Exscript/protocols/drivers/aruba.py @@ -30,7 +30,7 @@ from .driver import Driver if TYPE_CHECKING: - from ...account import Account, AccountProxy + from ...account import Account from ...util.impl import _Context from ..protocol import Protocol @@ -81,7 +81,7 @@ def init_terminal(self, conn: 'Protocol') -> None: def auto_authorize( self, conn: 'Protocol', - account: 'Account | AccountProxy | _Context | None', + account: 'Account | _Context | None', flush: bool, bailout: bool, ) -> None: diff --git a/src/Exscript/protocols/drivers/bigip.py b/src/Exscript/protocols/drivers/bigip.py index 051518e4..546fa5d6 100644 --- a/src/Exscript/protocols/drivers/bigip.py +++ b/src/Exscript/protocols/drivers/bigip.py @@ -30,7 +30,7 @@ from .driver import Driver if TYPE_CHECKING: - from ...account import Account, AccountProxy + from ...account import Account from ...util.impl import _Context from ..protocol import Protocol @@ -78,7 +78,7 @@ def init_terminal(self, conn: 'Protocol') -> None: def auto_authorize( self, conn: 'Protocol', - account: 'Account | AccountProxy | _Context | None', + account: 'Account | _Context | None', flush: bool, bailout: bool, ) -> None: diff --git a/src/Exscript/protocols/drivers/brocade.py b/src/Exscript/protocols/drivers/brocade.py index 609a1066..fac9fd82 100644 --- a/src/Exscript/protocols/drivers/brocade.py +++ b/src/Exscript/protocols/drivers/brocade.py @@ -30,7 +30,7 @@ from .driver import Driver if TYPE_CHECKING: - from ...account import Account, AccountProxy + from ...account import Account from ...util.impl import _Context from ..protocol import Protocol @@ -89,7 +89,7 @@ def init_terminal(self, conn: 'Protocol') -> None: def auto_authorize( self, conn: 'Protocol', - account: 'Account | AccountProxy | _Context | None', + account: 'Account | _Context | None', flush: bool, bailout: bool, ) -> None: diff --git a/src/Exscript/protocols/drivers/driver.py b/src/Exscript/protocols/drivers/driver.py index 4be6af20..78a285e9 100644 --- a/src/Exscript/protocols/drivers/driver.py +++ b/src/Exscript/protocols/drivers/driver.py @@ -29,7 +29,7 @@ from typing import TYPE_CHECKING, Any if TYPE_CHECKING: - from ...account import Account, AccountProxy + from ...account import Account from ...util.impl import _Context from ..protocol import Protocol @@ -249,7 +249,7 @@ def supports_auto_authorize(self) -> bool: def auto_authorize( self, conn: 'Protocol', - account: 'Account | AccountProxy | _Context | None', + account: 'Account | _Context | None', flush: bool, bailout: bool, ) -> None: diff --git a/src/Exscript/protocols/drivers/eos.py b/src/Exscript/protocols/drivers/eos.py index b349d557..db60cc83 100644 --- a/src/Exscript/protocols/drivers/eos.py +++ b/src/Exscript/protocols/drivers/eos.py @@ -30,7 +30,7 @@ from .driver import Driver if TYPE_CHECKING: - from ...account import Account, AccountProxy + from ...account import Account from ...util.impl import _Context from ..protocol import Protocol @@ -109,7 +109,7 @@ def init_terminal(self, conn: 'Protocol') -> None: def auto_authorize( self, conn: 'Protocol', - account: 'Account | AccountProxy | _Context | None', + account: 'Account | _Context | None', flush: bool, bailout: bool, ) -> None: diff --git a/src/Exscript/protocols/drivers/ios.py b/src/Exscript/protocols/drivers/ios.py index 826b0561..3c8f7983 100644 --- a/src/Exscript/protocols/drivers/ios.py +++ b/src/Exscript/protocols/drivers/ios.py @@ -31,7 +31,7 @@ from .driver import Driver if TYPE_CHECKING: - from ...account import Account, AccountProxy + from ...account import Account from ...util.impl import _Context from ..protocol import Protocol @@ -115,7 +115,7 @@ def init_terminal(self, conn: 'Protocol') -> None: def auto_authorize( self, conn: 'Protocol', - account: 'Account | AccountProxy | _Context | None', + account: 'Account | _Context | None', flush: bool, bailout: bool, ) -> None: diff --git a/src/Exscript/protocols/drivers/junos_erx.py b/src/Exscript/protocols/drivers/junos_erx.py index c27071eb..83c2442d 100644 --- a/src/Exscript/protocols/drivers/junos_erx.py +++ b/src/Exscript/protocols/drivers/junos_erx.py @@ -30,7 +30,7 @@ from .driver import Driver if TYPE_CHECKING: - from ...account import Account, AccountProxy + from ...account import Account from ...util.impl import _Context from ..protocol import Protocol @@ -78,7 +78,7 @@ def init_terminal(self, conn: 'Protocol') -> None: def auto_authorize( self, conn: 'Protocol', - account: 'Account | AccountProxy | _Context | None', + account: 'Account | _Context | None', flush: bool, bailout: bool, ) -> None: diff --git a/src/Exscript/protocols/drivers/nxos.py b/src/Exscript/protocols/drivers/nxos.py index f6ab4f02..30938e0d 100644 --- a/src/Exscript/protocols/drivers/nxos.py +++ b/src/Exscript/protocols/drivers/nxos.py @@ -30,7 +30,7 @@ from .driver import Driver if TYPE_CHECKING: - from ...account import Account, AccountProxy + from ...account import Account from ...util.impl import _Context from ..protocol import Protocol @@ -86,7 +86,7 @@ def init_terminal(self, conn: 'Protocol') -> None: def auto_authorize( self, conn: 'Protocol', - account: 'Account | AccountProxy | _Context | None', + account: 'Account | _Context | None', flush: bool, bailout: bool, ) -> None: diff --git a/src/Exscript/protocols/drivers/one_os.py b/src/Exscript/protocols/drivers/one_os.py index 158762e8..50e8f6ab 100644 --- a/src/Exscript/protocols/drivers/one_os.py +++ b/src/Exscript/protocols/drivers/one_os.py @@ -30,7 +30,7 @@ from .driver import Driver if TYPE_CHECKING: - from ...account import Account, AccountProxy + from ...account import Account from ...util.impl import _Context from ..protocol import Protocol @@ -81,7 +81,7 @@ def init_terminal(self, conn: 'Protocol') -> None: def auto_authorize( self, conn: 'Protocol', - account: 'Account | AccountProxy | _Context | None', + account: 'Account | _Context | None', flush: bool, bailout: bool, ) -> None: diff --git a/src/Exscript/protocols/drivers/rios.py b/src/Exscript/protocols/drivers/rios.py index 903681e2..30a29283 100644 --- a/src/Exscript/protocols/drivers/rios.py +++ b/src/Exscript/protocols/drivers/rios.py @@ -30,7 +30,7 @@ from .driver import Driver if TYPE_CHECKING: - from ...account import Account, AccountProxy + from ...account import Account from ...util.impl import _Context from ..protocol import Protocol @@ -73,7 +73,7 @@ def init_terminal(self, conn: 'Protocol') -> None: def auto_authorize( self, conn: 'Protocol', - account: 'Account | AccountProxy | _Context | None', + account: 'Account | _Context | None', flush: bool, bailout: bool, ) -> None: diff --git a/src/Exscript/protocols/drivers/smart_edge_os.py b/src/Exscript/protocols/drivers/smart_edge_os.py index e126f43f..67f1b3fe 100644 --- a/src/Exscript/protocols/drivers/smart_edge_os.py +++ b/src/Exscript/protocols/drivers/smart_edge_os.py @@ -30,7 +30,7 @@ from .driver import Driver if TYPE_CHECKING: - from ...account import Account, AccountProxy + from ...account import Account from ...util.impl import _Context from ..protocol import Protocol @@ -81,7 +81,7 @@ def init_terminal(self, conn: 'Protocol') -> None: def auto_authorize( self, conn: 'Protocol', - account: 'Account | AccountProxy | _Context | None', + account: 'Account | _Context | None', flush: bool, bailout: bool, ) -> None: diff --git a/src/Exscript/protocols/drivers/sp_vxoa.py b/src/Exscript/protocols/drivers/sp_vxoa.py index f1635992..199ce329 100644 --- a/src/Exscript/protocols/drivers/sp_vxoa.py +++ b/src/Exscript/protocols/drivers/sp_vxoa.py @@ -30,7 +30,7 @@ from .driver import Driver if TYPE_CHECKING: - from ...account import Account, AccountProxy + from ...account import Account from ...util.impl import _Context from ..protocol import Protocol @@ -77,7 +77,7 @@ def init_terminal(self, conn: 'Protocol') -> None: def auto_authorize( self, conn: 'Protocol', - account: 'Account | AccountProxy | _Context | None', + account: 'Account | _Context | None', flush: bool, bailout: bool, ) -> None: diff --git a/src/Exscript/protocols/drivers/vxworks.py b/src/Exscript/protocols/drivers/vxworks.py index 893b3b45..a833ed23 100644 --- a/src/Exscript/protocols/drivers/vxworks.py +++ b/src/Exscript/protocols/drivers/vxworks.py @@ -30,7 +30,7 @@ from .driver import Driver if TYPE_CHECKING: - from ...account import Account, AccountProxy + from ...account import Account from ...util.impl import _Context from ..protocol import Protocol @@ -73,7 +73,7 @@ def __init__(self) -> None: def auto_authorize( self, conn: 'Protocol', - account: 'Account | AccountProxy | _Context | None', + account: 'Account | _Context | None', flush: bool, bailout: bool, ) -> None: diff --git a/src/Exscript/protocols/drivers/zte.py b/src/Exscript/protocols/drivers/zte.py index 577f8d58..e6be27c5 100644 --- a/src/Exscript/protocols/drivers/zte.py +++ b/src/Exscript/protocols/drivers/zte.py @@ -30,7 +30,7 @@ from .driver import Driver if TYPE_CHECKING: - from ...account import Account, AccountProxy + from ...account import Account from ...util.impl import _Context from ..protocol import Protocol @@ -74,7 +74,7 @@ def __init__(self) -> None: def auto_authorize( self, conn: 'Protocol', - account: 'Account | AccountProxy | _Context | None', + account: 'Account | _Context | None', flush: bool, bailout: bool, ) -> None: diff --git a/src/Exscript/protocols/netconf.py b/src/Exscript/protocols/netconf.py index bb949c4f..920ea559 100644 --- a/src/Exscript/protocols/netconf.py +++ b/src/Exscript/protocols/netconf.py @@ -50,7 +50,7 @@ from lxml.etree import _Element from paramiko import Channel, Transport - from ..account import Account, AccountProxy + from ..account import Account from ..util.impl import _Context # ncclient and lxml are imported when the first NETCONF session starts, not @@ -535,8 +535,8 @@ def _receive_message(self, message: str) -> None: def login( self, - account: 'Account | AccountProxy | _Context | None' = None, - app_account: 'Account | AccountProxy | _Context | None' = None, + account: 'Account | _Context | None' = None, + app_account: 'Account | _Context | None' = None, flush: bool = True, ) -> None: """ @@ -560,8 +560,8 @@ def login( def authenticate( self, - account: 'Account | AccountProxy | _Context | None' = None, - app_account: 'Account | AccountProxy | _Context | None' = None, + account: 'Account | _Context | None' = None, + app_account: 'Account | _Context | None' = None, flush: bool = True, ) -> None: """ @@ -583,7 +583,7 @@ def authenticate( def app_authenticate( self, - account: 'Account | AccountProxy | _Context | None' = None, + account: 'Account | _Context | None' = None, flush: bool = True, bailout: bool = False, ) -> None: @@ -595,7 +595,7 @@ def app_authenticate( def app_authorize( self, - account: 'Account | AccountProxy | _Context | None' = None, + account: 'Account | _Context | None' = None, flush: bool = True, bailout: bool = False, ) -> None: @@ -607,7 +607,7 @@ def app_authorize( def auto_app_authorize( self, - account: 'Account | AccountProxy | _Context | None' = None, + account: 'Account | _Context | None' = None, flush: bool = True, bailout: bool = False, ) -> None: diff --git a/src/Exscript/protocols/protocol.py b/src/Exscript/protocols/protocol.py index a1b6acdb..5bd52fd2 100644 --- a/src/Exscript/protocols/protocol.py +++ b/src/Exscript/protocols/protocol.py @@ -63,7 +63,7 @@ if TYPE_CHECKING: from _typeshed import FileDescriptorLike - from ..account import Account, AccountProxy + from ..account import Account from ..key import PrivateKey try: @@ -254,9 +254,7 @@ def __init__( disabled_algorithms: dict[str, list[str]] | None = None, # SSH2 only preferred_authentications: list[str] | None = None, # SSH2 only verify_fingerprint: bool = True, # SSH2 only - account_factory: ( - Callable[['Account | AccountProxy | None'], 'Account | AccountProxy'] | None - ) = None, + account_factory: (Callable[['Account | None'], 'Account'] | None) = None, banner_timeout: float = 20, # SSH2 only encoding: str = 'latin-1', *, @@ -556,7 +554,7 @@ def _init(self) -> None: self._auto_driver = driver_map[self.guess_os()] self._driver_replaced = False self._expect_canceled = False - self._last_account: Account | AccountProxy | None = None + self._last_account: Account | None = None # The buffer position at the last send(); None before the first one. self._sent_at: int | None = None @@ -1120,7 +1118,7 @@ def connect(self, hostname: str | None = None, port: int | None = None) -> bool: self._update_auto_driver() return self._connected - def _get_account(self, account: 'Account | AccountProxy | _Context | None') -> _Context: + def _get_account(self, account: 'Account | _Context | None') -> _Context: """ Returns a context for the given account, or for that of the last call if None is given, and keeps the account for the next call; the account @@ -1147,8 +1145,8 @@ def _get_account(self, account: 'Account | AccountProxy | _Context | None') -> _ def login( self, - account: 'Account | AccountProxy | _Context | None' = None, - app_account: 'Account | AccountProxy | _Context | None' = None, + account: 'Account | _Context | None' = None, + app_account: 'Account | _Context | None' = None, flush: bool = True, ) -> None: """ @@ -1186,8 +1184,8 @@ def login( def authenticate( self, - account: 'Account | AccountProxy | _Context | None' = None, - app_account: 'Account | AccountProxy | _Context | None' = None, + account: 'Account | _Context | None' = None, + app_account: 'Account | _Context | None' = None, flush: bool = True, ) -> None: """ @@ -1223,9 +1221,7 @@ def _protocol_authenticate_by_key(self, user: str, key: 'PrivateKey') -> None: """ pass - def protocol_authenticate( - self, account: 'Account | AccountProxy | _Context | None' = None - ) -> None: + def protocol_authenticate(self, account: 'Account | _Context | None' = None) -> None: """ Low-level API to perform protocol-level authentication on protocols that support it. @@ -1367,7 +1363,7 @@ def _app_authenticate( def app_authenticate( self, - account: 'Account | AccountProxy | _Context | None' = None, + account: 'Account | _Context | None' = None, flush: bool = True, bailout: bool = False, ) -> None: @@ -1426,7 +1422,7 @@ def is_app_authenticated(self) -> bool: def app_authorize( self, - account: 'Account | AccountProxy | _Context | None' = None, + account: 'Account | _Context | None' = None, flush: bool = True, bailout: bool = False, ) -> None: @@ -1463,7 +1459,7 @@ def app_authorize( def auto_app_authorize( self, - account: 'Account | AccountProxy | _Context | None' = None, + account: 'Account | _Context | None' = None, flush: bool = True, bailout: bool = False, ) -> None: diff --git a/src/Exscript/queue.py b/src/Exscript/queue.py index cccdf678..c1ed5fc3 100644 --- a/src/Exscript/queue.py +++ b/src/Exscript/queue.py @@ -28,12 +28,10 @@ import logging import sys import threading -import weakref from functools import partial -from multiprocessing import Pipe -from .account import AccountManager, AccountProxy -from .logger import LoggerProxy, logger_registry +from .account import AccountManager +from .logger import logger_registry from .protocols import prepare from .util.cast import to_hosts from .util.decorator import get_label @@ -55,156 +53,100 @@ def flush(self): pass -def _account_factory(accm, host, account): +def _account_factory(accm, owner, host, account): + """ + Returns the account to log into the host with, acquired: the given + account, else the account of the host, else the one that the account + manager chooses for the host. Accounts of the pools of the manager are + acquired for the given owner, so release_accounts() releases them. + """ if account is None: account = host.get_account() - - # Specific account requested? - if account: - acquired = AccountProxy.for_account_hash(accm, account.__hash__()) - else: - acquired = AccountProxy.for_host(accm, host) - - # Thread-local accounts don't need a remote proxy. - if acquired: - return acquired - account.acquire() - return account + if account is None: + return accm.acquire_account_for(host, owner) + return accm.acquire_account(account, owner) -def _prepare_connection(func): +def _prepare_connection(func, account_manager): """ A decorator that unpacks the host and connection from the job argument - and passes them as separate arguments to the wrapped function. + and passes them as separate arguments to the wrapped function. The + connection takes its accounts from the given account manager; those + that the job still holds are released when the function returns. """ def _wrapped(job, *args, **kwargs): - job_id = id(job) - to_parent = job.data['pipe'] - host = job.data['host'] - - # Create a protocol adapter. - mkaccount = partial(_account_factory, to_parent, host) - pargs = {'account_factory': mkaccount, 'stdout': job.data['stdout']} - pargs.update(host.get_options()) - conn = prepare(host, **pargs) - - # Connect and run the function. - log_options = get_label(func, 'log_to') - if log_options is not None: - # Enable logging. - proxy = LoggerProxy(to_parent, log_options['logger_id']) - log_cb = partial(proxy.log, job_id) - proxy.add_log(job_id, job.name, job.failures + 1) - conn.data_received_event.listen(log_cb) - if not conn._receives_echo: - # The sent commands are either logged via the data_received_event - # when they are echoed back, or they are logged here explicitly. - conn.data_sent_event.listen(log_cb) - try: - conn.connect(host.get_address(), host.get_tcp_port()) - result = func(job, host, conn, *args, **kwargs) - conn.close(force=True) - except: - proxy.log_aborted(job_id, serializeable_sys_exc_info()) - raise - else: - proxy.log_succeeded(job_id) - finally: - conn.data_received_event.disconnect(log_cb) - if not conn._receives_echo: - conn.data_sent_event.disconnect(log_cb) - else: - conn.connect(host.get_address(), host.get_tcp_port()) - result = func(job, host, conn, *args, **kwargs) - conn.close(force=True) - return result + try: + return _run_job(func, account_manager, job, *args, **kwargs) + finally: + account_manager.release_accounts(job) return _wrapped +def _run_job(func, account_manager, job, *args, **kwargs): + """ + Connects to the host of the given job and calls the given function with + the job, the host and the connection, and the given arguments. Logs the + session if the function was decorated with log_to(). + """ + job_id = id(job) + host = job.data['host'] + + # Create a protocol adapter. + mkaccount = partial(_account_factory, account_manager, job, host) + pargs = {'account_factory': mkaccount, 'stdout': job.data['stdout']} + pargs.update(host.get_options()) + conn = prepare(host, **pargs) + + # Connect and run the function. + log_options = get_label(func, 'log_to') + if log_options is None: + conn.connect(host.get_address(), host.get_tcp_port()) + result = func(job, host, conn, *args, **kwargs) + conn.close(force=True) + return result + + logger_id = log_options['logger_id'] + log_cb = partial(_call_logger, 'log', logger_id, job_id) + _call_logger('add_log', logger_id, job_id, job.name, job.failures + 1) + conn.data_received_event.listen(log_cb) + if not conn._receives_echo: + # The sent commands are either logged via the data_received_event + # when they are echoed back, or they are logged here explicitly. + conn.data_sent_event.listen(log_cb) + try: + conn.connect(host.get_address(), host.get_tcp_port()) + result = func(job, host, conn, *args, **kwargs) + conn.close(force=True) + except: + _call_logger('log_aborted', logger_id, job_id, serializeable_sys_exc_info()) + raise + else: + _call_logger('log_succeeded', logger_id, job_id) + finally: + conn.data_received_event.disconnect(log_cb) + if not conn._receives_echo: + conn.data_sent_event.disconnect(log_cb) + return result + + def _is_recoverable_error(cls): - # Hack: We can't use isinstance(), because the classes may - # have been created by another python process; apparently this - # will cause isinstance() to return False. + # By name, so that this module does not import the template language. return cls.__name__ in ('CompileError', 'FailException') def _call_logger(funcname, logger_id, *args): + """ + Calls the method with the given name of the logger with the given id, + if the logger still exists, with the given arguments. + """ logger = logger_registry.get(logger_id) if not logger: return return getattr(logger, funcname)(*args) -class _PipeHandler(threading.Thread): - """ - Each PipeHandler holds an open pipe to a subprocess, to allow the - sub-process to access the accounts and communicate status information. - """ - - def __init__(self, account_manager): - threading.Thread.__init__(self) - self.daemon = True - self.accm = account_manager - self.to_child, self.to_parent = Pipe() - - def _send_account(self, account): - if account is None: - self.to_child.send(account) - return - response = ( - account.__hash__(), - account.get_name(), - account.get_password(), - account.get_authorization_password(), - account.get_key(), - ) - self.to_child.send(response) - - def _handle_request(self, request): - try: - command, arg = request - if command == 'acquire-account-for-host': - account = self.accm.acquire_account_for(arg, self) - self._send_account(account) - elif command == 'acquire-account-from-hash': - account = self.accm.get_account_from_hash(arg) - if account is not None: - account = self.accm.acquire_account(account, self) - self._send_account(account) - elif command == 'acquire-account': - account = self.accm.acquire_account(owner=self) - self._send_account(account) - elif command == 'release-account': - account = self.accm.get_account_from_hash(arg) - account.release() - self.to_child.send('ok') - elif command == 'log-add': - log = _call_logger('add_log', *arg) - self.to_child.send(log) - elif command == 'log-message': - _call_logger('log', *arg) - elif command == 'log-aborted': - _call_logger('log_aborted', *arg) - elif command == 'log-succeeded': - _call_logger('log_succeeded', *arg) - else: - raise Exception('invalid command on pipe: ' + repr(command)) - except Exception as e: - self.to_child.send(e) - raise - - def run(self): - while True: - try: - request = self.to_child.recv() - except (OSError, EOFError): - self.accm.release_accounts(self) - break - self._handle_request(request) - - class Queue: """ Manages hosts/tasks, accounts, connections, and threads. @@ -250,7 +192,8 @@ def __init__( :type verbose: int :param verbose: The verbosity level. :type mode: str - :param mode: 'multiprocessing' or 'threading' + :param mode: 'threading', the only mode. 'multiprocessing' raises a + ValueError, any other mode a TypeError. :type max_threads: int :param max_threads: The maximum number of concurrent threads. :type host_driver: str @@ -266,7 +209,6 @@ def __init__( self.workqueue = WorkQueue(mode=mode) self.workqueue.set_debug(verbose) self.account_manager = AccountManager() - self.pipe_handlers = weakref.WeakValueDictionary() self.domain = domain self.verbose = verbose self.stdout = stdout @@ -326,32 +268,6 @@ def _write(self, channel, msg): self.channel_map[channel].write(msg) self.channel_map[channel].flush() - def _create_pipe(self): - """ - Creates a new pipe and returns the child end of the connection. - To request an account from the pipe, use:: - - pipe = queue._create_pipe() - - # Let the account manager choose an account. - pipe.send(('acquire-account-for-host', host)) - account = pipe.recv() - ... - pipe.send(('release-account', account.id())) - - # Or acquire a specific account. - pipe.send(('acquire-account', account.id())) - account = pipe.recv() - ... - pipe.send(('release-account', account.id())) - - pipe.close() - """ - child = _PipeHandler(self.account_manager) - self.pipe_handlers[id(child)] = child - child.start() - return child.to_parent - def _del_status_bar(self): if self.status_bar_length == 0: return @@ -407,12 +323,8 @@ def _dbg(self, level, msg, exc_info=None): def _on_job_init(self, job): if job.data is None: job.data = {} - job.data['pipe'] = self._create_pipe() job.data['stdout'] = self.channel_map['connection'] - def _on_job_destroy(self, job): - job.data['pipe'].close() - def _on_job_started(self, job): self._del_status_bar() self._print_status_bar() @@ -429,14 +341,12 @@ def _on_job_error(self, job, exc_info): self._print('fatal_errors', trace) def _on_job_succeeded(self, job): - self._on_job_destroy(job) self.completed += 1 self._del_status_bar() self._write('status_bar', job.name + ' succeeded.' + '\n') self._print_status_bar(exclude=job.name) def _on_job_aborted(self, job): - self._on_job_destroy(job) self.completed += 1 self.failed += 1 self._del_status_bar() @@ -548,8 +458,6 @@ def join(self): """ self._dbg(2, 'Waiting for the queue to finish.') self.workqueue.wait_until_done() - for child in list(self.pipe_handlers.values()): - child.join() self._dbg(3, 'Queue has finished.') self._del_status_bar() self._print_status_bar() @@ -615,7 +523,7 @@ def reset(self): def _run(self, hosts, callback, queue_function, *args): hosts = to_hosts(hosts, default_domain=self.domain) self.total += len(hosts) - callback = _prepare_connection(callback) + callback = _prepare_connection(callback, self.account_manager) task = Task(self.workqueue) for host in hosts: name = host.get_name() diff --git a/src/Exscript/util/impl.py b/src/Exscript/util/impl.py index 0ac829ec..db4d8d41 100644 --- a/src/Exscript/util/impl.py +++ b/src/Exscript/util/impl.py @@ -25,6 +25,7 @@ """ import sys +import threading import traceback import warnings from collections.abc import Callable @@ -155,9 +156,7 @@ def wrapped(self: Any, *args: Any, **kwargs: Any) -> R: try: rlock = self._sync_lock except AttributeError: - from multiprocessing import RLock - - rlock = self.__dict__.setdefault('_sync_lock', RLock()) + rlock = self.__dict__.setdefault('_sync_lock', threading.RLock()) with rlock: return func(self, *args, **kwargs) diff --git a/src/Exscript/workqueue/job.py b/src/Exscript/workqueue/job.py index 560a887e..a1970029 100644 --- a/src/Exscript/workqueue/job.py +++ b/src/Exscript/workqueue/job.py @@ -20,76 +20,57 @@ # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -import multiprocessing import sys import threading -from copy import copy from functools import partial -from multiprocessing import Pipe from Exscript.util.impl import serializeable_sys_exc_info class _ChildWatcher(threading.Thread): + """ + Starts the thread of a job, waits for it to end, and calls the callback + with the information about its exception, or with None. + """ + def __init__(self, child, callback): threading.Thread.__init__(self) self.child = child self.cb = callback - def __copy__(self): - watcher = _ChildWatcher(copy(self.child), self.cb) - return watcher - def run(self): - to_child, to_self = Pipe() try: - self.child.start(to_self) - result = to_child.recv() + self.child.start() self.child.join() + result = self.child.exc_info except: result = sys.exc_info() - finally: - to_child.close() - to_self.close() - if result == '': - self.cb(None) - else: - self.cb(result) - - -def _make_process_class(base, clsname): - class process_cls(base): - def __init__(self, id, function, name, data): - base.__init__(self, name=name) - self.id = id - self.pipe = None - self.function = function - self.failures = 0 - self.data = data - - def run(self): - """ - Start the associated function. - """ - try: - self.function(self) - except: - self.pipe.send(serializeable_sys_exc_info()) - else: - self.pipe.send('') - finally: - self.pipe = None + self.cb(result) - def start(self, pipe): - self.pipe = pipe - base.start(self) - process_cls.__name__ = clsname - return process_cls +class Thread(threading.Thread): + """ + Runs the function of a job, which gets this object: it has the id, the + name, the data and the failures of the job. After an exception of the + function, exc_info holds the information about it, else None. + """ + def __init__(self, id, function, name, data): + threading.Thread.__init__(self, name=name) + self.id = id + self.function = function + self.failures = 0 + self.data = data + self.exc_info = None -Thread = _make_process_class(threading.Thread, 'Thread') -Process = _make_process_class(multiprocessing.Process, 'Process') + def run(self): + """ + Runs the function of the job. + """ + try: + self.function(self) + except: + self.exc_info = serializeable_sys_exc_info() class Job: diff --git a/src/Exscript/workqueue/mainloop.py b/src/Exscript/workqueue/mainloop.py index 15baeea2..99bb5192 100644 --- a/src/Exscript/workqueue/mainloop.py +++ b/src/Exscript/workqueue/mainloop.py @@ -21,7 +21,6 @@ # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. import logging -import multiprocessing import threading from ..util.event import Event @@ -30,9 +29,6 @@ _logger = logging.getLogger(__name__) _logger.addHandler(logging.NullHandler()) -# See http://bugs.python.org/issue1731717 -multiprocessing.process._cleanup = lambda: None # type: ignore[attr-defined] - class MainLoop(threading.Thread): def __init__(self, collection, job_cls): diff --git a/src/Exscript/workqueue/pipeline.py b/src/Exscript/workqueue/pipeline.py index 6b5f6915..266dcd7b 100644 --- a/src/Exscript/workqueue/pipeline.py +++ b/src/Exscript/workqueue/pipeline.py @@ -21,7 +21,7 @@ # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. from collections import deque -from multiprocessing import Condition, RLock +from threading import Condition, RLock from uuid import uuid4 diff --git a/src/Exscript/workqueue/workqueue.py b/src/Exscript/workqueue/workqueue.py index 9d3b3fce..31c5291b 100644 --- a/src/Exscript/workqueue/workqueue.py +++ b/src/Exscript/workqueue/workqueue.py @@ -23,7 +23,7 @@ import logging from ..util.event import Event -from .job import Process, Thread +from .job import Thread from .mainloop import MainLoop from .pipeline import Pipeline @@ -31,6 +31,21 @@ _logger.addHandler(logging.NullHandler()) +def _check_mode(mode): + """ + Raises an exception unless the given mode is 'threading': a ValueError + for 'multiprocessing', else a TypeError. + """ + if mode == 'multiprocessing': + raise ValueError( + "mode='multiprocessing' was removed in Exscript 3.0: jobs run in threads. " + 'For several CPU cores, run several processes yourself, or use the ' + 'free-threaded build of Python.' + ) + if mode != 'threading': + raise TypeError('invalid "mode" argument: ' + repr(mode)) + + class WorkQueue: """ This class implements the asynchronous workqueue and is the main API @@ -43,17 +58,12 @@ def __init__(self, collection=None, max_threads=1, mode='threading'): :type max_threads: int :param max_threads: The maximum number of concurrent threads. + :type mode: str + :param mode: 'threading', the only mode; see _check_mode(). """ self._logger = _logger - if mode == 'threading': - self.job_cls = Thread - elif mode == 'multiprocessing': - self._logger.warning( - "Mode 'multiprocessing' is not recommended, see: https://github.com/knipknap/exscript/issues/230" - ) - self.job_cls = Process - else: - raise TypeError('invalid "mode" argument: ' + repr(mode)) + _check_mode(mode) + self.job_cls = Thread if collection is None: self.collection = Pipeline(max_threads) else: diff --git a/tests/Exscript/QueueTest.py b/tests/Exscript/QueueTest.py index 3ed972e4..0b7d3ef9 100644 --- a/tests/Exscript/QueueTest.py +++ b/tests/Exscript/QueueTest.py @@ -7,10 +7,10 @@ import ctypes import shutil +import threading import time from functools import partial from multiprocessing import Value -from multiprocessing.managers import BaseManager from tempfile import mkdtemp from Exscript import Account, AccountPool, FileLogger, Queue @@ -74,23 +74,22 @@ def raise_if_not_myprotocol(job, host, conn): class Log: - data = '' + """Collects what the threads of the queue write.""" + + def __init__(self): + self.data = '' + self.lock = threading.Lock() def write(self, data): - self.data += data + with self.lock: + self.data += data def flush(self): pass def read(self): - return self.data - - -class LogManager(BaseManager): - pass - - -LogManager.register('Log', Log) + with self.lock: + return self.data class QueueTest(unittest.TestCase): @@ -100,8 +99,8 @@ class QueueTest(unittest.TestCase): def createQueue(self, logdir=None, **kwargs): if self.queue: self.queue.destroy() - self.out = self.manager.Log() - self.err = self.manager.Log() + self.out = Log() + self.err = Log() self.queue = Queue(mode=self.mode, stdout=self.out, stderr=self.err, **kwargs) self.accm = self.queue.account_manager if logdir is not None: @@ -111,8 +110,6 @@ def setUp(self): self.tempdir = mkdtemp() self.queue = None self.logger = None - self.manager = LogManager() - self.manager.start() self.createQueue(verbose=-1, logdir=self.tempdir) def tearDown(self): @@ -121,7 +118,6 @@ def tearDown(self): self.queue.destroy() except: pass # queue already destroyed - self.manager.shutdown() def assertVerbosity(self, channel, expected): data = channel.read() @@ -137,10 +133,6 @@ def assertVerbosity(self, channel, expected): msg = repr(expected) + ' not in ' + repr(data) self.assertTrue(expected in data, msg) - # IMPORTANT: If you see fails of this test, it might be related to the - # Logmanager as it uses Process, not Thread. Mixing both is not recommended - # and false-positive fails have been seen with stuck old processes. So check - # `ps -ef -u` on your system for any old test processes and kill them. def testConstructor(self): self.assertEqual(1, self.queue.get_max_threads()) @@ -191,24 +183,10 @@ def testConstructor(self): self.assertVerbosity(self.out, stdout) self.assertVerbosity(self.err, stderr) - def testCreatePipe(self): - account = Account('user', 'test') - self.accm.add_account(account) - pipe = self.queue._create_pipe() - pipe.send(('acquire-account', None)) - response = pipe.recv() - expected = ( - account.__hash__(), - account.get_name(), - account.get_password(), - account.get_authorization_password(), - account.get_key(), - ) - self.assertEqual(response, expected) - pipe.send(('release-account', account.__hash__())) - response = pipe.recv() - self.assertEqual(response, 'ok') - pipe.close() + def testProcessModeWasRemoved(self): + with self.assertRaisesRegex(ValueError, 'removed in Exscript 3.0'): + Queue(mode='multiprocessing') + self.assertRaises(TypeError, Queue, mode='greenlets') def testSetMaxThreads(self): self.assertEqual(1, self.queue.get_max_threads()) @@ -433,18 +411,9 @@ def testQueueLogging(self): content = fp.read() -# Mode 'multiprocessing' is not recommended and officially not supported with -# threads in parallel, see https://github.com/knipknap/exscript/issues/230 -# -# class QueueTestMultiProcessing(QueueTest): -# mode = 'multiprocessing' - - def suite(): loader = unittest.TestLoader() - suite1 = loader.loadTestsFromTestCase(QueueTest) - # suite2 = loader.loadTestsFromTestCase(QueueTestMultiProcessing) - return unittest.TestSuite((suite1,)) + return loader.loadTestsFromTestCase(QueueTest) if __name__ == '__main__': diff --git a/tests/Exscript/util/implTest.py b/tests/Exscript/util/implTest.py index f8f5c93d..66d506f5 100644 --- a/tests/Exscript/util/implTest.py +++ b/tests/Exscript/util/implTest.py @@ -51,7 +51,7 @@ def locked_for_other_threads(resource): result = [] def try_lock(): - acquired = resource._sync_lock.acquire(block=False) + acquired = resource._sync_lock.acquire(False) if acquired: resource._sync_lock.release() result.append(not acquired) diff --git a/tests/Exscript/workqueue/JobTest.py b/tests/Exscript/workqueue/JobTest.py index a4eebfc0..57def95b 100644 --- a/tests/Exscript/workqueue/JobTest.py +++ b/tests/Exscript/workqueue/JobTest.py @@ -1,14 +1,17 @@ import unittest -from multiprocessing import Pipe, get_all_start_methods from pickle import dumps, loads -from Exscript.workqueue.job import Job, Process, Thread +from Exscript.workqueue.job import Job, Thread def do_nothing(job): pass +def fail(job): + raise ValueError('intentional error') + + class ThreadTest(unittest.TestCase): CORRELATE = Thread @@ -18,26 +21,21 @@ def testConstructor(self): def testRun(self): job = self.CORRELATE(1, do_nothing, 'myaction', None) - to_child, to_self = Pipe() - job.start(to_self) - response = to_child.recv() - while job.is_alive(): - pass + job.start() + job.join() + self.assertIsNone(job.exc_info) + + def testRunWithAnError(self): + job = self.CORRELATE(1, fail, 'myaction', None) + job.start() job.join() - self.assertEqual(response, '') + self.assertIs(job.exc_info[0], ValueError) + self.assertIn('intentional error', job.exc_info[2]) # the traceback as text def testStart(self): pass # See testRun() -@unittest.skipIf( - get_all_start_methods()[0] != 'fork', - 'Process jobs need the "fork" start method (the default only on Linux < 3.14)', -) -class ProcessTest(ThreadTest): - CORRELATE = Process - - class JobTest(unittest.TestCase): def testConstructor(self): job = Job(do_nothing, 'myaction', 1, 'foo') @@ -57,9 +55,8 @@ def testPickle(self): def suite(): loader = unittest.TestLoader() suite1 = loader.loadTestsFromTestCase(ThreadTest) - suite2 = loader.loadTestsFromTestCase(ProcessTest) - suite3 = loader.loadTestsFromTestCase(JobTest) - return unittest.TestSuite((suite1, suite2, suite3)) + suite2 = loader.loadTestsFromTestCase(JobTest) + return unittest.TestSuite((suite1, suite2)) if __name__ == '__main__': diff --git a/tests/Exscript/workqueue/MainLoopTest.py b/tests/Exscript/workqueue/MainLoopTest.py index 4e6d8481..3f02ddbb 100644 --- a/tests/Exscript/workqueue/MainLoopTest.py +++ b/tests/Exscript/workqueue/MainLoopTest.py @@ -1,7 +1,7 @@ import threading import unittest -from Exscript.workqueue.job import Process +from Exscript.workqueue.job import Thread from Exscript.workqueue.mainloop import MainLoop from Exscript.workqueue.pipeline import Pipeline @@ -13,7 +13,7 @@ def setUp(self): def testMainLoop(self): lock = threading.Lock() data = {'sum': 0, 'randsum': 0} - ml = MainLoop(Pipeline(), Process) + ml = MainLoop(Pipeline(), Thread) nop = lambda x: None for i in range(12345): From 343c037b4fecdcbef16503d8db6eacc552d0f165 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 10 Nov 2026 22:45:47 +0100 Subject: [PATCH 138/250] Test the queue under gevent in CI, and document gevent and asyncio Exscript runs under gevent if the application patches the standard library before its other imports, which a library cannot do for it. The platforms page shows how, and how to call Exscript from asyncio code with asyncio.to_thread(). --- .github/workflows/ci.yml | 12 +++ docs/platforms.rst | 24 ++++- pyproject.toml | 2 + tests/Exscript/GeventTest.py | 77 ++++++++++++++ uv.lock | 192 +++++++++++++++++++++++++++++++++++ 5 files changed, 305 insertions(+), 2 deletions(-) create mode 100644 tests/Exscript/GeventTest.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0adce82d..925dc01f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -116,6 +116,18 @@ jobs: - name: Build the documentation (warnings are errors, like on Read the Docs) run: uv run --frozen --python 3.14 --group docs sphinx-build -W --keep-going docs docs/_build/html + gevent: + name: Queue under gevent + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + fetch-depth: 0 + - uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3 + - name: Run Telnet and SSH sessions through a queue under gevent + run: uv run --frozen --python 3.14 --group gevent pytest -p no:cacheprovider tests/Exscript/GeventTest.py + os-guessing: name: OS-guess accuracy runs-on: ubuntu-latest diff --git a/docs/platforms.rst b/docs/platforms.rst index 9aea2113..170c8517 100644 --- a/docs/platforms.rst +++ b/docs/platforms.rst @@ -31,8 +31,8 @@ Windows - The default known_hosts file is ``%USERPROFILE%\.ssh\known_hosts`` (:doc:`security`). -Threads and processes ---------------------- +Threads, processes and asyncio +------------------------------ The queue runs each job in a thread; Exscript 3.0 removed its process mode. To use several CPU cores, run the free-threaded build (below), or @@ -40,6 +40,26 @@ start several processes yourself, for example with a pool of :mod:`concurrent.futures` or the prefork pool of Celery, and create the queue or the connections in the child processes. +Exscript also runs under `gevent `_, which +replaces threads with greenlets, so a process can hold many more +connections. The application patches the standard library itself, before +it imports anything else:: + + from gevent import monkey + + monkey.patch_all() + + from Exscript import Queue # and the rest + +Exscript has no gevent dependency or setting; CI runs Telnet and SSH +sessions through a queue under gevent. gevent does not support the +free-threaded build. + +From :mod:`asyncio` code, run Exscript in a thread, so that it does not +block the event loop:: + + result = await asyncio.to_thread(conn.execute, 'show version') + Free-threaded Python -------------------- diff --git a/pyproject.toml b/pyproject.toml index c4fe4e35..3e61f895 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -65,6 +65,8 @@ test = [ ] types = ["mypy>=2.4", "types-lxml>=2026.2", "types-paramiko>=5.1"] docs = ["sphinx>=9", "furo>=2025.12", "myst-parser>=5"] +# Not a dependency of Exscript: CI checks that applications can run it under gevent. +gevent = [{ include-group = "test" }, "gevent>=26"] dev = [{ include-group = "test" }, { include-group = "types" }] [tool.hatch.version] diff --git a/tests/Exscript/GeventTest.py b/tests/Exscript/GeventTest.py new file mode 100644 index 00000000..af0f5b26 --- /dev/null +++ b/tests/Exscript/GeventTest.py @@ -0,0 +1,77 @@ +"""Exscript under gevent: the application patches the standard library, and the queue works.""" + +import importlib.util +import os +import subprocess +import sys +import unittest + +SCRIPT = ''' +from gevent import monkey + +monkey.patch_all() + +import sys + +from Exscript import Account, Host, PrivateKey, Queue +from Exscript.emulators import VirtualDevice +from Exscript.servers import SSHd, Telnetd + +key = PrivateKey.from_file(sys.argv[1]) +servers, hosts = [], [] +for i in range(5): + for scheme, cls, kwargs in ('telnet', Telnetd, {}), ('ssh', SSHd, {'key': key}): + device = VirtualDevice('r%d' % i, login_type=VirtualDevice.LOGIN_TYPE_NONE, strict=False) + device.add_command('show version', 'test device ' + scheme) + server = cls('127.0.0.1', 0, device, **kwargs) + server.timeout = 0.05 + server.start() + servers.append(server) + host = Host('%s://127.0.0.1:%d' % (scheme, server.port)) + host.set_option('connect_rate_limit', 0) + host.set_option('verify_fingerprint', False) + host.set_option('driver', 'generic') + hosts.append(host) + +responses = [] + + +def run(job, host, conn): + conn.protocol_authenticate(Account('user', 'password', needs_lock=False)) + conn.expect_prompt() + conn.execute('show version') + responses.append(conn.response) + + +queue = Queue(verbose=0, max_threads=len(hosts)) +queue.run(hosts, run) +queue.shutdown() +for server in servers: + server.exit() + server.join() +print(sorted(r.strip().splitlines()[-1] for r in responses)) +''' + + +@unittest.skipUnless(importlib.util.find_spec('gevent'), 'gevent is not installed') +class GeventTest(unittest.TestCase): + def testQueue(self): + # The queue hangs if it waits on a lock that gevent does not patch. + keyfile = os.path.join(os.path.dirname(__file__), 'protocols', 'id_rsa') + result = subprocess.run( # noqa: S603 (this interpreter with a constant script) + [sys.executable, '-W', 'ignore', '-c', SCRIPT, keyfile], + capture_output=True, + text=True, + timeout=60, + ) + self.assertEqual(result.returncode, 0, result.stderr) + expected = ['test device ssh'] * 5 + ['test device telnet'] * 5 + self.assertEqual(result.stdout.strip(), repr(expected)) + + +def suite(): + return unittest.TestLoader().loadTestsFromTestCase(GeventTest) + + +if __name__ == '__main__': + unittest.TextTestRunner(verbosity=2).run(suite()) diff --git a/uv.lock b/uv.lock index 7d9b8b1a..d174c3b8 100644 --- a/uv.lock +++ b/uv.lock @@ -639,6 +639,14 @@ docs = [ { name = "sphinx", version = "9.1.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13'" }, { name = "sphinx", version = "9.2.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.13'" }, ] +gevent = [ + { name = "gevent" }, + { name = "hypothesis" }, + { name = "opentelemetry-instrumentation" }, + { name = "opentelemetry-sdk" }, + { name = "pytest" }, + { name = "pytest-cov" }, +] test = [ { name = "hypothesis" }, { name = "opentelemetry-instrumentation" }, @@ -679,6 +687,14 @@ docs = [ { name = "myst-parser", specifier = ">=5" }, { name = "sphinx", specifier = ">=9" }, ] +gevent = [ + { name = "gevent", specifier = ">=26" }, + { name = "hypothesis", specifier = ">=6.170.14" }, + { name = "opentelemetry-instrumentation", specifier = ">=0.48b0" }, + { name = "opentelemetry-sdk", specifier = ">=1.27" }, + { name = "pytest", specifier = ">=9.2" }, + { name = "pytest-cov", specifier = ">=7.1" }, +] test = [ { name = "hypothesis", specifier = ">=6.170.14" }, { name = "opentelemetry-instrumentation", specifier = ">=0.48b0" }, @@ -709,6 +725,123 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f4/b2/50e9b292b5cac13e9e81272c7171301abc753a60460d21505b606e15cf21/furo-2025.12.19-py3-none-any.whl", hash = "sha256:bb0ead5309f9500130665a26bee87693c41ce4dbdff864dbfb6b0dae4673d24f", size = 339262, upload-time = "2025-12-19T17:34:38.905Z" }, ] +[[package]] +name = "gevent" +version = "26.9.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cffi", marker = "platform_python_implementation == 'CPython' and sys_platform == 'win32'" }, + { name = "greenlet", marker = "platform_python_implementation == 'CPython'" }, + { name = "zope-event" }, + { name = "zope-interface" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/2b/ac/dd3137ae695aef399373088c84c66398f3eac597fba542f0a22280bc21d6/gevent-26.9.0.tar.gz", hash = "sha256:4dd4703d71737a456c1c9df5cd43a82934e5b10c87549caa02495f487d1ef0b1", size = 6718097, upload-time = "2026-09-16T18:05:35.008Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f1/90/2f09ad04b52ad8888fe6a0a4a543c5445b27c78ccbde8f3104ee3ac618f8/gevent-26.9.0-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:979caf5b96f5806cb5b66fd2c7972f1043cc4069d1ee8b2998c42cb0b39dc445", size = 3078823, upload-time = "2026-09-16T16:16:12.412Z" }, + { url = "https://files.pythonhosted.org/packages/c3/7f/1068c8eef85f04bb9d8490140f6adba47c0676d95e66a2d9549bdad0c22c/gevent-26.9.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:0b3f0ad9dc8e2ba585e0f6498c96b78ba61b1214f5b2e17081839c93b69a58c3", size = 1861416, upload-time = "2026-09-16T17:23:55.662Z" }, + { url = "https://files.pythonhosted.org/packages/0a/7a/c237d66fe48e0391d88f03448576ad127befc9d30ff0f9e3269272e15d1c/gevent-26.9.0-cp312-cp312-manylinux_2_28_ppc64le.whl", hash = "sha256:83c51ffa0ef9c960fe3b6bc0a9de8997cd04a9476ff5d4e682c0c62481ef3924", size = 1965629, upload-time = "2026-09-16T17:09:24.075Z" }, + { url = "https://files.pythonhosted.org/packages/8a/95/7bcd42a2aaceb7ad464f66fdd2be8df640c288713fd3b932f86f22e0fa86/gevent-26.9.0-cp312-cp312-manylinux_2_28_s390x.whl", hash = "sha256:ab1db9defde9ea9bd1825057fd90474148f74dcc57d104ddc62343092eaa256f", size = 1912220, upload-time = "2026-09-16T17:10:08.2Z" }, + { url = "https://files.pythonhosted.org/packages/05/89/c07717de442a898229a5e8ec6fbaf878e4d328868362c905fe14c5a72521/gevent-26.9.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:c59d95daacf71dfb763824b85a89b06ca4faa74b2e7df926714d439d5a47ee26", size = 2201889, upload-time = "2026-09-16T16:39:07.925Z" }, + { url = "https://files.pythonhosted.org/packages/df/23/fad2ba73045e4ee0dccf2e35a6fe19908309bd6176d1e5e3a18bb780e96b/gevent-26.9.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:f91b87ca2ac3af502f7ee806c266ba6f64e4d1591e2e29456ed7cc538e5473ec", size = 1875679, upload-time = "2026-09-16T17:24:45.124Z" }, + { url = "https://files.pythonhosted.org/packages/a2/73/a4414d7e95be1287b3dbe6310331c2658395bd4ada69a19f98c3aecba4c9/gevent-26.9.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:810cd040eda484e8ce73d649fa994a4fc247b427023db52d4daaa10e8fd2f4aa", size = 2230243, upload-time = "2026-09-16T16:47:52.283Z" }, + { url = "https://files.pythonhosted.org/packages/a1/6a/d5e9de5e2dbe5a58814d7a04ada307d7aca145c40484aa30894edda7cc7b/gevent-26.9.0-cp312-cp312-win_amd64.whl", hash = "sha256:44a0d58301a333608aad5fef0c19ca8122eb7753484416f000c1f00b4b407697", size = 1725939, upload-time = "2026-09-16T16:19:41.956Z" }, + { url = "https://files.pythonhosted.org/packages/fc/4b/525d4da671e7b6d21dceaca33fa65edc13917189b80e9b3a30318e6345bd/gevent-26.9.0-cp312-cp312-win_arm64.whl", hash = "sha256:f9ff7c692028c577937ad00bdd1183371a086f7d6908c7c1f18f1c51ccf8caac", size = 1596884, upload-time = "2026-09-16T16:20:44.976Z" }, + { url = "https://files.pythonhosted.org/packages/b1/ec/2fc93e431ca1f42f0a554e9a74c881dc0ea8c84ca0e708445069ca255cc1/gevent-26.9.0-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:1e2b9508076350799def5eb7ac57a9d7c14234da201372d9f7329f45074f833a", size = 3105194, upload-time = "2026-09-16T16:17:08.632Z" }, + { url = "https://files.pythonhosted.org/packages/c9/40/31dcfe97c1a10e262264f9e0aea4b363aa69a26826305c5bd6fb9f419e76/gevent-26.9.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:c8b3bf3865f11504941d11bcca1dbf53beee79405b0da7577b1db29f94bb2209", size = 1862784, upload-time = "2026-09-16T17:23:57.57Z" }, + { url = "https://files.pythonhosted.org/packages/3f/03/0729ac615271b09c4eae6a2d8d034a60152f9f3d9fe98e82d0fa73a27b05/gevent-26.9.0-cp313-cp313-manylinux_2_28_ppc64le.whl", hash = "sha256:cb52241e8c691818853361663134a72c4d5601a9fa46ff7f9cb749878855b26f", size = 1966037, upload-time = "2026-09-16T17:09:25.594Z" }, + { url = "https://files.pythonhosted.org/packages/79/bb/c2f13d43f057f4b7c45df4abb9737414d05a25a7f835b2e4428a19b97f39/gevent-26.9.0-cp313-cp313-manylinux_2_28_s390x.whl", hash = "sha256:405d73327feecab8cc9976f7bc2a0dbd1adaccf2e4b5e86e97e7b87879fa5cfd", size = 1915058, upload-time = "2026-09-16T17:10:09.709Z" }, + { url = "https://files.pythonhosted.org/packages/ec/98/f05061aa7a1072ce41521ad18eceb6d028086c3f2c6249b21de142ef0be9/gevent-26.9.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:231058bdb60dbf1074b2e74fbb77c0b0f1b045886bf7203b816692c3663726cc", size = 2202062, upload-time = "2026-09-16T16:39:09.203Z" }, + { url = "https://files.pythonhosted.org/packages/98/05/8822af537754c8e46305f4948ceb6f6bb39b351dfcdc1ed8aa6dad946b18/gevent-26.9.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:23f08013256a3e9b5928b65856116f9bdc775ee8246c0361bc916ea283c9c6fd", size = 1876618, upload-time = "2026-09-16T17:24:46.645Z" }, + { url = "https://files.pythonhosted.org/packages/eb/82/47e88bd691879ba26588faa8cb2eee96a5b1fd862d654ecef40acb85bdd8/gevent-26.9.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:c38da261295c20066b352007703a2acec91644ada03a0e4f1a9d0efee8cb5a5c", size = 2233094, upload-time = "2026-09-16T16:47:53.703Z" }, + { url = "https://files.pythonhosted.org/packages/c7/9d/0af37ec9ab225ce0aed7fd5c5d75d0c78822805d0e1672692e75d6be61b8/gevent-26.9.0-cp313-cp313-win_amd64.whl", hash = "sha256:5902ecdd81454615a3bf610897592058c4fe347c8e4ce4313dc31aeb29ba0ca7", size = 1723617, upload-time = "2026-09-16T16:19:52.862Z" }, + { url = "https://files.pythonhosted.org/packages/ef/69/409483e91b8b0fa0dabcbc9f098261c55aa7533632d8310c91e4cd5af0a1/gevent-26.9.0-cp313-cp313-win_arm64.whl", hash = "sha256:1c56654619fc284091f82900469993de50263a9f6c44724e0f084167e9cc8917", size = 1594616, upload-time = "2026-09-16T16:19:51.959Z" }, + { url = "https://files.pythonhosted.org/packages/84/d1/f4b7b8d9a5e20dc525f9b7df5c55105a068774d94c1d62b3cdb5b89bc1e9/gevent-26.9.0-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:86999e6ec77ae16411c734658c88fde8b5c4be0112dc442ac498925fc881ddb2", size = 3123886, upload-time = "2026-09-16T16:18:27.99Z" }, + { url = "https://files.pythonhosted.org/packages/e7/f9/36de2881af1a254010c347e5af7366c1c76d5c5d9a2fc0e21939d72717fd/gevent-26.9.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:415f963d9b8e9022156afb091f6399de1d598aca173622cf5e2d0472178d57b1", size = 1869483, upload-time = "2026-09-16T17:23:59.335Z" }, + { url = "https://files.pythonhosted.org/packages/82/06/4421f7a1d00f4e3dbbede3d439065088401eabe931cd6443dfd9845ac3db/gevent-26.9.0-cp314-cp314-manylinux_2_28_ppc64le.whl", hash = "sha256:0ec6525fa2d55b96fc538be48a53a875c4b804738b016078a6eb49a6a2adf2e6", size = 1971600, upload-time = "2026-09-16T17:09:27.457Z" }, + { url = "https://files.pythonhosted.org/packages/5b/31/c4e8677cfdd4863ebb04b664aca5933156ca6986f0ad09ee4ca6659a5c03/gevent-26.9.0-cp314-cp314-manylinux_2_28_s390x.whl", hash = "sha256:afb17dfcb8e33ba4c84cf50a08974925c50a9d01306f199712897cfb00775d56", size = 1919004, upload-time = "2026-09-16T17:10:11.326Z" }, + { url = "https://files.pythonhosted.org/packages/fc/7a/17e39476d7418b2d4361d5283ec913f82fd1b596de0d8b756483475025ab/gevent-26.9.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:d05115c494183d032d5dd3ee4f1517f4caa145f38008cee46405c5c2c8a4214b", size = 2210366, upload-time = "2026-09-16T16:39:10.513Z" }, + { url = "https://files.pythonhosted.org/packages/89/9d/5b3242ab0a15ccbb00b09a50e69ee2fe3c32220c4839dd86e083599804c2/gevent-26.9.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:12e909b93dcda8d3a40eb8130de605a70eca95a58f4ef74133d07c11495f8c89", size = 1884587, upload-time = "2026-09-16T17:24:47.933Z" }, + { url = "https://files.pythonhosted.org/packages/59/f8/238c505a3d43eae760482190fbb92c2ed661fe8c9077ac3f9df4f1fb2ab7/gevent-26.9.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f5e894f892347e242742ab24c881be271c2ea4be149bdb80307bab7a8f506ccb", size = 2239409, upload-time = "2026-09-16T16:47:55.043Z" }, + { url = "https://files.pythonhosted.org/packages/5c/ad/39598321091044ed30bce8488dcfb3eca390e192a7f5c4c19ab2a4d498cc/gevent-26.9.0-cp314-cp314-win_amd64.whl", hash = "sha256:9eac1550fce3e356dee3448c2b95080d25e3affd560e22936fffc79d4d6c3a38", size = 1748028, upload-time = "2026-09-16T16:25:10.438Z" }, + { url = "https://files.pythonhosted.org/packages/32/b5/4cded556e3f06153d299881a1c3d104cba695161c9d283c08e94c80ffb28/gevent-26.9.0-cp314-cp314-win_arm64.whl", hash = "sha256:3427358b8dcde8abcfab45d649aeedab9eb5d31916886e277405f95660e12751", size = 1625381, upload-time = "2026-09-16T16:21:12.752Z" }, + { url = "https://files.pythonhosted.org/packages/a3/68/2a6b8bed9302e6a3034c1dc1eabe8a0a2cfb5138f5f18bacba4948efe972/gevent-26.9.0-cp315-cp315-macosx_11_0_universal2.whl", hash = "sha256:8f70c12e1ec091ed326ee8096245a12257c7c2f95b043ed953f934c63eaefd7e", size = 3116824, upload-time = "2026-09-16T16:16:58.43Z" }, + { url = "https://files.pythonhosted.org/packages/dd/f7/15a4ba572147462f544335baec518c376e357e0b7506857c0897e8c60cd2/gevent-26.9.0-cp315-cp315-manylinux_2_28_aarch64.whl", hash = "sha256:32c8236cb4b2911cee7d5caaa8fcd8ab2267354d46fc8223a880e3466859d0bf", size = 1872409, upload-time = "2026-09-16T17:24:01.329Z" }, + { url = "https://files.pythonhosted.org/packages/cd/3b/41d14598d581fa8588f45577deb344edb99cd4a33c03fb905bc1309e274d/gevent-26.9.0-cp315-cp315-manylinux_2_28_ppc64le.whl", hash = "sha256:3b6404d18df517663df90889568de931ae43aae765bae542edb9ada73a9595db", size = 1975098, upload-time = "2026-09-16T17:09:29.223Z" }, + { url = "https://files.pythonhosted.org/packages/37/73/2380f29c84f685a6a9189381fdeffee8effed675f26df324e2eccbcbbecc/gevent-26.9.0-cp315-cp315-manylinux_2_28_s390x.whl", hash = "sha256:ea5f8f84232f1900a1a56ad6f7ba6804c49eeb8efdf861a6bae00bcf226568f5", size = 1921721, upload-time = "2026-09-16T17:10:13.109Z" }, + { url = "https://files.pythonhosted.org/packages/f3/07/31c69eba6260c5f2d2d9f87c4484eec8662b30261a907e78d705a114362a/gevent-26.9.0-cp315-cp315-manylinux_2_28_x86_64.whl", hash = "sha256:e9c8cdf9ff3eac29abb5ae55da16dac02cc464fc0e1e13818fca0437e8cfee0a", size = 2204962, upload-time = "2026-09-16T16:39:12.142Z" }, + { url = "https://files.pythonhosted.org/packages/54/95/d5bc8e4c30822b7606c7893d3ae2bc41cf666bc8cf94ba29977ee622a3c0/gevent-26.9.0-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:460c6db10c8d9475efb9a24d84c4a0e47bf628dce569efa0821217d83c68e584", size = 1886583, upload-time = "2026-09-16T17:24:49.383Z" }, + { url = "https://files.pythonhosted.org/packages/67/0d/87cdbe340d2f0caf31d1352403a83093459f4fefe6e9c70495befde96268/gevent-26.9.0-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:4a698fa2f5cf096bd6c1f59fd38a0d420e8b3a815b01be197eb9529cdd57d06b", size = 2234802, upload-time = "2026-09-16T16:47:56.508Z" }, + { url = "https://files.pythonhosted.org/packages/94/1a/837a278fe6c47b809322d2b99fcc4be8e86c14c3e1b13d1e8345d7bf1557/gevent-26.9.0-cp315-cp315-win_amd64.whl", hash = "sha256:e7e9247b449ee69f275bc4d44ceebaa0b71772d02bb3c52c146b2f613c4ad8d7", size = 1747063, upload-time = "2026-09-16T16:21:49.858Z" }, + { url = "https://files.pythonhosted.org/packages/e7/fb/0fbe629e58eab460c9ddea4f391b61f65708d026c50eb7be2f7c9052efb4/gevent-26.9.0-cp315-cp315-win_arm64.whl", hash = "sha256:5b089f158cdecddf5ac8face23e1cf7318a704625a32998c37118818efc97f16", size = 1623946, upload-time = "2026-09-16T16:21:33.849Z" }, +] + +[[package]] +name = "greenlet" +version = "3.5.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/3e/6e/0091f175ccd02b02bc8811bbcbcc6ac2e980be116e3b2f7a736ca322bf84/greenlet-3.5.6.tar.gz", hash = "sha256:8e67c43bdfc88d5fee6db0d3e40175b362fc95fb85f0412d233b9b203c53a575", size = 207653, upload-time = "2026-09-14T15:42:51.806Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/72/18/3fc6d951466ae9a2a688edcddde3b2e388da0a8244e0caf7117bbeb0eb95/greenlet-3.5.6-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:a5876d0a60355af98d535c47f6cd6eb0f8a432396dab26845d380b92f8412422", size = 295668, upload-time = "2026-09-14T14:22:33.241Z" }, + { url = "https://files.pythonhosted.org/packages/27/89/366d2af5061eeefa5012f510d95a99c8620dcc457609838db4d538820318/greenlet-3.5.6-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e85880b538e59a59f55117b81f208a6660ad5ac328aad9305f812d9b8bc67a0f", size = 611700, upload-time = "2026-09-14T15:12:01.962Z" }, + { url = "https://files.pythonhosted.org/packages/54/1c/07f133f865fd58ae593dd2bbec3144acaee9b04ffe2eb48c6e121747ceef/greenlet-3.5.6-cp312-cp312-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f0ba7c2a329d650628f4c8572fd1db29f0a59dd70a3e3e0710dcf18a35cce9d8", size = 624223, upload-time = "2026-09-14T15:20:42.459Z" }, + { url = "https://files.pythonhosted.org/packages/a7/f2/844dc823ff2752ad049caa6b59d57e4572f9c445934b02d3518f4c67197c/greenlet-3.5.6-cp312-cp312-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ee7d9da3bf493909cf811a3f038840cb34fab5ae2956b8a263919f6e289ab188", size = 629529, upload-time = "2026-09-14T15:25:06.354Z" }, + { url = "https://files.pythonhosted.org/packages/66/6a/1594f3869c57c149abdb380492529e04d4c0229b5e4d79572c5bd0aaa673/greenlet-3.5.6-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:975736b002ed080d124cf81a79cb7e05cb26d6b3f5c7a7b651c0fcce70353aa1", size = 621404, upload-time = "2026-09-14T14:35:59.027Z" }, + { url = "https://files.pythonhosted.org/packages/c0/42/b1f8dbc89a53b9e77859fc1ad1627d106fc361daa3ea4bdf43a91ebb4338/greenlet-3.5.6-cp312-cp312-manylinux_2_39_riscv64.whl", hash = "sha256:71890d5247020c25c21a6b65202782bfc281d4e6e244842419d30e3492bb6dcc", size = 432385, upload-time = "2026-09-14T15:28:37.369Z" }, + { url = "https://files.pythonhosted.org/packages/a2/f5/33e5c9e48178b9259fd000f8f45caa4a65036f65d3d0c06a602f570f025d/greenlet-3.5.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:0616b8f878098c5681fd8f0dc92d887551717402342a70f0abcbfea5f5ad8a44", size = 1584998, upload-time = "2026-09-14T15:10:06.653Z" }, + { url = "https://files.pythonhosted.org/packages/ef/31/9b4e140bc24d0ad7927ebd651f5608b0acc2334d061748c3b6ad19085cfa/greenlet-3.5.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:3dbb4596a6a4e5d47121a33ff20533a81e60f302d9e67b69909a8bc21a43f0a7", size = 1647568, upload-time = "2026-09-14T14:35:49.787Z" }, + { url = "https://files.pythonhosted.org/packages/c3/71/d79f1791f824f8ff15c2978746640467ae932a2365e0201069f7f272395f/greenlet-3.5.6-cp312-cp312-win_amd64.whl", hash = "sha256:7ac4abb3877c43af320392c664774eef6fa2cc063c79a55fc02d844a3cbe7395", size = 324203, upload-time = "2026-09-14T14:22:54.504Z" }, + { url = "https://files.pythonhosted.org/packages/63/af/42aca4d56e8cb321912203069d8d34734cb288222f10ad2ae102718cc577/greenlet-3.5.6-cp312-cp312-win_arm64.whl", hash = "sha256:301102a49120b095e72a7838792b41233975fc1c155daec6d98f81c00c9280e0", size = 308310, upload-time = "2026-09-14T14:24:03.008Z" }, + { url = "https://files.pythonhosted.org/packages/f1/a1/e720a38852366c589e1a46cf570b886507ad2cf591050c203365638baab0/greenlet-3.5.6-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:f96f0e30b5a95c7631b12bfe214cbc90ec8fe8cfa36920596c10514a65743519", size = 294627, upload-time = "2026-09-14T14:24:40.102Z" }, + { url = "https://files.pythonhosted.org/packages/eb/c3/58187858df41354a11e6a55b421e7af9059798abdab3a384cc51b8567c38/greenlet-3.5.6-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c75116c9de79949de23006e2d9b35ee82874c594fcf5c0311b439acaa14b8441", size = 614356, upload-time = "2026-09-14T15:12:03.399Z" }, + { url = "https://files.pythonhosted.org/packages/ce/b9/3a7e67d5f05c9760b1ad411fa52264bd69cc08e22a2ebfb4018b90628ced/greenlet-3.5.6-cp313-cp313-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cad5782f93f7f738b62c6527b6f32a60694d924029f299a8b524758cfa53d815", size = 626756, upload-time = "2026-09-14T15:20:44.269Z" }, + { url = "https://files.pythonhosted.org/packages/c6/7c/40400455f5b5a65bb83e94fde66d1be9e5ec518638113f8083ace746c309/greenlet-3.5.6-cp313-cp313-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a93ee7c6e8fd0f8a83525a51bd777be57ee17787e91d805bd8d6faf9dcada18e", size = 632632, upload-time = "2026-09-14T15:25:07.813Z" }, + { url = "https://files.pythonhosted.org/packages/85/cb/ab0c123c514ed4e94c0dc9ee2e86362633e6b998cfc05de7fc9ac2eb9690/greenlet-3.5.6-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f98e8215e172f567ce80eeaed9107fb4d32b6c44f26983d9b8334658136a205a", size = 623779, upload-time = "2026-09-14T14:36:01.104Z" }, + { url = "https://files.pythonhosted.org/packages/f9/67/1f35cff30a6c51c3f23b63d4afcc7313ab4f97490ba3676fa78178984b27/greenlet-3.5.6-cp313-cp313-manylinux_2_39_riscv64.whl", hash = "sha256:7f731ebac68ea06d628658295cb2d217b10186329fcf9a3b6a149045059bf92e", size = 434933, upload-time = "2026-09-14T15:28:38.858Z" }, + { url = "https://files.pythonhosted.org/packages/a5/26/fda8a5a06e7073333ccb038133c5893b9e0c4fe29d5992a17e83c241bc6e/greenlet-3.5.6-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:df19e2d0b1620039af5102563fbd96e8938c7f5c3f5828528d641d9fc585525e", size = 1584930, upload-time = "2026-09-14T15:10:08.234Z" }, + { url = "https://files.pythonhosted.org/packages/2f/37/50f8813163148d6234e08b23dcad6a9e37f01d148c8ec976e4c44ea2d918/greenlet-3.5.6-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:06c0e933290fba8ffe53ead4ae1b8044b0e9754b75cebf381aa2bc3e50d82fac", size = 1647590, upload-time = "2026-09-14T14:35:51.173Z" }, + { url = "https://files.pythonhosted.org/packages/86/da/b7669b09586365654083a62bd0724cf06cb74bd5085a15cdd161271f992f/greenlet-3.5.6-cp313-cp313-win_amd64.whl", hash = "sha256:5b602b4201b965a8354d74e232364a66ff243dd142e350d035f46169bb36e13d", size = 324086, upload-time = "2026-09-14T14:23:48.428Z" }, + { url = "https://files.pythonhosted.org/packages/e5/5d/c9663cfe84a2a9e0aa96f066f5b0594c227ea4c647511e087e2e11d4ac0a/greenlet-3.5.6-cp313-cp313-win_arm64.whl", hash = "sha256:876077e7ebb8c84ed068e2b23d4c62ebb010d60df84b9591af1be2f39010ffb2", size = 308211, upload-time = "2026-09-14T14:28:01.634Z" }, + { url = "https://files.pythonhosted.org/packages/66/c0/d254544ae2b8bdd311aef000fafc02828c2771b17d994b3075620ea7cc6e/greenlet-3.5.6-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:8cddea1b8339451c2fb3388e138347b6126744f33b611bdb55b7357361cfef46", size = 295221, upload-time = "2026-09-14T14:25:11.583Z" }, + { url = "https://files.pythonhosted.org/packages/18/18/eb54be16b9cc3971e09ca5b73334e1b8c804a4630d9addaaf218a4fe300f/greenlet-3.5.6-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c59acfa8eb73a1e0d484392dc002bdf001fd4ce73394e0132df3d1ab6093d7cb", size = 660992, upload-time = "2026-09-14T15:12:04.876Z" }, + { url = "https://files.pythonhosted.org/packages/8f/b4/e193efe65671dcf294bc51fcc59efb52d154adf8612c4ea016da0d2c486c/greenlet-3.5.6-cp314-cp314-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a3b4a01c6da07ef9f80d4fe8933b994bc99747bcea3eab0330a9c34d3c12655b", size = 673428, upload-time = "2026-09-14T15:20:45.756Z" }, + { url = "https://files.pythonhosted.org/packages/fd/21/631bb45fafde1dca782152377c0676d182ec924820064047f533a3627b28/greenlet-3.5.6-cp314-cp314-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:dd0b83bed3405b586a3133629f1d1a5bc7bfd64822a3b7ab342bdc68e6dbc61b", size = 677688, upload-time = "2026-09-14T15:25:09.279Z" }, + { url = "https://files.pythonhosted.org/packages/45/ac/28fa7a9e50f2859466214c4ac584d776db52c1604ad4dd158960a5af2a1f/greenlet-3.5.6-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9a09d59bef1db94f384b5bcc2d523694d338f3df6b757aeeaf7baca5d0c0be88", size = 670773, upload-time = "2026-09-14T14:36:02.577Z" }, + { url = "https://files.pythonhosted.org/packages/40/30/2b0a73e68e1e18e30b601d0d183cfdfc2beca4de5a6843c630f0fc9fb90c/greenlet-3.5.6-cp314-cp314-manylinux_2_39_riscv64.whl", hash = "sha256:fdacf26402389bdd89857ad3c045a26fe8f3314f9a8b28226f82f88463a65b77", size = 480475, upload-time = "2026-09-14T15:28:40.741Z" }, + { url = "https://files.pythonhosted.org/packages/c3/cd/fb7d6cdd86ff3427c1494854f0e35437eba05142be91f530f6da75e09e19/greenlet-3.5.6-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:8b7c73d1cef3d9ae963e9ff03f6222df43efbb9054ffd2f1969c935b7fc84c02", size = 1631900, upload-time = "2026-09-14T15:10:09.745Z" }, + { url = "https://files.pythonhosted.org/packages/f6/40/143bdbb20a516628cb15074ae52ed17d850b450292609c7a6fccac6dbece/greenlet-3.5.6-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:8b27df301f56e3b3d2298095c8f7d6b68f2521f6b1693e901fa039bdbae34424", size = 1693740, upload-time = "2026-09-14T14:35:52.959Z" }, + { url = "https://files.pythonhosted.org/packages/c9/9e/019642432e6ae283301df1361227d47610709d2dc69a38f95edef266d713/greenlet-3.5.6-cp314-cp314-win_amd64.whl", hash = "sha256:f8f0bd690e1a41294ac87905e8121c81a3761ec2583c768f13467428606c8c7a", size = 327473, upload-time = "2026-09-14T14:28:12.948Z" }, + { url = "https://files.pythonhosted.org/packages/e9/7f/8aafc7bf70c948786dba7221d0dc0838e5329bebc6d434ef2208b4f0e760/greenlet-3.5.6-cp314-cp314-win_arm64.whl", hash = "sha256:8cda13494d86a4f12429641117cb6ac4bbbc9c30a33f711f7d3a2e5fbe4b0b7e", size = 311095, upload-time = "2026-09-14T14:28:00.7Z" }, + { url = "https://files.pythonhosted.org/packages/14/7e/7a205688a5b3074933b18a906608d46d106e9a79d776bdab5a4abf4b4feb/greenlet-3.5.6-cp314-cp314t-macosx_11_0_universal2.whl", hash = "sha256:97c5a53e8c1754df58e73f047a99e287d4da1bdfe64b0072fb25c87000897951", size = 305352, upload-time = "2026-09-14T14:21:31.962Z" }, + { url = "https://files.pythonhosted.org/packages/78/cb/9c4a57a9d9dd0256e20b8f7f4f06554c2c92badebf0ab73ce344321b78b9/greenlet-3.5.6-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fea4427d1ffdb3b523d7daa6712038428a4c16c450b9777bdd1221cfee0eab49", size = 672671, upload-time = "2026-09-14T15:12:06.347Z" }, + { url = "https://files.pythonhosted.org/packages/97/52/c6729681ebbd298f4decd28746815acc8a0b0a0fde21d2df33776fd4d042/greenlet-3.5.6-cp314-cp314t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:73a29b5ba642e35433166a03a3e02935e7238c4b3467fbd77523b99edea23e5b", size = 679489, upload-time = "2026-09-14T15:20:47.291Z" }, + { url = "https://files.pythonhosted.org/packages/71/76/3c11c21e0716b1f1dc7c1a4b3d690abb1d3b448c69a9d32049fecb64010a/greenlet-3.5.6-cp314-cp314t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:61a61b4a95a4f97922c3a6f5606d3e360851584bd47e500a5161373c53810e3d", size = 681303, upload-time = "2026-09-14T15:25:11.088Z" }, + { url = "https://files.pythonhosted.org/packages/58/c5/2b6c721ba8b8963da42d5a0f57f25b8aaeb1fe9bdd156875e57f3be648a2/greenlet-3.5.6-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:460e70b033aba8ed47e2ac9b5d0d2157b05a34fbfa30a241400aef4118902cdc", size = 676608, upload-time = "2026-09-14T14:36:03.959Z" }, + { url = "https://files.pythonhosted.org/packages/3f/26/3ae402202452cd5941bbbd483e5a74297e2397e7aa3182c2a5e3ab7d5666/greenlet-3.5.6-cp314-cp314t-manylinux_2_39_riscv64.whl", hash = "sha256:fe3170a69fe039b18ad18171e66faa9a75f6fe9d78f968fd9b54e09fbd714d81", size = 510112, upload-time = "2026-09-14T15:28:42.112Z" }, + { url = "https://files.pythonhosted.org/packages/b2/04/0d018e0d05bcdde19a0fcb907834155f1fc853a9bedd3f3f5e6acadcae19/greenlet-3.5.6-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:ca80a49b53ed1d22f7282da7255f7bb2fd1935fd0f623d8613fda38745f18961", size = 1641479, upload-time = "2026-09-14T15:10:11.216Z" }, + { url = "https://files.pythonhosted.org/packages/59/bb/f02ef9073919158f6403fe3701d4ed4403d646720e7201dfc6e9d264bac3/greenlet-3.5.6-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:916f92f2a8db10508f739d0b5e00b83defe5d1115a997c54532a6d7cf8c95404", size = 1698758, upload-time = "2026-09-14T14:35:54.336Z" }, + { url = "https://files.pythonhosted.org/packages/08/a5/1f48fe647473a2dcccfd1839b2ff2c78eb57009be776b4da071e901c9bff/greenlet-3.5.6-cp314-cp314t-win_amd64.whl", hash = "sha256:886bcf1870af74c32bc310fd00a6b803445e17e51b7d5a107c7b35c0f362cc16", size = 331574, upload-time = "2026-09-14T14:27:18.451Z" }, + { url = "https://files.pythonhosted.org/packages/cd/72/3882855a75838faeb54a58aeef4fd77d20b2a86d4bad570c70d41b565dcf/greenlet-3.5.6-cp315-cp315-macosx_11_0_universal2.whl", hash = "sha256:3ac3494c381dab876cad7d0b22f3a722f3e0c8deb3a65b9e7f35ad7f58b8fcb3", size = 295926, upload-time = "2026-09-14T14:27:21.16Z" }, + { url = "https://files.pythonhosted.org/packages/10/1f/be4d957d8a9b90bcbe8db206548a42134d96222d43e5ed3fc4708fb6e24b/greenlet-3.5.6-cp315-cp315-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:602024dae6d77e161f4b89491b62ca1d4f19949d79d47b2db057e476d21179d6", size = 666910, upload-time = "2026-09-14T15:12:07.901Z" }, + { url = "https://files.pythonhosted.org/packages/a1/af/60d62571a7d6de961e4ce7625d6c2faf359345659fc782d2cdf517c34577/greenlet-3.5.6-cp315-cp315-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f8e63209c3e1e828ee6a457529b4a6d8b05d050fe0ae03a7ae49e967c5d312e0", size = 677415, upload-time = "2026-09-14T15:20:48.817Z" }, + { url = "https://files.pythonhosted.org/packages/f5/41/b3114c97c10e796010f00a30f51c81470072bca4b53e396ccca87484fcf7/greenlet-3.5.6-cp315-cp315-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9133d68624b1f2e89ec2f554d56aea8a5b0d7168cd9320200ba58d4d794845a4", size = 681337, upload-time = "2026-09-14T15:25:12.812Z" }, + { url = "https://files.pythonhosted.org/packages/fb/16/ac9e547b611539aaed1870eb1d6ddc57abdd5924b3a99bb9b5f0b44176b8/greenlet-3.5.6-cp315-cp315-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ccadce0130fd813ec86ebfe969a6c58b42acc1d0fe55a47525375b740e07b605", size = 675927, upload-time = "2026-09-14T14:36:05.34Z" }, + { url = "https://files.pythonhosted.org/packages/48/1b/d41861c2fa00968e39e467a495ca8db9ce9b6310a5d9b57561b3d0dc48fa/greenlet-3.5.6-cp315-cp315-manylinux_2_39_riscv64.whl", hash = "sha256:5adcbbfe78bdc242c71740a02e0991cc1b2f34d33c8bb15ca45eee8fd1140942", size = 487339, upload-time = "2026-09-14T15:28:43.497Z" }, + { url = "https://files.pythonhosted.org/packages/c4/b1/b7ba08d6431121741f1d30be0d5d292e76873325179a63586cd9217b62f6/greenlet-3.5.6-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:9297fb9c39b9a2c039dbcd306c410bd6906b95244dec3bba4318d36c718c164c", size = 1637236, upload-time = "2026-09-14T15:10:12.442Z" }, + { url = "https://files.pythonhosted.org/packages/af/c5/3b1cbc68f0c082022fc8717f7fe4b8b13b8d583c52352be37f4e9f55bcd2/greenlet-3.5.6-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:b374e79ffa7511afc11773aef40a4ccea6191fba1c856ea2f9c56738dca69d7a", size = 1698526, upload-time = "2026-09-14T14:35:56.039Z" }, + { url = "https://files.pythonhosted.org/packages/de/56/12941ed2711400451c89d544e10f831800a2770f19dd55eac8f0f7f2003b/greenlet-3.5.6-cp315-cp315-win_amd64.whl", hash = "sha256:7969bffa322c097bd46ae595ada6a931cefda613f18ba64587e9cff4cb320756", size = 327739, upload-time = "2026-09-14T14:23:55.768Z" }, + { url = "https://files.pythonhosted.org/packages/c5/3b/576b9ed5ac929252e340cf60b4bcb6a8515350dc20797064b1922dc4ea75/greenlet-3.5.6-cp315-cp315-win_arm64.whl", hash = "sha256:8dba0129b93e7091dfefaf4cf7000172741bff7f47bf6326fcf17f32fbb54d6b", size = 311715, upload-time = "2026-09-14T14:28:25.154Z" }, + { url = "https://files.pythonhosted.org/packages/16/c2/86cfc5555a98e12b86966ddbd24fd39af32f71f2f785c6595b7feb2db156/greenlet-3.5.6-cp315-cp315t-macosx_11_0_universal2.whl", hash = "sha256:de3de000d459402cda015068fd135aa50c0bf6f2477a80d4da1e646f123b4e78", size = 306244, upload-time = "2026-09-14T14:27:57.565Z" }, + { url = "https://files.pythonhosted.org/packages/14/6d/83ffc9d05a75a80ab3a7595dbb1d9604e5d4fc2996d73a8ae2dbd1284900/greenlet-3.5.6-cp315-cp315t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:45663c01a4de48b9a64a2ee1509d92d1dfd3afb02b2ccfc9333029d11aef996a", size = 676578, upload-time = "2026-09-14T15:12:09.468Z" }, + { url = "https://files.pythonhosted.org/packages/5d/d6/c2cf684810e5caded075970aaadea654ecb58b8382b9aecf1d231b936894/greenlet-3.5.6-cp315-cp315t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:3deccbb57a481e3a408fe61cdfd5c13e0678fc0a30fdd09597917ca87b4be877", size = 684203, upload-time = "2026-09-14T15:20:50.261Z" }, + { url = "https://files.pythonhosted.org/packages/f2/d1/039c353d5593a97a89699e989324c9bc86af499e6c6152fe0180f5742204/greenlet-3.5.6-cp315-cp315t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:63aff70fe5aac59c72215f42ec39fcb59ff46774fa966e717f8ecb6ee2273577", size = 686023, upload-time = "2026-09-14T15:25:14.528Z" }, + { url = "https://files.pythonhosted.org/packages/62/19/00e1bee5d2af890dc8f400b54d0b0f9b489965f92bc12b407ff72cc6f469/greenlet-3.5.6-cp315-cp315t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:311018b46472fb26ee85870847fb89eb64cc8aaddb617400789d87076f7cfeec", size = 681253, upload-time = "2026-09-14T14:36:06.742Z" }, + { url = "https://files.pythonhosted.org/packages/8a/62/97ceb8e0b2ea96046cdf8e95b042715020ebb12d83ea0690db80a8f03d23/greenlet-3.5.6-cp315-cp315t-manylinux_2_39_riscv64.whl", hash = "sha256:520648db8fb92eef7b3e6013f5a6f901cdf0d6685f639c2f7a245879f865bef7", size = 517058, upload-time = "2026-09-14T15:28:44.924Z" }, + { url = "https://files.pythonhosted.org/packages/89/58/c9275fd0ca195d1d3402931bcce8cfcc74726ff76efb1883d229e6e1a3d7/greenlet-3.5.6-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:7f924a5a9d5890649566f2f6682e0d8ad8ca23028bacffbbac36dbd7fd680176", size = 1645988, upload-time = "2026-09-14T15:10:13.758Z" }, + { url = "https://files.pythonhosted.org/packages/e0/36/b35747582fa4f1a5453f8f3002405dbac788e450cec7674dc2d204b6ccb5/greenlet-3.5.6-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:de9923832f2d8c1a5ecd8d7260465a6ca5a86888a0d129e3bd5cf0406d2fc5bf", size = 1702383, upload-time = "2026-09-14T14:35:58.143Z" }, + { url = "https://files.pythonhosted.org/packages/ed/69/6ec22ac9351e474d2a134d0ff9400dc80362d1c20f0721088ffffdfc205b/greenlet-3.5.6-cp315-cp315t-win_amd64.whl", hash = "sha256:2ab5f42ac6c238eb71770715e6e909ad9a1a92b6c681ccb64cd5a0f07edb953f", size = 331845, upload-time = "2026-09-14T14:27:41.723Z" }, + { url = "https://files.pythonhosted.org/packages/30/cf/697c051fd534e223461fb8b523890e21a24eeca229cd50624cff6f02fabd/greenlet-3.5.6-cp315-cp315t-win_arm64.whl", hash = "sha256:f9fe868463ec7e1363733af77e38a5fda3e9b63940337048c945d69e0c80ff24", size = 315070, upload-time = "2026-09-14T14:22:21.476Z" }, +] + [[package]] name = "hypothesis" version = "6.170.14" @@ -1870,3 +2003,62 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/5f/ea/56eddba6249fae5361b53eb62e2e25ad86cabec4632d943d356aba8dea1b/wrapt-2.5.2-cp315-cp315t-win_arm64.whl", hash = "sha256:fa642d3781c061579cb138773add4a55b16f99de9ebc41603e89a756cf9e62c6", size = 106397, upload-time = "2026-10-29T23:06:20.826Z" }, { url = "https://files.pythonhosted.org/packages/77/8b/1ea57b936b70f98e8e7620a352578096d778dbee95b262f73c066d210ffb/wrapt-2.5.2-py3-none-any.whl", hash = "sha256:61eb54e9540ea522c2c4752a57549a83d3c67e6704ed708403d28e067aaf90f0", size = 82025, upload-time = "2026-10-29T23:06:38.56Z" }, ] + +[[package]] +name = "zope-event" +version = "6.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/93/41/faa10af34d48d9cd6fa0249a1162943ad84a9590bd1a06939981e6640416/zope_event-6.2.tar.gz", hash = "sha256:b97d5d6327067ee6b9dfcbdf606ade9ade70991e19c162e808ea39e5fcf0f8d3", size = 18958, upload-time = "2026-04-28T06:24:10.578Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9e/33/848922889e946d4befc415c219fe516af75c49555d8e736e183bfd30db42/zope_event-6.2-py3-none-any.whl", hash = "sha256:5e755153ac4faf64c10a4b6dd3307680166a3edf65b38df22df592610f8fa874", size = 6525, upload-time = "2026-04-28T06:24:09.176Z" }, +] + +[[package]] +name = "zope-interface" +version = "8.7" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/de/ff/a1f0021a26033da0df223fd05a7455d6d2881b67daf2c6dc897b4fe0a427/zope_interface-8.7.tar.gz", hash = "sha256:0b47b62e8d0d99b24bcdd32f4f2120425e5019c3bee2ad69a0e1d75737487a96", size = 266343, upload-time = "2026-10-15T07:25:14.851Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0e/6f/4a4c37a69f30761b36ba8a3b18789c52e9dd166ceaec4c3f49a862947e74/zope_interface-8.7-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:f70a3af6efb813b8d406a449a8afc800ef8e9e32a62d6d52e37e8cb10674b70f", size = 219048, upload-time = "2026-10-15T07:23:57.986Z" }, + { url = "https://files.pythonhosted.org/packages/cc/40/8fe168cff93670859815e78c6fc4c2e47f11b8e8277cf26a69363dcd5fdd/zope_interface-8.7-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:85c30b18b8fd75ccd1b8ad202e9130ca6f8997a574ee2a7d1619e4138d3acb0a", size = 219127, upload-time = "2026-10-15T07:23:59.63Z" }, + { url = "https://files.pythonhosted.org/packages/54/80/f1ddbfce94864624727c1c34e863c6108b35d9b7cc8407a0b961a99e4f26/zope_interface-8.7-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:a52c56e7a53d884506b785248191cc50f1c69161aec93f7e6e79feddb1d06b7a", size = 278695, upload-time = "2026-10-15T07:24:01.849Z" }, + { url = "https://files.pythonhosted.org/packages/54/af/0eddc2dd0fcfa3da3a6256c4f58278729076c77296b6f00567b03718026d/zope_interface-8.7-cp312-cp312-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:90aef6e0a9924af18f60528895f2fc50cb634191939d65b10a96d9ced05030b5", size = 284199, upload-time = "2026-10-15T07:24:03.745Z" }, + { url = "https://files.pythonhosted.org/packages/a4/0f/a25f7e0866e65db2a756ee7e444568796ddbf0ffb97d950a268835324228/zope_interface-8.7-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:383c04293dbcfee8ae8d24f85592291207d5bb6a703af437343e44ddb94fb68c", size = 283694, upload-time = "2026-10-15T07:24:05.334Z" }, + { url = "https://files.pythonhosted.org/packages/c2/54/5311f7d2605c3693b1c729c2c3b171c60b11a6126a5f41dc44047e7493fb/zope_interface-8.7-cp312-cp312-win_amd64.whl", hash = "sha256:68acf0f25707f9c6277552a3d10114405235385ea1f66bffc89612e0b84f6edd", size = 220958, upload-time = "2026-10-15T07:24:06.935Z" }, + { url = "https://files.pythonhosted.org/packages/7f/fa/1809f8e709024046298bc8655e2291d5722a549d4e60e741fa8d34dcae01/zope_interface-8.7-cp312-cp312-win_arm64.whl", hash = "sha256:b5045f223dcfe8792ad78df2b9ce06797988df02912e832e3ee564af7c3ca9ca", size = 220864, upload-time = "2026-10-15T07:24:08.572Z" }, + { url = "https://files.pythonhosted.org/packages/83/06/e382f0fa24b5d7bf44f44cc82dc1a27d1375f4ec70190c2b02b9944d5e95/zope_interface-8.7-cp313-cp313-macosx_10_9_x86_64.whl", hash = "sha256:78dcd615fe437ed995378478c266dac10a7635c2474fe6ad33bac43af8498a1d", size = 219273, upload-time = "2026-10-15T07:24:10.569Z" }, + { url = "https://files.pythonhosted.org/packages/52/94/bde065c2cd987dad779bafeeb9ec6a8bb0cff09f6b77df327e1e776f65df/zope_interface-8.7-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:ae33b2ff2acff7b0ebd4272c3396a97c43f06cb2ac83820e16200ad50183bd50", size = 219285, upload-time = "2026-10-15T07:24:12.413Z" }, + { url = "https://files.pythonhosted.org/packages/aa/1f/263e83fef05e343e95b4c8fa2768301b7cd5964dd94afe5608561584c180/zope_interface-8.7-cp313-cp313-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:96c9f040f7449b8dc2cfd58b2320c070c18dda5c98bfec27c6420dceea6a0f5b", size = 277484, upload-time = "2026-10-15T07:24:14.051Z" }, + { url = "https://files.pythonhosted.org/packages/94/0c/a80dd47fdca2c210111218e8b4132fefe93e1e34fe0ae129436128d6cbfa/zope_interface-8.7-cp313-cp313-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:d30ed06ef78e9e1b41a50683b7d01727a3c363143c5bda09017e33f19827afc2", size = 283452, upload-time = "2026-10-15T07:24:15.848Z" }, + { url = "https://files.pythonhosted.org/packages/23/4b/0989b9c683a7c88a40c46eb35e1a8890aabee511f9b863d52bc1a2ba006c/zope_interface-8.7-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:75ae2cca3a82dc37834cd8277044ee3a571bc2f81849541689a76997dc50812e", size = 282636, upload-time = "2026-10-15T07:24:17.426Z" }, + { url = "https://files.pythonhosted.org/packages/c2/fe/97712b2ade92f285da7d4d7b908a023082c08e2202cc858172db536d3c4d/zope_interface-8.7-cp313-cp313-win_amd64.whl", hash = "sha256:294aca67c65b10341cc6ed2e103ef6d49d6c2f1bca30135d668db38be522c364", size = 221260, upload-time = "2026-10-15T07:24:19.151Z" }, + { url = "https://files.pythonhosted.org/packages/80/be/258bd4262c533f2e5be125334cf4053552c6a0fa47406dcc03d1719dc558/zope_interface-8.7-cp313-cp313-win_arm64.whl", hash = "sha256:eeec8bb03f69706876a2bfdfa93b6f70c23230f9c655f8d14726b5bad1319b68", size = 220813, upload-time = "2026-10-15T07:24:20.841Z" }, + { url = "https://files.pythonhosted.org/packages/94/92/617979e355fc9ff5ab7baf40a2d0586c813b0a43617be9b2b500129f1144/zope_interface-8.7-cp314-cp314-macosx_10_9_x86_64.whl", hash = "sha256:3876907cdeb4f94335ec2748b7017b44e2d054497f09bf9cc32bcdab984ce7c6", size = 219292, upload-time = "2026-10-15T07:24:22.764Z" }, + { url = "https://files.pythonhosted.org/packages/ce/56/6812c4becde5edff05dd6add20bdd2a8c3a3bbf0418dbf159485e113ef3d/zope_interface-8.7-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:e0bd27434ec193f4213da3d7868b5328e71c946ddca97b868ba72232dd42d9ea", size = 219361, upload-time = "2026-10-15T07:24:24.571Z" }, + { url = "https://files.pythonhosted.org/packages/a1/28/678804c8ebf8994c7704166f20d736555b82dab81dd7662ba926418214a1/zope_interface-8.7-cp314-cp314-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:8cfa8c8ee0fbccb9cd9f354771198fe412af8377ddab86887dcab044430f2968", size = 279376, upload-time = "2026-10-15T07:24:26.341Z" }, + { url = "https://files.pythonhosted.org/packages/50/03/372676f4a91df53b9fae808b26fac6fce3d8e02bfa0e134162d11a7b607b/zope_interface-8.7-cp314-cp314-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:6260ccc856a2c561b20341a74a8c1d9bb13916f6b52e880f336a0ddf61a1b726", size = 282796, upload-time = "2026-10-15T07:24:28.099Z" }, + { url = "https://files.pythonhosted.org/packages/e5/10/f885be266bf4e2edd239f2ead7400bf39d605e01e27a35c540ec9276f728/zope_interface-8.7-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:6cc109b5d1faef084ab1a1d1291d768dd8fcfb87685a3a15259066ded25c1d73", size = 282504, upload-time = "2026-10-15T07:24:30.196Z" }, + { url = "https://files.pythonhosted.org/packages/c4/04/e58700ee9a85aa5c245ad2a2f363422c011e06250b6cdda9545783aee894/zope_interface-8.7-cp314-cp314-win_amd64.whl", hash = "sha256:e53386608f473d78dc7f968aceaaed5c0df7184efbc2bc0dda07bde3a6b9bd0b", size = 221569, upload-time = "2026-10-15T07:24:31.89Z" }, + { url = "https://files.pythonhosted.org/packages/61/73/b16250960b01fe6e4d011b2fb5fb4a49832ecd47fcc78a367461d06570e6/zope_interface-8.7-cp314-cp314-win_arm64.whl", hash = "sha256:3aff75b2e0e18fba9cb3f221be321852c262d89ffe60590bbb8daad20bf6bcbd", size = 220971, upload-time = "2026-10-15T07:24:33.745Z" }, + { url = "https://files.pythonhosted.org/packages/a3/f0/58a434974db9591f4256f8c56d0421993608fdf960fac13ec1e6411d2787/zope_interface-8.7-cp314-cp314t-macosx_10_9_x86_64.whl", hash = "sha256:2d632afb26be0bc0a021c188ace8d95604460809b75a1b80218fe0173f19b9bd", size = 220823, upload-time = "2026-10-15T07:24:36.311Z" }, + { url = "https://files.pythonhosted.org/packages/67/64/d8a92fbfaba961cdc04e96d9a431203f050c188a3e0af9420ce98f187e49/zope_interface-8.7-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:bd466a59274435a628d03697996fda99e22276af6516011a038b97da830664d3", size = 220848, upload-time = "2026-10-15T07:24:38.035Z" }, + { url = "https://files.pythonhosted.org/packages/aa/e8/6203725ec87e586be6e09a584fda4c6baa0d67579c0b2d1279e6847a4849/zope_interface-8.7-cp314-cp314t-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:36e3ec353100356dcdd711c6f5a328095b33cc573c82d01e106e4a13a874c0f4", size = 308516, upload-time = "2026-10-15T07:24:39.701Z" }, + { url = "https://files.pythonhosted.org/packages/83/7b/3ebc85e0b9769e686feadf669a1629910728b3ac1fc8242589eb7a5c1abc/zope_interface-8.7-cp314-cp314t-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:dad0ede8e243d5dc17b453c995e330815e524df5c502757c6221fc6a12380823", size = 314827, upload-time = "2026-10-15T07:24:41.461Z" }, + { url = "https://files.pythonhosted.org/packages/42/53/c81d54a200097eeb85a2ee830b6121c31ef316037e705019f82183f23570/zope_interface-8.7-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:12ef0f3338c07bc00cc64f80a32003105bee5be43e8577d535acdd16b3b03967", size = 319628, upload-time = "2026-10-15T07:24:43.287Z" }, + { url = "https://files.pythonhosted.org/packages/b1/80/856de74a33738691c372abadcbc5cb7fa034f91e1ae12f4f3505600cce38/zope_interface-8.7-cp314-cp314t-win_amd64.whl", hash = "sha256:d051d031e6e73c5ea55fc84389dc77b5a317cbece1d16e8a35e9433eabe70e16", size = 222601, upload-time = "2026-10-15T07:24:44.96Z" }, + { url = "https://files.pythonhosted.org/packages/82/bc/966eec3963317acf7bc5d9e19e8d0b7f41ff35595b8e60a2145d76232340/zope_interface-8.7-cp314-cp314t-win_arm64.whl", hash = "sha256:48c98219d718e48d98c6c9ca3c2102894410e542d09f730b9d67b3431027e3c8", size = 222184, upload-time = "2026-10-15T07:24:47.241Z" }, + { url = "https://files.pythonhosted.org/packages/44/e4/66c961c0a4cb7b8561a8855036f8fca6a6e9feac54fc609835e95f57d3a5/zope_interface-8.7-cp315-cp315-macosx_10_9_x86_64.whl", hash = "sha256:6c84d5a260db4de770c9dbff542b28cfe7802c7d286d211d59f32b1b05fb1e69", size = 219298, upload-time = "2026-10-15T07:24:48.785Z" }, + { url = "https://files.pythonhosted.org/packages/ad/17/c6ae2f1265a9be806841df2890f2e12cbe16ef6287781ee06db3f4e37cef/zope_interface-8.7-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:a319373c6fb786f47d816ad16c8bda604438fd4a32ddc77af411d551ec210cd4", size = 219360, upload-time = "2026-10-15T07:24:50.416Z" }, + { url = "https://files.pythonhosted.org/packages/f4/de/9c7002982a3b2f130375b74e8df0df8c7656e910b1dd61cc89dfa948a425/zope_interface-8.7-cp315-cp315-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:8dacae53e12f22d6d3041420579c1e1c43cece47525350619a2cc88e93581a2c", size = 280101, upload-time = "2026-10-15T07:24:52.161Z" }, + { url = "https://files.pythonhosted.org/packages/b3/86/9e545fe873140dc61c875f013e0d873ab006ca7f6ace933e6e9fd81d5e45/zope_interface-8.7-cp315-cp315-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:a0d84e36c426afb6469aa6c4d438d12e18394ace596f5698f835fc434bd0ae1d", size = 282835, upload-time = "2026-10-15T07:24:54.401Z" }, + { url = "https://files.pythonhosted.org/packages/6a/54/28590cfa4adcc21d5960c3ab2ed5c651b60c084a6d844c1cbafda57cb6d9/zope_interface-8.7-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:39299d2f03fb1eada8ee7f754a834d0a4e9d5421284ed7b0d9ea37a8fa0eb58e", size = 282798, upload-time = "2026-10-15T07:24:55.926Z" }, + { url = "https://files.pythonhosted.org/packages/2b/17/dbfbc44a870f9e87fac9d85d8d48ac49603baf6aaa59898fc7b6c5ca4d08/zope_interface-8.7-cp315-cp315-win_amd64.whl", hash = "sha256:10f15d6b70842405755d6ef128d731ff14f2f655bad56b7fe5d19588c24d08bc", size = 221568, upload-time = "2026-10-15T07:24:57.586Z" }, + { url = "https://files.pythonhosted.org/packages/d5/8a/b54dbd04a7800e6101b49b64ba7991fda5d7c3b9945a843af0565b87225a/zope_interface-8.7-cp315-cp315-win_arm64.whl", hash = "sha256:31979c1841fb58f69a19a1593348a4e86bfcd5619e02909bd6a0c78a1e670af7", size = 220966, upload-time = "2026-10-15T07:24:59.249Z" }, + { url = "https://files.pythonhosted.org/packages/99/94/e6ee2713d41b57592d89000b91a847360603726d509c3386a06c223cc366/zope_interface-8.7-cp315-cp315t-macosx_10_9_x86_64.whl", hash = "sha256:f23736eda7fbd9125b41e41e437217c6328dddb303be522b1938a70eeb6eaf1e", size = 220778, upload-time = "2026-10-15T07:25:01.27Z" }, + { url = "https://files.pythonhosted.org/packages/61/1c/f5d51fdfb1ab50d21f3c4289e079051df8735a6896984107423aebdddd44/zope_interface-8.7-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:8a6f644b6bb37e4248c3f5a526912aa35237a8ad7b9fa512540c4e230c8a4dad", size = 220812, upload-time = "2026-10-15T07:25:03.359Z" }, + { url = "https://files.pythonhosted.org/packages/ab/c7/7eb16d3cba771959eb7288674aca011b48014c73d0cdb4dc15bc5feec702/zope_interface-8.7-cp315-cp315t-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:cb074d4e2a5197812ebb954b718f4f989d6c20a4e12c5e4cc6d6ea57d53d571e", size = 308392, upload-time = "2026-10-15T07:25:05.002Z" }, + { url = "https://files.pythonhosted.org/packages/26/f3/4d5859c3dae41442757e3ee92a9ec2dbebca4aa4ef4e9cda03688afc01e6/zope_interface-8.7-cp315-cp315t-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:c616440ba2237dfdef6cc8a2c4a7fcdb489151cd0b89ae664180b4d9bf2a2f12", size = 314604, upload-time = "2026-10-15T07:25:07.122Z" }, + { url = "https://files.pythonhosted.org/packages/6d/25/31fc42cbd539734040ed95df708d94a86c6d318b6e508e174760ea73e9c8/zope_interface-8.7-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cefec3205cac03bb9955d44b95d68ffcfd0bdf8c7ab40a5bd969797279a82b51", size = 319194, upload-time = "2026-10-15T07:25:09.133Z" }, + { url = "https://files.pythonhosted.org/packages/b0/a4/33055e2590fd00d84ecd1e6d19f69a891a72aade2211fd3740aa317145f7/zope_interface-8.7-cp315-cp315t-win_amd64.whl", hash = "sha256:53672982c9b963c04f2ebbba164d7a7dc4fed4b5e16b5210f37edc96b2e64741", size = 222557, upload-time = "2026-10-15T07:25:11.229Z" }, + { url = "https://files.pythonhosted.org/packages/f8/f6/e1e0af070c94d3be176f6e44aa9280213aa657de4c6b42320b50d906b417/zope_interface-8.7-cp315-cp315t-win_arm64.whl", hash = "sha256:d964fac37a2877d46d797e8b12496b52e3cb5b5acde10ed1510d873d7875e57e", size = 222179, upload-time = "2026-10-15T07:25:13.112Z" }, +] From e504d851c3a164c51787515e17464856a1ce5dd5 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 10 Nov 2026 23:07:09 +0100 Subject: [PATCH 139/250] Treat the fork warning in the tests as an error again Only the removed process mode of the queue forked in the tests. --- pyproject.toml | 2 -- 1 file changed, 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 3e61f895..55c6c081 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -176,8 +176,6 @@ python_files = ["*Test.py"] addopts = ["-ra", "--strict-config", "--strict-markers"] filterwarnings = [ "error", - # The process mode of the queue forks a process that has threads (Python 3.12 and 3.13). - "ignore:This process .* is multi-threaded, use of fork\\(\\) may lead to deadlocks:DeprecationWarning", # lxml enables the GIL of free-threaded Python; CI runs these tests with PYTHON_GIL=0. "ignore:The global interpreter lock \\(GIL\\) has been enabled:RuntimeWarning", ] From 36b37a80ad625fb669805b661d48d28b640602c7 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 10 Nov 2026 23:28:23 +0100 Subject: [PATCH 140/250] Annotate the queue and the work queue Check them strictly with mypy, and add docstrings. Correct the docstrings that contradicted the code, e.g. of is_paused() and shutdown(), and say that job ids are strings. --- pyproject.toml | 6 - src/Exscript/queue.py | 258 +++++++++++++++++++++------- src/Exscript/workqueue/job.py | 119 +++++++++++-- src/Exscript/workqueue/mainloop.py | 128 ++++++++++++-- src/Exscript/workqueue/pipeline.py | 166 +++++++++++++----- src/Exscript/workqueue/task.py | 31 +++- src/Exscript/workqueue/workqueue.py | 145 +++++++++++----- 7 files changed, 669 insertions(+), 184 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 55c6c081..f27d8cf7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -132,13 +132,7 @@ ignore_missing_imports = true # ncclient and others ship no type hints # Modules that are not annotated yet (P8). Remove a module from this list when # it passes the strict checks; the list only shrinks. module = [ - "Exscript.queue", "Exscript.util.sigint", - "Exscript.workqueue.job", - "Exscript.workqueue.mainloop", - "Exscript.workqueue.pipeline", - "Exscript.workqueue.task", - "Exscript.workqueue.workqueue", ] disallow_any_generics = false disallow_subclassing_any = false diff --git a/src/Exscript/queue.py b/src/Exscript/queue.py index c1ed5fc3..3f418623 100644 --- a/src/Exscript/queue.py +++ b/src/Exscript/queue.py @@ -28,7 +28,10 @@ import logging import sys import threading +from collections.abc import Callable, Hashable, Iterable from functools import partial +from types import TracebackType +from typing import TYPE_CHECKING, Any from .account import AccountManager from .logger import logger_registry @@ -39,6 +42,12 @@ from .util.tty import get_terminal_size from .workqueue import Task, WorkQueue +if TYPE_CHECKING: + from .account import Account, AccountPool + from .host import Host + from .protocols import Protocol + from .workqueue.job import Job, Thread + _logger = logging.getLogger(__name__) _logger.addHandler(logging.NullHandler()) @@ -46,14 +55,22 @@ class _Discard: """Discards what is written to it, like os.devnull, but has no file to close.""" - def write(self, data): + def write(self, data: str) -> int: + """ + Discards the given data, and returns its length. + """ return len(data) - def flush(self): + def flush(self) -> None: + """ + Does nothing. + """ pass -def _account_factory(accm, owner, host, account): +def _account_factory( + accm: AccountManager, owner: Hashable, host: 'Host', account: 'Account | None' +) -> 'Account': """ Returns the account to log into the host with, acquired: the given account, else the account of the host, else the one that the account @@ -67,7 +84,9 @@ def _account_factory(accm, owner, host, account): return accm.acquire_account(account, owner) -def _prepare_connection(func, account_manager): +def _prepare_connection( + func: 'Callable[[Thread, Host, Protocol], object]', account_manager: AccountManager +) -> 'Callable[[Thread], object]': """ A decorator that unpacks the host and connection from the job argument and passes them as separate arguments to the wrapped function. The @@ -75,7 +94,7 @@ def _prepare_connection(func, account_manager): that the job still holds are released when the function returns. """ - def _wrapped(job, *args, **kwargs): + def _wrapped(job: 'Thread', *args: Any, **kwargs: Any) -> object: try: return _run_job(func, account_manager, job, *args, **kwargs) finally: @@ -84,11 +103,19 @@ def _wrapped(job, *args, **kwargs): return _wrapped -def _run_job(func, account_manager, job, *args, **kwargs): +def _run_job( + func: 'Callable[[Thread, Host, Protocol], object]', + account_manager: AccountManager, + job: 'Thread', + *args: Any, + **kwargs: Any, +) -> object: """ Connects to the host of the given job and calls the given function with the job, the host and the connection, and the given arguments. Logs the session if the function was decorated with log_to(). + + :return: The result of the function. """ job_id = id(job) host = job.data['host'] @@ -131,15 +158,23 @@ def _run_job(func, account_manager, job, *args, **kwargs): return result -def _is_recoverable_error(cls): +def _is_recoverable_error(cls: type[BaseException]) -> bool: + """ + Returns True if the given exception class is CompileError or + FailException of the template language. The queue writes the + tracebacks of these only at verbosity 2 or higher. + """ # By name, so that this module does not import the template language. return cls.__name__ in ('CompileError', 'FailException') -def _call_logger(funcname, logger_id, *args): +# Any: the result of the method, whose name is given at runtime. +def _call_logger(funcname: str, logger_id: int, *args: Any) -> Any: """ Calls the method with the given name of the logger with the given id, if the logger still exists, with the given arguments. + + :return: The result of the method, or None. """ logger = logger_registry.get(logger_id) if not logger: @@ -154,15 +189,18 @@ class Queue: def __init__( self, - domain='', - verbose=1, - mode='threading', - max_threads=1, - host_driver=None, - exc_cb=None, - stdout=sys.stdout, - stderr=sys.stderr, - ): + domain: str = '', + verbose: int = 1, + mode: str = 'threading', + max_threads: int = 1, + host_driver: str | None = None, + exc_cb: ( + Callable[[str, tuple[type[BaseException], BaseException, TracebackType | str]], object] + | None + ) = None, + stdout: Any = sys.stdout, # any object with write() and flush(), like Protocol.stdout + stderr: Any = sys.stderr, # any object with write() and flush() + ) -> None: """ Constructor. All arguments should be passed as keyword arguments. Depending on the verbosity level, the following types @@ -201,9 +239,11 @@ def __init__( :type exc_cb: func(jobname, exc_info) :param exc_cb: callback function to call on exceptions :type stdout: file - :param stdout: The output channel, defaults to sys.stdout. + :param stdout: The output channel, defaults to sys.stdout. Any + object with write() and flush() works. :type stderr: file - :param stderr: The error channel, defaults to sys.stderr. + :param stderr: The error channel, defaults to sys.stderr. Any + object with write() and flush() works. """ self._logger = _logger self.workqueue = WorkQueue(mode=mode) @@ -231,7 +271,11 @@ def __init__( self.workqueue.job_succeeded_event.listen(self._on_job_succeeded) self.workqueue.job_aborted_event.listen(self._on_job_aborted) - def _update_verbosity(self): + def _update_verbosity(self) -> None: + """ + Maps the output channels to stdout, stderr or nothing, by the + verbosity and the maximum number of threads; see the constructor. + """ if self.verbose < 0: self.channel_map['status_bar'] = self.devnull self.channel_map['connection'] = self.devnull @@ -263,18 +307,24 @@ def _update_verbosity(self): self.channel_map['errors'] = self.stderr self.channel_map['tracebacks'] = self.stderr - def _write(self, channel, msg): + def _write(self, channel: str, msg: str) -> None: + """ + Writes the given message to the given output channel, and flushes it. + """ with self.lock: self.channel_map[channel].write(msg) self.channel_map[channel].flush() - def _del_status_bar(self): + def _del_status_bar(self) -> None: + """ + Erases the status bar, if any, with backspaces. + """ if self.status_bar_length == 0: return self._write('status_bar', '\b \b' * self.status_bar_length) self.status_bar_length = 0 - def get_progress(self): + def get_progress(self) -> float: """ Returns the progress in percent. @@ -285,7 +335,11 @@ def get_progress(self): return 0.0 return 100.0 / self.total * self.completed - def _print_status_bar(self, exclude=None): + def _print_status_bar(self, exclude: str | None = None) -> None: + """ + Writes the status bar: the names of the running jobs, except the + given one, and the progress. Writes nothing if no job is running. + """ if self.total == 0: return percent = 100.0 / self.total * self.completed @@ -310,26 +364,49 @@ def _print_status_bar(self, exclude=None): self._write('status_bar', text) self.status_bar_length = len(text) - def _print(self, channel, msg): + def _print(self, channel: str, msg: str) -> None: + """ + Writes the given message and a newline to the given output channel, + and redraws the status bar after it. + """ self._del_status_bar() self._write(channel, msg + '\n') self._print_status_bar() - def _dbg(self, level, msg, exc_info=None): + def _dbg(self, level: int, msg: str, exc_info: bool | BaseException | None = None) -> None: + """ + Logs the given message if the verbosity is at least the given level. + """ if level > self.verbose: return self._logger.info(msg, exc_info=exc_info) - def _on_job_init(self, job): + def _on_job_init(self, job: 'Job') -> None: + """ + Adds the output channel of the connection to the data of the given + job, before its first attempt. + """ if job.data is None: job.data = {} job.data['stdout'] = self.channel_map['connection'] - def _on_job_started(self, job): + def _on_job_started(self, job: 'Thread') -> None: + """ + Redraws the status bar when an attempt of a job starts. + """ self._del_status_bar() self._print_status_bar() - def _on_job_error(self, job, exc_info): + def _on_job_error( + self, + job: 'Thread', + exc_info: tuple[type[BaseException], BaseException, TracebackType | str], + ) -> None: + """ + Handles a failed attempt of the given job: calls the exception + callback, and writes the error and its traceback to the output + channels. + """ msg = job.name + ' error: ' + str(exc_info[1]) trace = ''.join(format_exception(*exc_info)) if self.exc_cb: @@ -340,20 +417,28 @@ def _on_job_error(self, job, exc_info): else: self._print('fatal_errors', trace) - def _on_job_succeeded(self, job): + def _on_job_succeeded(self, job: 'Thread') -> None: + """ + Counts the given job as completed, and reports its success on the + status bar channel. + """ self.completed += 1 self._del_status_bar() self._write('status_bar', job.name + ' succeeded.' + '\n') self._print_status_bar(exclude=job.name) - def _on_job_aborted(self, job): + def _on_job_aborted(self, job: 'Thread') -> None: + """ + Counts the given job as completed and failed, after its last + attempt failed, and reports it on the errors channel. + """ self.completed += 1 self.failed += 1 self._del_status_bar() self._write('errors', job.name + ' finally failed.' + '\n') self._print_status_bar(exclude=job.name) - def set_max_threads(self, n_connections): + def set_max_threads(self, n_connections: int) -> None: """ Sets the maximum number of concurrent connections. @@ -363,7 +448,7 @@ def set_max_threads(self, n_connections): self.workqueue.set_max_threads(n_connections) self._update_verbosity() - def get_max_threads(self): + def get_max_threads(self) -> int: """ Returns the maximum number of concurrent threads. @@ -372,7 +457,9 @@ def get_max_threads(self): """ return self.workqueue.get_max_threads() - def add_account_pool(self, pool, match=None): + def add_account_pool( + self, pool: 'AccountPool', match: 'Callable[[Host], bool] | None' = None + ) -> None: """ Adds a new account pool. If the given match argument is None, the pool is the default pool. Otherwise, the match argument is @@ -432,17 +519,17 @@ def use_this_pool(host): """ self.account_manager.add_pool(pool, match) - def add_account(self, account): + def add_account(self, account: 'Account | Iterable[Account]') -> None: """ Adds the given account to the default account pool that Exscript uses to log into all hosts that have no specific :class:`Account` attached. - :type account: Account - :param account: The account that is added. + :type account: Account|list[Account] + :param account: The account or accounts that are added. """ self.account_manager.add_account(account) - def is_completed(self): + def is_completed(self) -> bool: """ Returns True if the task is completed, False otherwise. In other words, this method returns True if the queue is empty. @@ -452,7 +539,7 @@ def is_completed(self): """ return self.workqueue.get_length() == 0 - def join(self): + def join(self) -> None: """ Waits until all jobs are completed. """ @@ -463,7 +550,7 @@ def join(self): self._print_status_bar() gc.collect() - def shutdown(self, force=False): + def shutdown(self, force: bool = False) -> None: """ Stop executing any further jobs. If the force argument is True, the function does not wait until any queued jobs are completed but @@ -473,7 +560,7 @@ def shutdown(self, force=False): after using this method. :type force: bool - :param force: Whether to wait until all jobs were processed. + :param force: Whether to stop without waiting until all jobs were processed. """ if not force: self.join() @@ -483,14 +570,14 @@ def shutdown(self, force=False): self._dbg(3, 'Queue has shut down.') self._del_status_bar() - def destroy(self, force=False): + def destroy(self, force: bool = False) -> None: """ Like shutdown(), but also removes all accounts, hosts, etc., and does not restart the queue. In other words, the queue can no longer be used after calling this method. :type force: bool - :param force: Whether to wait until all jobs were processed. + :param force: Whether to stop without waiting until all jobs were processed. """ try: if not force: @@ -506,7 +593,7 @@ def destroy(self, force=False): self._dbg(3, 'Queue was destroyed.') self._del_status_bar() - def reset(self): + def reset(self) -> None: """ Remove all accounts, hosts, etc. """ @@ -520,7 +607,22 @@ def reset(self): self._dbg(3, 'Queue was reset.') self._del_status_bar() - def _run(self, hosts, callback, queue_function, *args): + def _run( + self, + hosts: 'str | Host | Iterable[str | Host]', + callback: Callable[..., object], + queue_function: Callable[..., str | None], + *args: Any, + ) -> Task | None: + """ + Enqueues a job for each of the given hosts with the given function + of the work queue, which gets the job function, the name of the host, + the given arguments and the job data. + + :param callback: The function that the jobs call with the job, the + host and the connection. + :return: The task of the jobs, or None if no job was enqueued. + """ hosts = to_hosts(hosts, default_domain=self.domain) self.total += len(hosts) callback = _prepare_connection(callback, self.account_manager) @@ -542,28 +644,39 @@ def _run(self, hosts, callback, queue_function, *args): self._dbg(2, 'All jobs enqueued.') return task - def run(self, hosts, function, attempts=1): + def run( + self, + hosts: 'str | Host | Iterable[str | Host]', + function: 'Callable[[Thread, Host, Protocol], object]', + attempts: int = 1, + ) -> Task | None: """ Add the given function to a queue, and call it once for each host according to the threading options. Use :func:`Exscript.util.decorator.bind` if you also want to pass additional arguments to the callback function. - Returns an object that represents the queued task, and that may be - passed to is_completed() to check the status. + Returns the task of the jobs, whose is_completed() and wait() + methods check its status, or None if no job was enqueued. :type hosts: string|list(string)|Host|list(Host) :param hosts: A hostname or Host object, or a list of them. :type function: function - :param function: The function to execute. + :param function: The function to execute. It gets the job, the host + and the connection, which is not logged in yet. :type attempts: int :param attempts: The number of attempts on failure. - :rtype: object - :return: An object representing the task. + :rtype: Task|None + :return: An object representing the task, or None. """ return self._run(hosts, function, self.workqueue.enqueue, attempts) - def run_or_ignore(self, hosts, function, attempts=1): + def run_or_ignore( + self, + hosts: 'str | Host | Iterable[str | Host]', + function: 'Callable[[Thread, Host, Protocol], object]', + attempts: int = 1, + ) -> Task | None: """ Like run(), but only appends hosts that are not already in the queue. @@ -574,12 +687,17 @@ def run_or_ignore(self, hosts, function, attempts=1): :param function: The function to execute. :type attempts: int :param attempts: The number of attempts on failure. - :rtype: object + :rtype: Task|None :return: A task object, or None if all hosts were duplicates. """ return self._run(hosts, function, self.workqueue.enqueue_or_ignore, attempts) - def priority_run(self, hosts, function, attempts=1): + def priority_run( + self, + hosts: 'str | Host | Iterable[str | Host]', + function: 'Callable[[Thread, Host, Protocol], object]', + attempts: int = 1, + ) -> Task | None: """ Like run(), but adds the task to the front of the queue. @@ -589,12 +707,17 @@ def priority_run(self, hosts, function, attempts=1): :param function: The function to execute. :type attempts: int :param attempts: The number of attempts on failure. - :rtype: object - :return: An object representing the task. + :rtype: Task|None + :return: An object representing the task, or None. """ return self._run(hosts, function, self.workqueue.priority_enqueue, False, attempts) - def priority_run_or_raise(self, hosts, function, attempts=1): + def priority_run_or_raise( + self, + hosts: 'str | Host | Iterable[str | Host]', + function: 'Callable[[Thread, Host, Protocol], object]', + attempts: int = 1, + ) -> Task | None: """ Like priority_run(), but if a host is already in the queue, the existing host is moved to the top of the queue instead of enqueuing @@ -606,12 +729,17 @@ def priority_run_or_raise(self, hosts, function, attempts=1): :param function: The function to execute. :type attempts: int :param attempts: The number of attempts on failure. - :rtype: object + :rtype: Task|None :return: A task object, or None if all hosts were duplicates. """ return self._run(hosts, function, self.workqueue.priority_enqueue_or_raise, False, attempts) - def force_run(self, hosts, function, attempts=1): + def force_run( + self, + hosts: 'str | Host | Iterable[str | Host]', + function: 'Callable[[Thread, Host, Protocol], object]', + attempts: int = 1, + ) -> Task | None: """ Like priority_run(), but starts the task immediately even if that max_threads is exceeded. @@ -622,24 +750,26 @@ def force_run(self, hosts, function, attempts=1): :param function: The function to execute. :type attempts: int :param attempts: The number of attempts on failure. - :rtype: object - :return: An object representing the task. + :rtype: Task|None + :return: An object representing the task, or None. """ return self._run(hosts, function, self.workqueue.priority_enqueue, True, attempts) - def enqueue(self, function, name=None, attempts=1): + def enqueue( + self, function: 'Callable[[Thread], object]', name: str | None = None, attempts: int = 1 + ) -> Task: """ Places the given function in the queue and calls it as soon as a thread is available. To pass additional arguments to the callback, use Python's functools.partial(). :type function: function - :param function: The function to execute. + :param function: The function to execute. It gets the job. :type name: string :param name: A name for the task. :type attempts: int :param attempts: The number of attempts on failure. - :rtype: object + :rtype: Task :return: An object representing the task. """ self.total += 1 diff --git a/src/Exscript/workqueue/job.py b/src/Exscript/workqueue/job.py index a1970029..e180feac 100644 --- a/src/Exscript/workqueue/job.py +++ b/src/Exscript/workqueue/job.py @@ -20,9 +20,16 @@ # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" +The jobs of the work queue, and the threads that run them. +""" + import sys import threading +from collections.abc import Callable from functools import partial +from types import TracebackType +from typing import Any, cast from Exscript.util.impl import serializeable_sys_exc_info @@ -33,16 +40,42 @@ class _ChildWatcher(threading.Thread): with the information about its exception, or with None. """ - def __init__(self, child, callback): + def __init__( + self, + child: 'Thread', + callback: Callable[ + [ + tuple[type[BaseException] | None, BaseException | None, str | TracebackType | None] + | None + ], + object, + ], + ) -> None: + """ + Constructor. + + :param child: The thread of the job, not started yet. + :param callback: Called with the exception information of the job, + or with None if it raised no exception. + """ threading.Thread.__init__(self) self.child = child self.cb = callback - def run(self): + def run(self) -> None: + """ + Starts the thread of the job and waits for it to end, then calls the + callback with the exception information of the job, or with None. + If the thread cannot be started, the callback gets the information + about that exception. + """ try: self.child.start() self.child.join() - result = self.child.exc_info + result: ( + tuple[type[BaseException] | None, BaseException | None, str | TracebackType | None] + | None + ) = self.child.exc_info except: result = sys.exc_info() self.cb(result) @@ -55,15 +88,27 @@ class Thread(threading.Thread): function, exc_info holds the information about it, else None. """ - def __init__(self, id, function, name, data): + # Any: the data of a job is any object that the user of the work queue + # passes; the Exscript queue passes a dict. + def __init__( + self, id: str | None, function: 'Callable[[Thread], object]', name: str, data: Any + ) -> None: + """ + Constructor. + + :param id: The id of the job. + :param function: The function of the job, called with this object. + :param name: The name of the job, also the name of the thread. + :param data: The data of the job. + """ threading.Thread.__init__(self, name=name) self.id = id self.function = function self.failures = 0 self.data = data - self.exc_info = None + self.exc_info: tuple[type[BaseException] | None, BaseException | None, str] | None = None - def run(self): + def run(self) -> None: """ Runs the function of the job. """ @@ -74,19 +119,60 @@ def run(self): class Job: + """ + A function in the work queue, with its name, its maximum number of + attempts and its data. Each attempt runs in a new Thread. + """ + __slots__ = ('id', 'func', 'name', 'times', 'failures', 'data', 'child', 'watcher') - def __init__(self, function, name, times, data): - self.id = None + # Any: the data of the job, see Thread.__init__(). + def __init__( + self, function: Callable[[Thread], object], name: str | None, times: int, data: Any + ) -> None: + """ + Constructor. The work queue sets the id when it adds the job. + + :param function: The function, called with the Thread of each attempt. + :param name: The name of the job. Defaults to the id() of the + function, as a string. + :param times: The maximum number of attempts. + :param data: The data of the job, passed to each Thread. + """ + self.id: str | None = None self.func = function - self.name = name is None and str(id(function)) or name + # Never None, as str(id(function)) is not empty; the cast is for the type checker. + self.name = cast(str, name is None and str(id(function)) or name) self.times = times self.failures = 0 self.data = data - self.child = None - self.watcher = None + self.child: Thread | None = None + self.watcher: _ChildWatcher | None = None - def start(self, child_cls, on_complete, on_start=None): + def start( + self, + child_cls: type[Thread], + on_complete: Callable[ + [ + 'Job', + tuple[type[BaseException] | None, BaseException | None, str | TracebackType | None] + | None, + ], + object, + ], + on_start: Callable[[Thread], object] | None = None, + ) -> None: + """ + Starts a new attempt of the job. Creates the thread of the attempt + from the given class, calls on_start with it, and starts a watcher + thread that runs it. + + :param child_cls: The class of the thread. + :param on_complete: Called by the watcher thread when the attempt has + ended, with this job and the exception information of the + attempt, or None if it raised no exception. + :param on_start: Called with the thread before it starts. + """ self.child = child_cls(self.id, self.func, self.name, self.data) self.child.failures = self.failures if on_start is not None: @@ -94,6 +180,11 @@ def start(self, child_cls, on_complete, on_start=None): self.watcher = _ChildWatcher(self.child, partial(on_complete, self)) self.watcher.start() - def join(self): - self.watcher.join() + def join(self) -> None: + """ + Waits until the watcher thread of the last attempt has ended, and + sets child to None. Only valid after start(). + """ + # start() set the watcher; the cast is for the type checker. + cast(_ChildWatcher, self.watcher).join() self.child = None diff --git a/src/Exscript/workqueue/mainloop.py b/src/Exscript/workqueue/mainloop.py index 99bb5192..ce47f470 100644 --- a/src/Exscript/workqueue/mainloop.py +++ b/src/Exscript/workqueue/mainloop.py @@ -20,18 +20,45 @@ # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" +The main loop of the work queue, which starts the jobs. +""" + import logging import threading +from collections.abc import Callable +from types import TracebackType +from typing import TYPE_CHECKING, Any from ..util.event import Event from .job import Job +if TYPE_CHECKING: + from .job import Thread + from .pipeline import Pipeline + _logger = logging.getLogger(__name__) _logger.addHandler(logging.NullHandler()) class MainLoop(threading.Thread): - def __init__(self, collection, job_cls): + """ + The thread that takes the jobs from the pipeline and starts them, and + restarts each failed job until it has used up its attempts. + + job_init_event gets the Job before its first attempt. job_started_event, + job_succeeded_event and job_aborted_event (after the last failed attempt) + get the Thread of the attempt; job_error_event gets the Thread and the + exception information. queue_empty_event gets no arguments. + """ + + def __init__(self, collection: 'Pipeline', job_cls: 'type[Thread]') -> None: + """ + Constructor. + + :param collection: The pipeline of the jobs. + :param job_cls: The class of the threads that run the jobs. + """ threading.Thread.__init__(self) self._logger = _logger self.job_init_event = Event() @@ -45,18 +72,39 @@ def __init__(self, collection, job_cls): self.debug = 0 self.daemon = True - def _dbg(self, level, msg, exc_info=None): + def _dbg(self, level: int, msg: str, exc_info: bool | BaseException | None = None) -> None: + """ + Logs the given message if the debug level is at least the given level. + """ if level > self.debug: return self._logger.debug(msg, exc_info=exc_info) - def enqueue(self, function, name, times, data): + # Any: the data of the job, see job.Thread.__init__(). + def enqueue( + self, function: 'Callable[[Thread], object]', name: str | None, times: int, data: Any + ) -> str: + """ + Appends a new job to the pipeline. + + :return: The id of the job. + """ job = Job(function, name, times, data) job.id = self.collection.append(job) return job.id - def enqueue_or_ignore(self, function, name, times, data): - def conditional_append(queue): + # Any: the data of the job, see job.Thread.__init__(). + def enqueue_or_ignore( + self, function: 'Callable[[Thread], object]', name: str | None, times: int, data: Any + ) -> str | None: + """ + Like enqueue(), but does nothing if a job with the given name is in + the pipeline already. + + :return: The id of the new job, or None. + """ + + def conditional_append(queue: 'Pipeline') -> str | None: if queue.get_from_name(name) is not None: return None job = Job(function, name, times, data) @@ -65,13 +113,45 @@ def conditional_append(queue): return self.collection.with_lock(conditional_append) - def priority_enqueue(self, function, name, force_start, times, data): + # Any: the data of the job, see job.Thread.__init__(). + def priority_enqueue( + self, + function: 'Callable[[Thread], object]', + name: str | None, + force_start: bool, + times: int, + data: Any, + ) -> str: + """ + Adds a new job to the start of the pipeline. + + :param force_start: Whether to start the job even if the maximum + number of jobs is in progress already. + :return: The id of the job. + """ job = Job(function, name, times, data) job.id = self.collection.appendleft(job, name, force=force_start) return job.id - def priority_enqueue_or_raise(self, function, name, force_start, times, data): - def conditional_append(queue): + # Any: the data of the job, see job.Thread.__init__(). + def priority_enqueue_or_raise( + self, + function: 'Callable[[Thread], object]', + name: str | None, + force_start: bool, + times: int, + data: Any, + ) -> str | None: + """ + Like enqueue_or_ignore(), but if a job with the given name is in the + pipeline already, moves that job to the start of the pipeline, see + Pipeline.prioritize(). + + :param force_start: Whether to force the existing job. + :return: The id of the new job, or None. + """ + + def conditional_append(queue: 'Pipeline') -> str | None: job = queue.get_from_name(name) if job is None: job = Job(function, name, times, data) @@ -82,13 +162,33 @@ def conditional_append(queue): return self.collection.with_lock(conditional_append) - def wait_for(self, job_id): + def wait_for(self, job_id: str) -> None: + """ + Waits until the job with the given id is removed from the pipeline. + """ self.collection.wait_for_id(job_id) - def get_queue_length(self): + def get_queue_length(self) -> int: + """ + Returns the number of jobs in the pipeline, including those in + progress. + """ return len(self.collection) - def _on_job_completed(self, job, exc_info): + def _on_job_completed( + self, + job: Job, + exc_info: ( + tuple[type[BaseException] | None, BaseException | None, str | TracebackType | None] + | None + ), + ) -> None: + """ + Handles the end of an attempt of the given job, with the exception + information of the attempt, or None. Emits the events, then restarts + the job if it failed and has attempts left, else removes it from the + pipeline. The watcher thread of the job calls this. + """ # This function is called in a sub-thread, so we need to be # careful that we are not in a lock while sending an event. self._dbg(4, 'Job "%s" called completed()' % job.name) @@ -119,7 +219,11 @@ def _on_job_completed(self, job, exc_info): else: self.collection.task_done(job) - def run(self): + def run(self) -> None: + """ + Takes the jobs from the pipeline and starts them, until the pipeline + is stopped. + """ while True: # Get the next job from the queue. This blocks until a task # is available or until self.collection.stop() is called. diff --git a/src/Exscript/workqueue/pipeline.py b/src/Exscript/workqueue/pipeline.py index 266dcd7b..fa968c68 100644 --- a/src/Exscript/workqueue/pipeline.py +++ b/src/Exscript/workqueue/pipeline.py @@ -20,8 +20,15 @@ # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" +The collection of the work queue: a queue of items that also tracks the +items in progress. +""" + from collections import deque +from collections.abc import Callable, Hashable from threading import Condition, RLock +from typing import Any, cast from uuid import uuid4 @@ -31,30 +38,46 @@ class Pipeline: it also tracks items that are currently sleeping or in progress. """ - def __init__(self, max_working=1): + def __init__(self, max_working: int = 1) -> None: + """ + Constructor. + + :param max_working: The maximum number of items in progress. + """ self.condition = Condition(RLock()) self.max_working = max_working self.running = True self.paused = False - self.queue = None - self.force = None - self.sleeping = None - self.working = None - self.item2id = None - self.id2item = None # for performance reasons - self.name2id = None - self.id2name = None + # clear() sets the containers; the casts are for the type checker. + # Any: the items are any hashable objects (the work queue adds Job + # objects), and the pipeline returns them as they are. name2id never + # has the key None, but get_from_name() looks it up. + self.queue: deque[Any] = cast('deque[Any]', None) + self.force: deque[Any] = cast('deque[Any]', None) + self.sleeping: set[Any] = cast('set[Any]', None) + self.working: set[Any] = cast('set[Any]', None) + self.item2id: dict[Any, str] = cast('dict[Any, str]', None) + self.id2item: dict[str, Any] = cast('dict[str, Any]', None) # for performance reasons + self.name2id: dict[str | None, str] = cast('dict[str | None, str]', None) + self.id2name: dict[str, str] = cast('dict[str, str]', None) self.clear() - def __len__(self): + def __len__(self) -> int: with self.condition: return len(self.id2item) - def __contains__(self, item): + def __contains__(self, item: object) -> bool: with self.condition: return item in self.item2id - def _register_item(self, name, item): + def _register_item(self, name: str | None, item: Hashable) -> str: + """ + Registers the given item under a new id, and under the given name + unless it is None. Raises an AttributeError if an item with the name + is registered already. + + :return: The id of the item. + """ uuid = uuid4().hex self.id2item[uuid] = item self.item2id[item] = uuid @@ -67,7 +90,7 @@ def _register_item(self, name, item): self.id2name[uuid] = name return uuid - def get_from_name(self, name): + def get_from_name(self, name: str | None) -> Any: """ Returns the item with the given name, or None if no such item is known. @@ -80,13 +103,17 @@ def get_from_name(self, name): return self.id2item[item_id] return None - def has_id(self, item_id): + def has_id(self, item_id: str) -> bool: """ Returns True if the queue contains an item with the given id. """ return item_id in self.id2item - def task_done(self, item): + def task_done(self, item: Hashable) -> None: + """ + Removes the given item, which was in progress, from the pipeline, + and wakes up the threads that wait on the pipeline. + """ with self.condition: try: self.working.remove(item) @@ -107,9 +134,12 @@ def task_done(self, item): self.name2id.pop(name) self.condition.notify_all() - def append(self, item, name=None): + def append(self, item: Hashable, name: str | None = None) -> str: """ Adds the given item to the end of the pipeline. + + :param name: A unique name for the item, see get_from_name(). + :return: The id of the item. """ with self.condition: self.queue.append(item) @@ -117,7 +147,16 @@ def append(self, item, name=None): self.condition.notify_all() return uuid - def appendleft(self, item, name=None, force=False): + def appendleft(self, item: Hashable, name: str | None = None, force: bool = False) -> str: + """ + Adds the given item to the start of the pipeline. A forced item is + returned by next() before all others, even if max_working items are + in progress already. + + :param name: A unique name for the item, see get_from_name(). + :param force: Whether the item is forced. + :return: The id of the item. + """ with self.condition: if force: self.force.append(item) @@ -127,9 +166,12 @@ def appendleft(self, item, name=None, force=False): self.condition.notify_all() return uuid - def prioritize(self, item, force=False): + def prioritize(self, item: Hashable, force: bool = False) -> None: """ - Moves the item to the very left of the queue. + Moves the item to the very left of the queue. Does nothing if the + item is in progress or forced already. + + :param force: Whether to force the item, see appendleft(). """ with self.condition: # If the job is already running (or about to be forced), @@ -143,7 +185,10 @@ def prioritize(self, item, force=False): self.queue.appendleft(item) self.condition.notify_all() - def clear(self): + def clear(self) -> None: + """ + Removes all items, and wakes up the threads that wait on the pipeline. + """ with self.condition: self.queue = deque() self.force = deque() @@ -155,7 +200,7 @@ def clear(self): self.id2name = dict() self.condition.notify_all() - def stop(self): + def stop(self) -> None: """ Force the next() method to return while in another thread. The return value of next() will be None. @@ -164,38 +209,57 @@ def stop(self): self.running = False self.condition.notify_all() - def start(self): + def start(self) -> None: + """ + Undoes stop(): next() returns items again. + """ with self.condition: self.running = True self.condition.notify_all() - def pause(self): + def pause(self) -> None: + """ + Makes next() wait until unpause() is called. + """ with self.condition: self.paused = True self.condition.notify_all() - def unpause(self): + def unpause(self) -> None: + """ + Undoes pause(): next() returns items again. + """ with self.condition: self.paused = False self.condition.notify_all() - def sleep(self, item): + def sleep(self, item: Hashable) -> None: + """ + Marks the given item as sleeping until wake() is called: next() + skips it, and it does not count against max_working. + """ with self.condition: self.sleeping.add(item) self.condition.notify_all() - def wake(self, item): + def wake(self, item: Hashable) -> None: + """ + Undoes sleep() for the given item, which must be sleeping. + """ assert item in self.sleeping with self.condition: self.sleeping.remove(item) self.condition.notify_all() - def wait_for_id(self, item_id): + def wait_for_id(self, item_id: str) -> None: + """ + Waits until the item with the given id is removed from the pipeline. + """ with self.condition: while self.has_id(item_id): self.condition.wait() - def wait(self): + def wait(self) -> None: """ Waits for all currently running tasks to complete. """ @@ -203,7 +267,7 @@ def wait(self): while self.working: self.condition.wait() - def wait_all(self): + def wait_all(self) -> None: """ Waits for all queued and running tasks to complete. """ @@ -211,23 +275,42 @@ def wait_all(self): while len(self) > 0: self.condition.wait() - def with_lock(self, function, *args, **kwargs): + def with_lock[R](self, function: Callable[..., R], *args: Any, **kwargs: Any) -> R: + """ + Calls the given function with the pipeline and the given arguments + while holding the lock of the pipeline. + + :return: The result of the function. + """ with self.condition: return function(self, *args, **kwargs) - def set_max_working(self, max_working): + def set_max_working(self, max_working: int) -> None: + """ + Sets the maximum number of items in progress. + """ with self.condition: self.max_working = int(max_working) self.condition.notify_all() - def get_max_working(self): + def get_max_working(self) -> int: + """ + Returns the maximum number of items in progress. + """ return self.max_working - def get_working(self): + def get_working(self) -> list[Any]: + """ + Returns a list of the items in progress. + """ return list(self.working) - def _popleft_sleeping(self): - sleeping = [] + def _popleft_sleeping(self) -> list[Any]: + """ + Removes the sleeping items from the start of the queue, and returns + them. + """ + sleeping: list[Any] = [] while True: try: node = self.queue[0] @@ -239,7 +322,12 @@ def _popleft_sleeping(self): self.queue.popleft() return sleeping - def _get_next(self, pop=True): + def _get_next(self, pop: bool = True) -> Any: + """ + Returns the first item of the queue that is not sleeping, or None. + + :param pop: Whether to remove the item from the queue. + """ # We need to leave sleeping items in the queue because else we # would not know their original position after they wake up. # So we need to temporarily remove sleeping items from the top of @@ -262,10 +350,10 @@ def _get_next(self, pop=True): self.queue.extendleft(sleeping) return next - def try_next(self): + def try_next(self) -> Any: """ Like next(), but only returns the item that would be selected - right now, without locking and without changing the queue. + right now, without waiting and without changing the queue. """ with self.condition: try: @@ -275,7 +363,7 @@ def try_next(self): return self._get_next(False) - def __next__(self): + def __next__(self) -> Any: with self.condition: while self.running: if self.paused: diff --git a/src/Exscript/workqueue/task.py b/src/Exscript/workqueue/task.py index 48ec9915..0e9d643a 100644 --- a/src/Exscript/workqueue/task.py +++ b/src/Exscript/workqueue/task.py @@ -24,30 +24,47 @@ Represents a batch of enqueued actions. """ +from typing import TYPE_CHECKING + from Exscript.util.event import Event +if TYPE_CHECKING: + from .job import Thread + from .workqueue import WorkQueue + class Task: """ Represents a batch of running actions. """ - def __init__(self, workqueue): + def __init__(self, workqueue: 'WorkQueue') -> None: + """ + Constructor. The task listens to the events of the given work queue + to count its completed jobs; its done_event is emitted when all are + completed. + + :param workqueue: The work queue that runs the jobs of the task. + """ self.done_event = Event() self.workqueue = workqueue - self.job_ids = set() + self.job_ids: set[str] = set() self.completed = 0 self.workqueue.job_succeeded_event.listen(self._on_job_done) self.workqueue.job_aborted_event.listen(self._on_job_done) - def _on_job_done(self, job): + def _on_job_done(self, job: 'Thread') -> None: + """ + Counts the given job as completed if it belongs to the task, and + emits done_event when all jobs of the task are completed. + """ if job.id not in self.job_ids: return self.completed += 1 if self.is_completed(): self.done_event() - def is_completed(self): + def is_completed(self) -> bool: """ Returns True if all actions in the task are completed, returns False otherwise. @@ -57,7 +74,7 @@ def is_completed(self): """ return self.completed == len(self.job_ids) - def wait(self): + def wait(self) -> None: """ Waits until all actions in the task have completed. Does not use any polling. @@ -65,11 +82,11 @@ def wait(self): for theid in self.job_ids: self.workqueue.wait_for(theid) - def add_job_id(self, theid): + def add_job_id(self, theid: str) -> None: """ Adds a job to the task. - :type theid: int + :type theid: str :param theid: The id of the job. """ self.job_ids.add(theid) diff --git a/src/Exscript/workqueue/workqueue.py b/src/Exscript/workqueue/workqueue.py index 31c5291b..ec61a34b 100644 --- a/src/Exscript/workqueue/workqueue.py +++ b/src/Exscript/workqueue/workqueue.py @@ -20,18 +20,27 @@ # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" +The work queue, which runs functions in threads. +""" + import logging +from collections.abc import Callable +from typing import TYPE_CHECKING, Any, cast from ..util.event import Event from .job import Thread from .mainloop import MainLoop from .pipeline import Pipeline +if TYPE_CHECKING: + from .job import Job + _logger = logging.getLogger(__name__) _logger.addHandler(logging.NullHandler()) -def _check_mode(mode): +def _check_mode(mode: str) -> None: """ Raises an exception unless the given mode is 'threading': a ValueError for 'multiprocessing', else a TypeError. @@ -52,10 +61,14 @@ class WorkQueue: for using the workqueue module. """ - def __init__(self, collection=None, max_threads=1, mode='threading'): + def __init__( + self, collection: Pipeline | None = None, max_threads: int = 1, mode: str = 'threading' + ) -> None: """ Constructor. + :param collection: The pipeline of the jobs. Defaults to a new + Pipeline. :type max_threads: int :param max_threads: The maximum number of concurrent threads. :type mode: str @@ -76,10 +89,14 @@ def __init__(self, collection=None, max_threads=1, mode='threading'): self.job_aborted_event = Event() self.queue_empty_event = Event() self.debug = 0 - self.main_loop = None + self.main_loop: MainLoop | None = None self._init() - def _init(self): + def _init(self) -> None: + """ + Creates and starts a new main loop, and forwards its events to the + events of the work queue. + """ self.main_loop = MainLoop(self.collection, self.job_cls) self.main_loop.debug = self.debug self.main_loop.job_init_event.listen(self.job_init_event) @@ -90,16 +107,25 @@ def _init(self): self.main_loop.queue_empty_event.listen(self.queue_empty_event) self.main_loop.start() - def _check_if_ready(self): + # After this check, main_loop is not None; the methods that use it cast + # it to MainLoop for the type checker. + def _check_if_ready(self) -> None: + """ + Raises an exception if the work queue has no main loop: after + destroy(), or after shutdown() without restart. + """ if self.main_loop is None: raise Exception('main loop is already destroyed') - def _dbg(self, level, msg, exc_info=None): + def _dbg(self, level: int, msg: str, exc_info: bool | BaseException | None = None) -> None: + """ + Logs the given message if the debug level is at least the given level. + """ if level > self.debug: return self._logger.debug(msg, exc_info=exc_info) - def set_debug(self, level): + def set_debug(self, level: int) -> None: """ Set the debug level. @@ -108,9 +134,9 @@ def set_debug(self, level): """ self._check_if_ready() self.debug = level - self.main_loop.debug = level + cast(MainLoop, self.main_loop).debug = level - def get_max_threads(self): + def get_max_threads(self) -> int: """ Returns the maximum number of concurrent threads. @@ -120,7 +146,7 @@ def get_max_threads(self): self._check_if_ready() return self.collection.get_max_working() - def set_max_threads(self, max_threads): + def set_max_threads(self, max_threads: int) -> None: """ Set the maximum number of concurrent threads. @@ -132,7 +158,14 @@ def set_max_threads(self, max_threads): self._check_if_ready() self.collection.set_max_working(max_threads) - def enqueue(self, function, name=None, times=1, data=None): + # Any: the data of the job, see job.Thread.__init__(). + def enqueue( + self, + function: Callable[[Thread], object], + name: str | None = None, + times: int = 1, + data: Any = None, + ) -> str: """ Appends a function to the queue for execution. The times argument specifies the number of attempts if the function raises an exception. @@ -147,13 +180,20 @@ def enqueue(self, function, name=None, times=1, data=None): :param times: The maximum number of attempts. :type data: object :param data: Optional data to store in Job.data. - :rtype: int + :rtype: str :return: The id of the new job. """ self._check_if_ready() - return self.main_loop.enqueue(function, name, times, data) + return cast(MainLoop, self.main_loop).enqueue(function, name, times, data) - def enqueue_or_ignore(self, function, name=None, times=1, data=None): + # Any: the data of the job, see job.Thread.__init__(). + def enqueue_or_ignore( + self, + function: Callable[[Thread], object], + name: str | None = None, + times: int = 1, + data: Any = None, + ) -> str | None: """ Like enqueue(), but does nothing if a function with the same name is already in the queue. @@ -167,13 +207,21 @@ def enqueue_or_ignore(self, function, name=None, times=1, data=None): :param times: The maximum number of attempts. :type data: object :param data: Optional data to store in Job.data. - :rtype: int or None + :rtype: str or None :return: The id of the new job. """ self._check_if_ready() - return self.main_loop.enqueue_or_ignore(function, name, times, data) - - def priority_enqueue(self, function, name=None, force_start=False, times=1, data=None): + return cast(MainLoop, self.main_loop).enqueue_or_ignore(function, name, times, data) + + # Any: the data of the job, see job.Thread.__init__(). + def priority_enqueue( + self, + function: Callable[[Thread], object], + name: str | None = None, + force_start: bool = False, + times: int = 1, + data: Any = None, + ) -> str: """ Like :class:`enqueue()`, but adds the given function at the top of the queue. @@ -190,13 +238,23 @@ def priority_enqueue(self, function, name=None, force_start=False, times=1, data :param times: The maximum number of attempts. :type data: object :param data: Optional data to store in Job.data. - :rtype: int + :rtype: str :return: The id of the new job. """ self._check_if_ready() - return self.main_loop.priority_enqueue(function, name, force_start, times, data) + return cast(MainLoop, self.main_loop).priority_enqueue( + function, name, force_start, times, data + ) - def priority_enqueue_or_raise(self, function, name=None, force_start=False, times=1, data=None): + # Any: the data of the job, see job.Thread.__init__(). + def priority_enqueue_or_raise( + self, + function: Callable[[Thread], object], + name: str | None = None, + force_start: bool = False, + times: int = 1, + data: Any = None, + ) -> str | None: """ Like priority_enqueue(), but if a function with the same name is already in the queue, the existing function is moved to the top of @@ -207,17 +265,20 @@ def priority_enqueue_or_raise(self, function, name=None, force_start=False, time :param function: The function that is executed. :type name: str :param name: Stored in Job.name. + :param force_start: Whether to start execution immediately. :type times: int :param times: The maximum number of attempts. :type data: object :param data: Optional data to store in Job.data. - :rtype: int or None + :rtype: str or None :return: The id of the new job. """ self._check_if_ready() - return self.main_loop.priority_enqueue_or_raise(function, name, force_start, times, data) + return cast(MainLoop, self.main_loop).priority_enqueue_or_raise( + function, name, force_start, times, data + ) - def unpause(self): + def unpause(self) -> None: """ Restart the execution of enqueued jobs after pausing them. This method is the opposite of pause(). @@ -225,7 +286,7 @@ def unpause(self): """ self.collection.unpause() - def pause(self): + def pause(self) -> None: """ Stop the execution of enqueued jobs. Executing may later be resumed by calling unpause(). @@ -233,30 +294,30 @@ def pause(self): """ self.collection.pause() - def wait_for(self, job_id): + def wait_for(self, job_id: str) -> None: """ Waits until the job with the given id is completed. - :type job_id: int - :param job_id: The job that is executed. + :type job_id: str + :param job_id: The id of the job. """ self._check_if_ready() - self.main_loop.wait_for(job_id) + cast(MainLoop, self.main_loop).wait_for(job_id) - def wait_until_done(self): + def wait_until_done(self) -> None: """ Waits until the queue is empty. """ self.collection.wait_all() - def shutdown(self, restart=True): + def shutdown(self, restart: bool = True) -> None: """ Stop the execution of enqueued jobs, and wait for all running jobs to complete. This method is synchronous and returns as soon as all jobs are terminated (i.e. all threads are stopped). - If restart is True, the workqueue is restarted and paused, - so you may fill it with new jobs. + If restart is True, the workqueue is restarted, so you may fill it + with new jobs; if it was paused, it stays paused. If restart is False, the WorkQueue can no longer be used after calling this method. @@ -267,37 +328,37 @@ def shutdown(self, restart=True): self._check_if_ready() self.collection.stop() self.collection.wait() - self.main_loop.join() + cast(MainLoop, self.main_loop).join() self.main_loop = None self.collection.clear() if restart: self.collection.start() self._init() - def destroy(self): + def destroy(self) -> None: """ Like shutdown(), but does not restart the queue and does not wait for already started jobs to complete. """ self._check_if_ready() self.collection.stop() - self.main_loop.join() + cast(MainLoop, self.main_loop).join() self.main_loop = None self.collection.clear() - def is_paused(self): + def is_paused(self) -> bool: """ - Returns True if the queue is currently active (i.e. not - paused and not shut down), False otherwise. + Returns True if the queue is paused, shut down without restart or + destroyed, False otherwise. :rtype: bool - :return: Whether enqueued jobs are currently executed. + :return: Whether enqueued jobs are currently not executed. """ if self.main_loop is None: return True return self.collection.paused - def get_running_jobs(self): + def get_running_jobs(self) -> 'list[Job]': """ Returns a list of all jobs that are currently in progress. @@ -306,7 +367,7 @@ def get_running_jobs(self): """ return self.collection.get_working() - def get_length(self): + def get_length(self) -> int: """ Returns the number of currently non-completed jobs. From c35f376c6d2bf9787ebc94934a30f09f82608fb1 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Tue, 10 Nov 2026 23:28:56 +0100 Subject: [PATCH 141/250] Drop the cast of the now annotated Queue in util.start --- src/Exscript/util/start.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/Exscript/util/start.py b/src/Exscript/util/start.py index d0df9595..820f5d09 100644 --- a/src/Exscript/util/start.py +++ b/src/Exscript/util/start.py @@ -25,7 +25,7 @@ """ from collections.abc import Callable, Iterable -from typing import Any, cast +from typing import Any from .. import Account, Host, Queue from .decorator import autoauthenticate, autologin @@ -70,9 +70,7 @@ def my_callback(job, host, conn, my_arg, **kwargs): attempts = kwargs.get("attempts", 1) if "attempts" in kwargs: del kwargs["attempts"] - # The queue is not annotated (it will be reworked); the cast is for the - # type checker. - queue = cast(Any, Queue)(**kwargs) + queue = Queue(**kwargs) queue.add_account(users) queue.run(hosts, func, attempts) queue.destroy() From 83fce70f295542cc93c3d8a1971c1c0623553b36 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 00:23:26 +0100 Subject: [PATCH 142/250] Match the prompts of junos and mrv in linear time Their patterns began with (?:\r\n|[\r\n])+, which backtracks exponentially on line breaks that no prompt follows, so a block of empty lines could stall the session. Match each run of line breaks once, possessively and only from its start. --- CHANGES.md | 6 ++ src/Exscript/protocols/drivers/junos.py | 10 +++- src/Exscript/protocols/drivers/mrv.py | 3 +- tests/Exscript/protocols/PromptRegexTest.py | 66 ++++++++++++++++++--- 4 files changed, 72 insertions(+), 13 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index a3c9f775..003fadca 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -334,6 +334,12 @@ development versions; "before" refers to them. constructor ignored such a name silently and guessed the driver, so a typo, e.g. in an inventory, went unnoticed. To migrate, correct the name; `drivers.driver_map` lists the names. +- **Junos and MRV Prompts in Linear Time:** The prompt patterns of the `junos` and `mrv` + drivers backtracked exponentially on line breaks that no prompt follows: 22 empty + lines took 41 seconds to check with `junos`, and every further line doubled the time, + so a device that sent a block of empty lines could stall the session. They now match + each run of line breaks once, in linear time, and match the same texts at the same + positions, verified against the old patterns. - **Generic Prompt Pattern in Linear Time:** The prompt pattern of the generic driver, which other drivers inherit, backtracked catastrophically on long lines that are no prompt: a line with a run of 100 word characters took 1.3 seconds to check, 200 diff --git a/src/Exscript/protocols/drivers/junos.py b/src/Exscript/protocols/drivers/junos.py index 3a8fd0df..064e4ef0 100644 --- a/src/Exscript/protocols/drivers/junos.py +++ b/src/Exscript/protocols/drivers/junos.py @@ -64,9 +64,13 @@ _ps = r'(?:\{primary:node\d+\}|\{secondary:node\d+\})' _re_re = r'(?:' + _mb + r'|' + _ps + r')' _edit = r'(?:\[edit[^\]\r\n]*\])' -_prefix = r'(?:(?:\r\n|[\r\n])+' + _re_re + r'?' + _edit + r'?)' -_prompt = r'(?:\r\n|[\r\n])+[\w\-\.]+@[\-\w+\.:]++(?:[~/][\w\-\./]*+ ?)?[%>#] $' -_prompt_re = [re.compile(_prefix + r'?' + _prompt)] +# A run of line breaks, matched as a whole and only from its start. A pattern +# that can split a run in several ways, such as (?:\r\n|[\r\n])+, backtracks +# exponentially on many line breaks that no prompt follows. +_breaks = r'(?#] $' +_prompt_re = [re.compile(_breaks + r'(?:' + _tags + r'[\r\n]++)?' + _prompt)] _error_re = [re.compile(r'^(unknown|invalid|error|syntax error)', re.I)] _junos_re = re.compile(r'\bjunos\b', re.I) # Junos prints its BSD terminal name, e.g. "router1 (ttyp0)", before "login:". diff --git a/src/Exscript/protocols/drivers/mrv.py b/src/Exscript/protocols/drivers/mrv.py index 78f4edd0..3ea1f435 100644 --- a/src/Exscript/protocols/drivers/mrv.py +++ b/src/Exscript/protocols/drivers/mrv.py @@ -35,7 +35,8 @@ _user_re = [re.compile(r'(?:\r\n|[\r\n])Authorized Login: $')] _password_re = [re.compile(r'(?:\r\n|[\r\n])Password: $')] _mrv_re = re.compile(r'\bMRV\b') -_prompt_re = [re.compile(r'(?:\r\n|[\r\n])+[^$\r\n]+\$ ?$')] +# The run of line breaks is matched as a whole and only from its start; see junos.py. +_prompt_re = [re.compile(r'(?#] $') +JUNOS_BEFORE = re.compile( + _JUNOS_PREFIX + r'(?:\r\n|[\r\n])+[\w\-\.]+@[\-\w+\.:]++(?:[~/][\w\-\./]*+ ?)?[%>#] $' +) +MRV_BEFORE = re.compile(r'(?:\r\n|[\r\n])+[^$\r\n]+\$ ?$') + + +def random_texts(alphabet, count, seed): + """Returns random texts that start with a line break, of up to 16 items of the alphabet.""" + rng = random.Random(seed) + for _ in range(count): + yield '\r\n' + ''.join(rng.choice(alphabet) for _ in range(rng.randint(0, 16))) + + class PromptRegexTest(unittest.TestCase): """The prompt regexes of the drivers.""" @@ -69,7 +89,7 @@ def testJunosPrompts(self): '{primary:node0}[edit interfaces]\r\npheller@fw1# ', 'root@router% ', 'root@router:RE:0% ', - 'root@testhost:~ # ', # csh, the shell of root (upstream pull request #211) + 'root@testhost:~ # ', # csh, the shell of root 'root@router:/var/tmp # ', 'root@router:~# ', ] @@ -83,15 +103,41 @@ def testJunosPrompts(self): ): self.assertIsNone(regex.search('output\r\n' + text), text) - # Everything that the pattern of 2.x matched still matches, at the same position. - old = re.compile(regex.pattern.replace(r'++(?:[~/][\w\-\./]*+ ?)?', '+')) - self.assertNotEqual(old.pattern, regex.pattern) + # The pattern matches everything that JUNOS_2X matches, at the same position. alphabet = list('ab_1.-@:~/ #>%{}[]') + ['\r\n'] - rng = random.Random(20261110) - for _ in range(20000): - text = '\r\n' + ''.join(rng.choice(alphabet) for _ in range(rng.randint(0, 16))) - if old.search(text): - self.assertEqual(span(old, text), span(regex, text), repr(text)) + for text in random_texts(alphabet, 20000, 20261110): + if JUNOS_2X.search(text): + self.assertEqual(span(JUNOS_2X, text), span(regex, text), repr(text)) + + def testRunsOfLineBreaksAsBefore(self): + # The patterns match each run of line breaks as a whole, and the same texts at the + # same positions as JUNOS_BEFORE and MRV_BEFORE, which backtrack on the runs. + tokens = [ + '\r\n', + '\r', + '\n', + ' ', + '$', + '#', + '>', + '%', + 'a', + 'b1', + '.', + '-', + '@', + ':', + '~', + '/', + ] + tags = ['{master}', '{backup:1}', '{primary:node0}', '[edit]', '[edit interfaces]'] + for name, before, alphabet in ( + ('junos', JUNOS_BEFORE, tokens + tags), + ('mrv', MRV_BEFORE, tokens), + ): + regex = driver_map[name].prompt_re[0] + for text in random_texts(alphabet, 30000, 20261111): + self.assertEqual(span(before, text), span(regex, text), f'{name}: {text!r}') def testAllRegexesNeedLinearTime(self): # Lines that are no prompt but contain long runs of the characters that @@ -110,6 +156,8 @@ def testAllRegexesNeedLinearTime(self): '\x1b[0m' * 30 + 'a' * 60 + '!', ] texts = ['\r\n' + line for line in lines] + [' ' * 120 + '!', 'x\r\n' * 1000] + # Runs of line breaks as long as the window in which prompts are searched. + texts += ['\r\n' * 512 + '!', '\n' * 1024 + '!', '\r\n' * 400 + '{master}\r\n!'] for name, driver in driver_map.items(): for attr, value in vars(driver).items(): if not isinstance(value, list): From 2ea80e0bf01af5d5e3bb0375cc7839849032c6b1 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 00:30:41 +0100 Subject: [PATCH 143/250] Keep passwords out of data_sent_event The login sends passwords within _secret(), so the debug log shows , but _send_cb() passed them unchanged to data_sent_event and, if the host does not echo, to the output and the logfile. --- CHANGES.md | 3 +++ src/Exscript/protocols/protocol.py | 6 +++++- tests/Exscript/protocols/DummyTest.py | 21 +++++++++++++++++++++ 3 files changed, 29 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 003fadca..9baeb9e6 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -84,6 +84,9 @@ marked with **BREAKING**; a migration guide will accompany the release. login are logged as ``. Before, debug level 4 logged them in clear text. Debug levels above 5 no longer turn on paramiko's packet dumps, which showed the password of an SSH login in clear text. + Listeners of `data_sent_event`, and the output and the logfile of a protocol whose + host does not echo, get `` instead of the password and the S/Key response, + too; before, `data_sent_event` passed them in clear text. - **BREAKING - `InvalidCommandException` Message:** The message contains only the line that matched an error pattern, for example `Error in the response from the device: % Invalid input detected`. Before, it contained the full response, which diff --git a/src/Exscript/protocols/protocol.py b/src/Exscript/protocols/protocol.py index 5bd52fd2..681a9560 100644 --- a/src/Exscript/protocols/protocol.py +++ b/src/Exscript/protocols/protocol.py @@ -597,8 +597,12 @@ def _receive_cb(self, data: str) -> None: def _send_cb(self, data: str) -> None: """ Handles data that was sent to the host: writes it to stdout and to the - logfile if the host does not echo it, and emits data_sent_event. + logfile if the host does not echo it, and emits data_sent_event. Data + sent as a secret, such as the password of the login, appears there as + "". """ + if self._sending_secret: + data = '' if not self._receives_echo: # The sent commands are either logged via the _receive_cb() when # they are echoed back, or they are logged here explicitly. diff --git a/tests/Exscript/protocols/DummyTest.py b/tests/Exscript/protocols/DummyTest.py index ce9b5f6b..1428161f 100644 --- a/tests/Exscript/protocols/DummyTest.py +++ b/tests/Exscript/protocols/DummyTest.py @@ -1,4 +1,6 @@ +import os import re +import tempfile import unittest from Exscript import Account @@ -136,6 +138,25 @@ def testDebugLogRedactsPassword(self): self.assertIn('Sending: ', '\n'.join(logs.output)) self.assertEqual([line for line in logs.output if self.PASSWORD in line], []) + def testSentDataRedactsPassword(self): + # The listeners of data_sent_event, and the output and the logfile of a protocol + # whose host does not echo, see instead of the password. + for echo in True, False: + with self.subTest(echo=echo), tempfile.TemporaryDirectory() as tmpdir: + logfile = os.path.join(tmpdir, 'session.log') + conn = Dummy(device=VirtualDevice('r1', echo=False, strict=False), logfile=logfile) + conn._receives_echo = echo + sent = [] + conn.data_sent_event.connect(sent.append) + conn.connect('r1') + conn.login(Account('admin', self.PASSWORD, needs_lock=False)) + conn.close(force=True) + self.assertIn('admin\r', sent) + self.assertIn('', sent) + with open(logfile, encoding='utf-8') as fp: + output = ''.join(sent) + conn.stdout.getvalue() + fp.read() + self.assertNotIn(self.PASSWORD, output) + def testErrorMessageShowsOnlyTheErrorLine(self): device = VirtualDevice('r1', echo=False, strict=False) device.add_command('show run', 'enable secret PRIVATE\r\n% Invalid input detected') From b94e418874ede2614ba2387aa04d0ce93188c5c7 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 00:31:01 +0100 Subject: [PATCH 144/250] Explain why the S/Key one-time password uses MD4 MD4 on a password looks like a weak password hash, but S/Key (RFC 1760) defines the one-time password with it, and the result is not stored. --- src/Exscript/util/crypt.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Exscript/util/crypt.py b/src/Exscript/util/crypt.py index 08188b1a..743207ec 100644 --- a/src/Exscript/util/crypt.py +++ b/src/Exscript/util/crypt.py @@ -2173,6 +2173,8 @@ def otp(password: str, seed: str, sequence: int) -> str: seed_bytes = seed.encode('utf-8') password_bytes = password.encode('utf-8') + # S/Key (RFC 1760) defines the one-time password with MD4, and the device checks it, + # so no other hash can be used. The result is a one-time password, not a stored hash. # Discard the first keys thehash = MD4.new(seed_bytes + password_bytes).digest() thehash = _fold_md4_or_md5(thehash) From f9547eb10596288b4be9d467bb5c1404274eb466 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 00:46:37 +0100 Subject: [PATCH 145/250] Make two network tests independent of WSL's consomme networking It accepts TCP connections to closed ports on 127.0.0.1 and loses UDP datagrams sent there; the library is not affected. Connect to a port that is bound but not listening, and send the syslog datagrams to ::1 where possible. --- tests/Exscript/protocols/telnetlibTest.py | 11 +++++++---- tests/Exscript/util/syslogTest.py | 18 ++++++++++++------ 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/tests/Exscript/protocols/telnetlibTest.py b/tests/Exscript/protocols/telnetlibTest.py index a1c004c0..b3b19b84 100644 --- a/tests/Exscript/protocols/telnetlibTest.py +++ b/tests/Exscript/protocols/telnetlibTest.py @@ -168,10 +168,13 @@ def testOpen(self): tn = Telnet(address, listener.getsockname()[1], connect_timeout=5) tn.close() self.assertTrue(tn.eof) - with socket.create_server(('127.0.0.1', 0)) as listener: - port = listener.getsockname()[1] - with self.assertRaisesRegex(OSError, f'=> telnet://127.0.0.1:{port}'): - Telnet('127.0.0.1', port, connect_timeout=5) + # Bound but not listening: refused everywhere, also by WSL's consomme networking, which accepts + # connections to closed ports and closes them right away. + with socket.socket() as bound: + bound.bind(('127.0.0.1', 0)) + port = bound.getsockname()[1] + with self.assertRaisesRegex(OSError, f'=> telnet://127.0.0.1:{port}'): + Telnet('127.0.0.1', port, connect_timeout=5) def testOpenErrorNamesTheUrl(self): error = ConnectionRefusedError(111, 'Connection refused') diff --git a/tests/Exscript/util/syslogTest.py b/tests/Exscript/util/syslogTest.py index b8cf57e7..190262f2 100644 --- a/tests/Exscript/util/syslogTest.py +++ b/tests/Exscript/util/syslogTest.py @@ -11,20 +11,26 @@ class syslogTest(unittest.TestCase): CORRELATE = Exscript.util.syslog def setUp(self): - self.server = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - self.server.bind(('127.0.0.1', 0)) + # IPv6 first: WSL's consomme networking loses datagrams sent with sendto() to 127.0.0.1. + for family, self.host in (socket.AF_INET6, '::1'), (socket.AF_INET, '127.0.0.1'): + self.server = socket.socket(family, socket.SOCK_DGRAM) + self.addCleanup(self.server.close) + try: + self.server.bind((self.host, 0)) + break + except OSError: # no IPv6 on this host + continue self.server.settimeout(5) - self.addCleanup(self.server.close) self.port = self.server.getsockname()[1] def testNetlog(self): - netlog('hellö', source='router1', host='127.0.0.1', port=self.port) + netlog('hellö', source='router1', host=self.host, port=self.port) self.assertEqual(self.server.recv(1024), '<15>router1: hellö'.encode()) # user.debug - netlog('x', source='s', host='127.0.0.1', port=self.port, priority=3, facility=4 << 3) + netlog('x', source='s', host=self.host, port=self.port, priority=3, facility=4 << 3) self.assertEqual(self.server.recv(1024), b'<35>s: x') # auth.err def testDefaultSource(self): - netlog('x', host='127.0.0.1', port=self.port) + netlog('x', host=self.host, port=self.port) expected = '<15>%s[%d]: x' % (sys.argv[0], os.getpid()) self.assertEqual(self.server.recv(1024), expected.encode()) From e8a2834bc63518a2c18cc036b37592334151b830 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 00:48:11 +0100 Subject: [PATCH 146/250] Add docstrings to the util and workqueue packages --- src/Exscript/util/__init__.py | 6 ++++++ src/Exscript/workqueue/__init__.py | 5 +++++ 2 files changed, 11 insertions(+) diff --git a/src/Exscript/util/__init__.py b/src/Exscript/util/__init__.py index fae7955d..210db315 100644 --- a/src/Exscript/util/__init__.py +++ b/src/Exscript/util/__init__.py @@ -20,4 +20,10 @@ # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" +Helpers for scripts and for the other packages: starting jobs on hosts, decorators for jobs, and utilities +for files, IP addresses, URLs, regular expressions, mail, logs and syslog, events, templates, terminals and +interactive input. +""" + __all__: list[str] = [] diff --git a/src/Exscript/workqueue/__init__.py b/src/Exscript/workqueue/__init__.py index b06bb5b9..dade9f1e 100644 --- a/src/Exscript/workqueue/__init__.py +++ b/src/Exscript/workqueue/__init__.py @@ -20,6 +20,11 @@ # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" +The thread pool that runs the jobs of :class:`Exscript.Queue`: a pipeline of waiting and running jobs, and +the tasks that group them. +""" + from .pipeline import Pipeline from .task import Task from .workqueue import WorkQueue From 438aaedc6dd1f013ab8a67d3469eec679f6e366a Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 00:52:26 +0100 Subject: [PATCH 147/250] Check the type coverage in CI Nothing kept new code from falling back to Any. tools/type_coverage.py runs mypy with --any-exprs-report and fails if less than 95 percent of the expressions have a known type; the lint job runs it instead of mypy. --- .github/workflows/ci.yml | 4 ++- CHANGES.md | 6 ++-- CONTRIBUTING.md | 10 ++++++- tools/type_coverage.py | 59 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 75 insertions(+), 4 deletions(-) create mode 100755 tools/type_coverage.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 925dc01f..74d9341d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,7 +24,9 @@ jobs: fetch-depth: 0 # hatch-vcs derives the version from tags - uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3 - run: uvx pre-commit run --all-files --show-diff-on-failure - - run: uv run --frozen mypy + - name: Type check and type coverage (mypy; fails below the minimum) + shell: bash # with pipefail; the default shell lets tee hide a failure + run: uv run --frozen python tools/type_coverage.py --check | tee -a "$GITHUB_STEP_SUMMARY" test: name: Test ${{ matrix.python }} on ${{ matrix.os }} diff --git a/CHANGES.md b/CHANGES.md index 9baeb9e6..c12f895e 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -20,8 +20,10 @@ marked with **BREAKING**; a migration guide will accompany the release. versions before 3.12 are no longer supported. - **Type Annotations:** Exscript is annotated and ships the `py.typed` marker (PEP 561), so type checkers such as mypy and pyright check the code that uses it. All modules - pass `mypy --strict`, except the queue (`Exscript.queue`, `Exscript.workqueue`) and - `Exscript.util.sigint`, which are annotated only in part. + pass `mypy --strict`, except `Exscript.util.sigint`, which is annotated only in part. + mypy knows the type of 95.7 % of the expressions; the rest is `Any` on purpose + (callbacks, options that are passed on, template variables) or comes from ncclient, + which is not annotated. CI checks that this share does not drop. - **Dependencies:** `future` and `configparser`, backports for Python 2, are no longer needed. paramiko 3.4 to 5.x is supported (before: 1.17 to 4.x). New: ncclient and lxml 5.0 or later, for NETCONF. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7977b6bb..2c07908b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -36,6 +36,14 @@ The tests need no devices and no network: they run against local test servers an recorded sessions. To run them with another Python version, use e.g. `uv run --python 3.15 pytest`. +CI also measures three kinds of coverage, each with a minimum that only goes up: + +- tests: `uv run pytest --cov=Exscript` (lines and branches; the minimum is + `COV_FAIL_UNDER` in `.github/workflows/ci.yml`), +- docstrings: interrogate in pre-commit (`fail-under` in `pyproject.toml`), +- types: `uv run python tools/type_coverage.py --check`, the share of expressions whose + type mypy knows (`MIN_PERCENT` in the tool). + ## Changes - A fix comes with a test that fails without it. @@ -46,7 +54,7 @@ recorded sessions. To run them with another Python version, use e.g. them, and pre-commit fails if the share of documented functions drops. - Annotate new code with types. mypy checks every module strictly, except those that are still in the list of `[[tool.mypy.overrides]]` in `pyproject.toml`; remove a module from - that list when you annotate it. + that list when you annotate it. Use `Any` only where a type is open on purpose. - The public API and the template language stay compatible. Please open an issue before you work on a change that breaks them. diff --git a/tools/type_coverage.py b/tools/type_coverage.py new file mode 100755 index 00000000..0c2d3160 --- /dev/null +++ b/tools/type_coverage.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 +""" +Type coverage: runs mypy with the settings in pyproject.toml and prints the +share of expressions whose type mypy knows, i.e. is not ``Any``, in Markdown. + +Every function is annotated (mypy checks strictly). ``Any`` remains where a +type is open on purpose (callbacks, options that are passed on, template +variables) and where an untyped library returns it (ncclient). + +With --check, it fails if mypy reports an error or if the coverage is below +MIN_PERCENT, so the coverage only goes up; raise MIN_PERCENT when a change +improves it. Run from the repository root:: + + uv run python tools/type_coverage.py [--check] +""" + +import argparse +import os +import sys +import tempfile + +from mypy import api + +MIN_PERCENT = 95.0 + + +def main() -> int: + """Run mypy, print the coverage, and return the exit status.""" + parser = argparse.ArgumentParser(description=__doc__.split('\n\n')[0].strip()) + parser.add_argument('--check', action='store_true', help=f'fail below {MIN_PERCENT} %%') + args = parser.parse_args() + with tempfile.TemporaryDirectory() as report_dir: + stdout, stderr, status = api.run(['--any-exprs-report', report_dir]) + sys.stderr.write(stderr) + if status: + sys.stdout.write(stdout) + return status + with open(os.path.join(report_dir, 'any-exprs.txt'), encoding='utf-8') as file: + rows = [line.split() for line in file] + modules = { + row[0]: (int(row[1]), int(row[2])) for row in rows if len(row) == 4 and row[1].isdigit() + } + anys, exprs = modules.pop('Total') + percent = 100.0 * (exprs - anys) / exprs + print( + f'Type coverage: **{percent:.2f} %** of {exprs} expressions have a known type (minimum {MIN_PERCENT} %).' + ) + low = sorted( + (100.0 * (e - a) / e, name) for name, (a, e) in modules.items() if e and a / e > 0.1 + ) + if low: + print('\nBelow 90 %:\n') + for module_percent, name in low: + print(f'- {name}: {module_percent:.1f} %') + return 1 if args.check and percent < MIN_PERCENT else 0 + + +if __name__ == '__main__': + sys.exit(main()) From 8bd89b5274140244f154c851a13bf558e63b4a2e Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 00:26:09 +0100 Subject: [PATCH 148/250] Keep the order of sleeping items at the head of the pipeline next() and try_next() put the sleeping items that they skipped back with deque.extendleft(), which reversed their order. Put them back in reverse. --- CHANGES.md | 4 ++++ src/Exscript/workqueue/pipeline.py | 4 ++-- tests/Exscript/workqueue/PipelineTest.py | 17 +++++++++++++++++ 3 files changed, 23 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index c12f895e..7fe4a8b9 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -619,6 +619,10 @@ The names below refer to functions callable from templates, not protocol methods `Exscript.account.LoggerProxy`, `Exscript.workqueue.job.Process` and `Queue._create_pipe()`. The queue, accounts and account pools use the locks of `threading` instead of those of `multiprocessing`. +- **Order of Sleeping Items:** `next()` and `try_next()` of + `Exscript.workqueue.Pipeline` leave the sleeping items at the start of the queue in + their order, so they come next in that order after `wake()`. Before, each call + reversed it. ## Demos diff --git a/src/Exscript/workqueue/pipeline.py b/src/Exscript/workqueue/pipeline.py index fa968c68..0b139052 100644 --- a/src/Exscript/workqueue/pipeline.py +++ b/src/Exscript/workqueue/pipeline.py @@ -346,8 +346,8 @@ def _get_next(self, pop: bool = True) -> Any: except IndexError: next = None - # Re-insert sleeping items. - self.queue.extendleft(sleeping) + # Re-insert sleeping items, in their order. + self.queue.extendleft(reversed(sleeping)) return next def try_next(self) -> Any: diff --git a/tests/Exscript/workqueue/PipelineTest.py b/tests/Exscript/workqueue/PipelineTest.py index 6515777a..1fc66205 100644 --- a/tests/Exscript/workqueue/PipelineTest.py +++ b/tests/Exscript/workqueue/PipelineTest.py @@ -196,6 +196,23 @@ def testSleep(self): def testWake(self): self.testSleep() + def testSleepingItemsKeepTheirOrder(self): + # next() and try_next() skip the sleeping items at the start of the + # queue, and leave them in their order. + for get in (next, Pipeline.try_next): + with self.subTest(get=get): + pipeline = Pipeline(max_working=3) + item1, item2, item3 = object(), object(), object() + for item in (item1, item2, item3): + pipeline.append(item) + pipeline.sleep(item1) + pipeline.sleep(item2) + self.assertIs(get(pipeline), item3) + pipeline.wake(item1) + pipeline.wake(item2) + self.assertIs(next(pipeline), item1) + self.assertIs(next(pipeline), item2) + def testWaitForId(self): item1 = object() item2 = object() From 13b13fa70a7399491775938fbcc281b88dc32136 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 00:28:20 +0100 Subject: [PATCH 149/250] Add new jobs of priority_enqueue_or_raise() at the front It appended a new job at the end and ignored force_start for it, although its docstring promises priority. force_start now starts the job even if max_threads jobs are in progress. --- CHANGES.md | 5 +++++ src/Exscript/workqueue/mainloop.py | 11 ++++++----- tests/Exscript/QueueTest.py | 11 +++++++++++ tests/Exscript/workqueue/WorkQueueTest.py | 19 +++++++++++++++++++ 4 files changed, 41 insertions(+), 5 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 7fe4a8b9..a3b76d33 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -623,6 +623,11 @@ The names below refer to functions callable from templates, not protocol methods `Exscript.workqueue.Pipeline` leave the sleeping items at the start of the queue in their order, so they come next in that order after `wake()`. Before, each call reversed it. +- **`priority_run_or_raise()` Puts New Hosts First:** `Queue.priority_run_or_raise()` + adds a host that is not queued yet at the front of the queue, like `priority_run()`. + `WorkQueue.priority_enqueue_or_raise()` adds a new job at the front, and with + `force_start=True` starts it at once. Before, both added it at the end, and + `force_start` applied only to a job that was queued already. ## Demos diff --git a/src/Exscript/workqueue/mainloop.py b/src/Exscript/workqueue/mainloop.py index ce47f470..5cbe1820 100644 --- a/src/Exscript/workqueue/mainloop.py +++ b/src/Exscript/workqueue/mainloop.py @@ -143,11 +143,12 @@ def priority_enqueue_or_raise( data: Any, ) -> str | None: """ - Like enqueue_or_ignore(), but if a job with the given name is in the - pipeline already, moves that job to the start of the pipeline, see - Pipeline.prioritize(). + Like priority_enqueue(), but if a job with the given name is in the + pipeline already, moves that job to the start of the pipeline + instead, see Pipeline.prioritize(). - :param force_start: Whether to force the existing job. + :param force_start: Whether to start the new or the existing job + even if the maximum number of jobs is in progress already. :return: The id of the new job, or None. """ @@ -155,7 +156,7 @@ def conditional_append(queue: 'Pipeline') -> str | None: job = queue.get_from_name(name) if job is None: job = Job(function, name, times, data) - job.id = queue.append(job, name) + job.id = queue.appendleft(job, name, force=force_start) return job.id queue.prioritize(job, force=force_start) return None diff --git a/tests/Exscript/QueueTest.py b/tests/Exscript/QueueTest.py index 0b7d3ef9..96a848a5 100644 --- a/tests/Exscript/QueueTest.py +++ b/tests/Exscript/QueueTest.py @@ -363,6 +363,17 @@ def testPriorityRunOrRaise(self): self.queue.destroy() self.assertEqual(data.value, 3) + def testPriorityRunOrRaiseRunsANewHostFirst(self): + order = [] + self.queue.workqueue.pause() + self.queue.enqueue(lambda job: order.append('enqueued')) + self.queue.priority_run_or_raise( + 'dummy://dummy1', lambda job, host, conn: order.append(host.get_name()) + ) + self.queue.workqueue.unpause() + self.queue.join() + self.assertEqual(order, ['dummy1', 'enqueued']) + def testForceRun(self): data = Value('i', 0) hosts = ['dummy://dummy1', 'dummy://dummy2'] diff --git a/tests/Exscript/workqueue/WorkQueueTest.py b/tests/Exscript/workqueue/WorkQueueTest.py index a9e3b579..62fde797 100644 --- a/tests/Exscript/workqueue/WorkQueueTest.py +++ b/tests/Exscript/workqueue/WorkQueueTest.py @@ -1,4 +1,5 @@ import random +import threading import time import unittest from multiprocessing import Lock, Value @@ -115,6 +116,24 @@ def testPriorityEnqueueOrRaise(self): self.assertEqual(2, self.wq.get_length()) self.assertEqual(id, None) + def testPriorityEnqueueOrRaiseAddsANewJobAtTheFront(self): + self.addCleanup(self.wq.destroy) + order = [] + self.wq.pause() + self.wq.enqueue(lambda job: order.append('first')) + self.wq.priority_enqueue_or_raise(lambda job: order.append('second'), 'second') + self.wq.unpause() + self.wq.wait_until_done() + self.assertEqual(order, ['second', 'first']) + + def testPriorityEnqueueOrRaiseForcesANewJob(self): + # With force_start, the new job starts although max_threads jobs run. + self.addCleanup(self.wq.destroy) + self.wq.set_max_threads(0) + started = threading.Event() + self.wq.priority_enqueue_or_raise(lambda job: started.set(), 'forced', force_start=True) + self.assertTrue(started.wait(10)) + def testPause(self): pass # See testEnqueue() From 6a3d57b6d788f5e6d9e8c30da8b192b8d6cc7e3e Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 00:30:01 +0100 Subject: [PATCH 150/250] Leave the pipeline unchanged when it rejects a duplicate name append() and appendleft() queued the item before they found that its name was taken, so the rejected item stayed in the queue and was run. Check the name before anything changes. --- CHANGES.md | 5 +++++ src/Exscript/workqueue/pipeline.py | 18 +++++++++++------- tests/Exscript/QueueTest.py | 14 ++++++++++++++ tests/Exscript/workqueue/PipelineTest.py | 22 ++++++++++++++++++++++ 4 files changed, 52 insertions(+), 7 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index a3b76d33..c237f4a8 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -628,6 +628,11 @@ The names below refer to functions callable from templates, not protocol methods `WorkQueue.priority_enqueue_or_raise()` adds a new job at the front, and with `force_start=True` starts it at once. Before, both added it at the end, and `force_start` applied only to a job that was queued already. +- **Rejected Duplicates Are Not Queued:** `Pipeline.append()` and `appendleft()` still + raise an `AttributeError` for an item with the name of a queued item, but leave the + pipeline unchanged. Before, the item stayed queued: e.g. `Queue.priority_run()` on a + host that `run_or_ignore()` had queued raised the error, and yet ran the host a second + time, in a job with the id `None`. ## Demos diff --git a/src/Exscript/workqueue/pipeline.py b/src/Exscript/workqueue/pipeline.py index 0b139052..f9d51d2d 100644 --- a/src/Exscript/workqueue/pipeline.py +++ b/src/Exscript/workqueue/pipeline.py @@ -73,19 +73,19 @@ def __contains__(self, item: object) -> bool: def _register_item(self, name: str | None, item: Hashable) -> str: """ Registers the given item under a new id, and under the given name - unless it is None. Raises an AttributeError if an item with the name - is registered already. + unless it is None. Raises an AttributeError, and registers nothing, + if an item with the name is registered already. :return: The id of the item. """ + if name in self.name2id: + msg = 'an item named %s is already queued' % repr(name) + raise AttributeError(msg) uuid = uuid4().hex self.id2item[uuid] = item self.item2id[item] = uuid if name is None: return uuid - if name in self.name2id: - msg = 'an item named %s is already queued' % repr(name) - raise AttributeError(msg) self.name2id[name] = uuid self.id2name[uuid] = name return uuid @@ -139,11 +139,13 @@ def append(self, item: Hashable, name: str | None = None) -> str: Adds the given item to the end of the pipeline. :param name: A unique name for the item, see get_from_name(). + :raises AttributeError: If an item with the name is queued already; + the pipeline is unchanged then. :return: The id of the item. """ with self.condition: - self.queue.append(item) uuid = self._register_item(name, item) + self.queue.append(item) self.condition.notify_all() return uuid @@ -155,14 +157,16 @@ def appendleft(self, item: Hashable, name: str | None = None, force: bool = Fals :param name: A unique name for the item, see get_from_name(). :param force: Whether the item is forced. + :raises AttributeError: If an item with the name is queued already; + the pipeline is unchanged then. :return: The id of the item. """ with self.condition: + uuid = self._register_item(name, item) if force: self.force.append(item) else: self.queue.appendleft(item) - uuid = self._register_item(name, item) self.condition.notify_all() return uuid diff --git a/tests/Exscript/QueueTest.py b/tests/Exscript/QueueTest.py index 96a848a5..2d93d2b8 100644 --- a/tests/Exscript/QueueTest.py +++ b/tests/Exscript/QueueTest.py @@ -374,6 +374,20 @@ def testPriorityRunOrRaiseRunsANewHostFirst(self): self.queue.join() self.assertEqual(order, ['dummy1', 'enqueued']) + def testPriorityRunRejectsAQueuedHost(self): + names = [] + + def record(job, host, conn): + names.append(host.get_name()) + + self.queue.workqueue.pause() + self.queue.run_or_ignore('dummy://dummy1', record) + with self.assertRaisesRegex(AttributeError, "'dummy1'"): + self.queue.priority_run('dummy://dummy1', record) + self.queue.workqueue.unpause() + self.queue.join() + self.assertEqual(names, ['dummy1']) + def testForceRun(self): data = Value('i', 0) hosts = ['dummy://dummy1', 'dummy://dummy2'] diff --git a/tests/Exscript/workqueue/PipelineTest.py b/tests/Exscript/workqueue/PipelineTest.py index 1fc66205..eee9832a 100644 --- a/tests/Exscript/workqueue/PipelineTest.py +++ b/tests/Exscript/workqueue/PipelineTest.py @@ -1,5 +1,6 @@ import time import unittest +from functools import partial from multiprocessing import Value from threading import Thread @@ -67,6 +68,27 @@ def testTaskDone(self): def testAppend(self): self.testContains() + def testAppendRejectsAQueuedName(self): + # An item with the name of a queued item is rejected, and leaves the + # pipeline unchanged. + item1, item2 = object(), object() + self.pipeline.append(item1, 'foo') + appends = ( + self.pipeline.append, + self.pipeline.appendleft, + partial(self.pipeline.appendleft, force=True), + ) + for append in appends: + with self.subTest(append=append): + with self.assertRaisesRegex(AttributeError, "'foo'"): + append(item2, 'foo') + self.assertNotIn(item2, self.pipeline) + self.assertEqual(len(self.pipeline), 1) + self.assertIs(self.pipeline.get_from_name('foo'), item1) + self.assertIs(next(self.pipeline), item1) + self.pipeline.task_done(item1) + self.assertIsNone(self.pipeline.try_next()) + def testAppendleft(self): item1 = object() item2 = object() From c3b549f5ddf79e79696e6de31a2a3ec412986b79 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 00:32:22 +0100 Subject: [PATCH 151/250] Give a job its id before the main loop can start it enqueue() and priority_enqueue() set the id only after the job was in the pipeline, so a job that started at once got the id None, and a Task never counted it as completed. Set the id under the lock of the pipeline. --- CHANGES.md | 3 ++ src/Exscript/workqueue/mainloop.py | 22 ++++++++--- tests/Exscript/workqueue/WorkQueueTest.py | 47 +++++++++++++++++++++++ 3 files changed, 66 insertions(+), 6 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index c237f4a8..716a01e8 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -633,6 +633,9 @@ The names below refer to functions callable from templates, not protocol methods pipeline unchanged. Before, the item stayed queued: e.g. `Queue.priority_run()` on a host that `run_or_ignore()` had queued raised the error, and yet ran the host a second time, in a job with the id `None`. +- **Jobs Have Their Id When They Start:** `WorkQueue.enqueue()` and `priority_enqueue()` + give the job its id before the main loop can start it. Before, a job that started at + once could run with the id `None`, and its `Task` then never counted it as completed. ## Demos diff --git a/src/Exscript/workqueue/mainloop.py b/src/Exscript/workqueue/mainloop.py index 5cbe1820..307a017b 100644 --- a/src/Exscript/workqueue/mainloop.py +++ b/src/Exscript/workqueue/mainloop.py @@ -89,9 +89,14 @@ def enqueue( :return: The id of the job. """ - job = Job(function, name, times, data) - job.id = self.collection.append(job) - return job.id + + # Under the lock, so the job has its id before the main loop takes it. + def append(queue: 'Pipeline') -> str: + job = Job(function, name, times, data) + job.id = queue.append(job) + return job.id + + return self.collection.with_lock(append) # Any: the data of the job, see job.Thread.__init__(). def enqueue_or_ignore( @@ -129,9 +134,14 @@ def priority_enqueue( number of jobs is in progress already. :return: The id of the job. """ - job = Job(function, name, times, data) - job.id = self.collection.appendleft(job, name, force=force_start) - return job.id + + # Under the lock, so the job has its id before the main loop takes it. + def appendleft(queue: 'Pipeline') -> str: + job = Job(function, name, times, data) + job.id = queue.appendleft(job, name, force=force_start) + return job.id + + return self.collection.with_lock(appendleft) # Any: the data of the job, see job.Thread.__init__(). def priority_enqueue_or_raise( diff --git a/tests/Exscript/workqueue/WorkQueueTest.py b/tests/Exscript/workqueue/WorkQueueTest.py index 62fde797..69995fa7 100644 --- a/tests/Exscript/workqueue/WorkQueueTest.py +++ b/tests/Exscript/workqueue/WorkQueueTest.py @@ -3,6 +3,7 @@ import time import unittest from multiprocessing import Lock, Value +from unittest import mock from Exscript.workqueue import WorkQueue @@ -19,6 +20,18 @@ def burn_time(job): time.sleep(random.random()) +def holds_lock(pipeline): + """ + Returns whether the current thread holds the lock of the given pipeline, + which keeps the main loop from taking a job. + """ + try: + pipeline.condition.notify(0) # raises unless the lock is held + except RuntimeError: + return False + return True + + nop = lambda x: None @@ -134,6 +147,40 @@ def testPriorityEnqueueOrRaiseForcesANewJob(self): self.wq.priority_enqueue_or_raise(lambda job: started.set(), 'forced', force_start=True) self.assertTrue(started.wait(10)) + def assertJobHasItsIdWhenItStarts(self, enqueue, add): + """ + Enqueues a job with the given method of the work queue, which adds it + to the pipeline with the given method of the pipeline. The job runs + at the earliest moment the main loop can take it: before the method + of the pipeline returns, unless the lock of the pipeline is held. + """ + ids = [] + pipeline = self.wq.collection + add_item = getattr(pipeline, add) + + def add_and_run(*args, **kwargs): + item_id = add_item(*args, **kwargs) + if not holds_lock(pipeline): + pipeline.wait_for_id(item_id) + return item_id + + with mock.patch.object(pipeline, add, add_and_run): + job_id = getattr(self.wq, enqueue)(lambda job: ids.append(job.id)) + self.wq.wait_for(job_id) + self.assertEqual(ids, [job_id]) + + def testJobHasItsIdWhenItStarts(self): + self.addCleanup(self.wq.destroy) + methods = ( + ('enqueue', 'append'), + ('enqueue_or_ignore', 'append'), + ('priority_enqueue', 'appendleft'), + ('priority_enqueue_or_raise', 'appendleft'), + ) + for enqueue, add in methods: + with self.subTest(enqueue=enqueue): + self.assertJobHasItsIdWhenItStarts(enqueue, add) + def testPause(self): pass # See testEnqueue() From 5c48a6bc7a573c5abc4efd3a4323e0c87b7f774d Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 00:34:37 +0100 Subject: [PATCH 152/250] Let tasks of the queue know each job before it can start The queue added the id of a job to the task after the job was queued, so a job that ended before was never counted, and run() could return None. Enqueue the job and add its id under the lock of the pipeline. --- CHANGES.md | 6 ++++ src/Exscript/queue.py | 28 ++++++++++++----- tests/Exscript/QueueTest.py | 61 +++++++++++++++++++++++++++++++++++++ 3 files changed, 88 insertions(+), 7 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 716a01e8..1ace1efa 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -636,6 +636,12 @@ The names below refer to functions callable from templates, not protocol methods - **Jobs Have Their Id When They Start:** `WorkQueue.enqueue()` and `priority_enqueue()` give the job its id before the main loop can start it. Before, a job that started at once could run with the id `None`, and its `Task` then never counted it as completed. +- **Tasks Count Jobs That End at Once:** `Queue.run()`, its variants and + `Queue.enqueue()` add the id of each job to the task before the job can start. Before, + the task never counted a job that ended before its id was added, so + `Task.is_completed()` stayed `False`, and `run()` returned `None` instead of the task + if the jobs ended before it returned. `run()` and its variants return `None` only if + they enqueued no job, as all hosts were queued already. ## Demos diff --git a/src/Exscript/queue.py b/src/Exscript/queue.py index 3f418623..ee4c2b7d 100644 --- a/src/Exscript/queue.py +++ b/src/Exscript/queue.py @@ -46,6 +46,7 @@ from .account import Account, AccountPool from .host import Host from .protocols import Protocol + from .workqueue import Pipeline from .workqueue.job import Job, Thread _logger = logging.getLogger(__name__) @@ -607,6 +608,23 @@ def reset(self) -> None: self._dbg(3, 'Queue was reset.') self._del_status_bar() + def _enqueue( + self, task: Task, queue_function: Callable[..., str | None], *args: Any, **kwargs: Any + ) -> None: + """ + Enqueues a job by calling the given function of the work queue with + the given arguments, and adds the id of the job to the given task, + unless the function ignored the job. Holds the lock of the pipeline + meanwhile, so the job cannot start before the task knows its id. + """ + + def enqueue(pipeline: 'Pipeline') -> None: + job_id = queue_function(*args, **kwargs) + if job_id is not None: + task.add_job_id(job_id) + + self.workqueue.collection.with_lock(enqueue) + def _run( self, hosts: 'str | Host | Iterable[str | Host]', @@ -630,14 +648,12 @@ def _run( for host in hosts: name = host.get_name() data = {'host': host} - job_id = queue_function(callback, name, *args, data=data) - if job_id is not None: - task.add_job_id(job_id) + self._enqueue(task, queue_function, callback, name, *args, data=data) if self.host_driver is not None: host.set_option('driver', self.host_driver) - if task.is_completed(): + if not task.job_ids: self._dbg(2, 'No jobs enqueued.') return None @@ -774,8 +790,6 @@ def enqueue( """ self.total += 1 task = Task(self.workqueue) - job_id = self.workqueue.enqueue(function, name, attempts) - if job_id is not None: - task.add_job_id(job_id) + self._enqueue(task, self.workqueue.enqueue, function, name, attempts) self._dbg(2, 'Function enqueued.') return task diff --git a/tests/Exscript/QueueTest.py b/tests/Exscript/QueueTest.py index 2d93d2b8..9ff63818 100644 --- a/tests/Exscript/QueueTest.py +++ b/tests/Exscript/QueueTest.py @@ -12,17 +12,41 @@ from functools import partial from multiprocessing import Value from tempfile import mkdtemp +from unittest import mock from Exscript import Account, AccountPool, FileLogger, Queue from Exscript.interpreter.exception import FailException from Exscript.protocols import Dummy, Protocol from Exscript.util.decorator import bind from Exscript.util.log import log_to +from Exscript.workqueue import Task logger = logging.getLogger('TestSuite.QueueTest') logger.addHandler(logging.NullHandler()) +def holds_lock(queue): + """ + Returns whether the current thread holds the lock of the pipeline of the + given queue, which keeps the main loop from taking a job. + """ + try: + queue.workqueue.collection.condition.notify(0) # raises unless the lock is held + except RuntimeError: + return False + return True + + +def end_now(queue, job_id): + """ + Waits until the job with the given id has ended, as if the main loop had + taken it at once: unless the current thread holds the lock of the + pipeline, which keeps the main loop from taking it. + """ + if not holds_lock(queue): + queue.workqueue.wait_for(job_id) + + def count_calls(job, data, **kwargs): assert hasattr(job, 'start') assert 'testarg' in kwargs @@ -388,6 +412,43 @@ def record(job, host, conn): self.queue.join() self.assertEqual(names, ['dummy1']) + def testTaskCountsJobsThatEndedAtOnce(self): + # Each job ends before its task learns its id, if it can start then. + add_job_id = Task.add_job_id + + def end_and_add(task, job_id): + end_now(self.queue, job_id) + add_job_id(task, job_id) + + with mock.patch.object(Task, 'add_job_id', end_and_add): + tasks = { + 'run': self.queue.run(['dummy://dummy1', 'dummy://dummy2'], do_nothing), + 'enqueue': self.queue.enqueue(lambda job: None), + } + for method, task in tasks.items(): + with self.subTest(method=method): + task.wait() + self.assertTrue(task.is_completed()) + + def testRunReturnsTheTaskOfJobsThatEndedAtOnce(self): + # Each job ends right after its task learned its id, if it can start + # then. + add_job_id = Task.add_job_id + + def add_and_end(task, job_id): + add_job_id(task, job_id) + end_now(self.queue, job_id) + + with mock.patch.object(Task, 'add_job_id', add_and_end): + task = self.queue.run('dummy://dummy1', do_nothing) + self.assertIsNotNone(task) + + # Only a run that enqueues no job returns None. + self.queue.workqueue.pause() + self.assertIsNotNone(self.queue.run_or_ignore('dummy://dummy2', do_nothing)) + self.assertIsNone(self.queue.run_or_ignore('dummy://dummy2', do_nothing)) + self.queue.workqueue.unpause() + def testForceRun(self): data = Value('i', 0) hosts = ['dummy://dummy1', 'dummy://dummy2'] From 185537fe81180da260bf9f08f774fe573d7a4a9a Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 00:36:08 +0100 Subject: [PATCH 153/250] Set the host driver of the queue before the job is enqueued A job that started at once could connect with a guessed driver instead of the one of Queue(host_driver=...). --- CHANGES.md | 3 +++ src/Exscript/queue.py | 5 ++--- tests/Exscript/QueueTest.py | 32 +++++++++++++++++++++++++++++++- 3 files changed, 36 insertions(+), 4 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 1ace1efa..4f39497d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -642,6 +642,9 @@ The names below refer to functions callable from templates, not protocol methods `Task.is_completed()` stayed `False`, and `run()` returned `None` instead of the task if the jobs ended before it returned. `run()` and its variants return `None` only if they enqueued no job, as all hosts were queued already. +- **Jobs Connect With `host_driver`:** The `host_driver` of `Queue` is set on each host + before its job is enqueued. Before, it was set after, so a job that started at once + could connect without it, with the driver that Exscript guessed. ## Demos diff --git a/src/Exscript/queue.py b/src/Exscript/queue.py index ee4c2b7d..02512cd2 100644 --- a/src/Exscript/queue.py +++ b/src/Exscript/queue.py @@ -646,13 +646,12 @@ def _run( callback = _prepare_connection(callback, self.account_manager) task = Task(self.workqueue) for host in hosts: + if self.host_driver is not None: + host.set_option('driver', self.host_driver) name = host.get_name() data = {'host': host} self._enqueue(task, queue_function, callback, name, *args, data=data) - if self.host_driver is not None: - host.set_option('driver', self.host_driver) - if not task.job_ids: self._dbg(2, 'No jobs enqueued.') return None diff --git a/tests/Exscript/QueueTest.py b/tests/Exscript/QueueTest.py index 9ff63818..97992e76 100644 --- a/tests/Exscript/QueueTest.py +++ b/tests/Exscript/QueueTest.py @@ -14,7 +14,7 @@ from tempfile import mkdtemp from unittest import mock -from Exscript import Account, AccountPool, FileLogger, Queue +from Exscript import Account, AccountPool, FileLogger, Host, Queue from Exscript.interpreter.exception import FailException from Exscript.protocols import Dummy, Protocol from Exscript.util.decorator import bind @@ -449,6 +449,36 @@ def add_and_end(task, job_id): self.assertIsNone(self.queue.run_or_ignore('dummy://dummy2', do_nothing)) self.queue.workqueue.unpause() + def testJobsConnectWithTheHostDriver(self): + # Each job ends before the driver is set on its host, if it can start + # then. + self.createQueue(verbose=-1, host_driver='ios') + drivers = [] + job_ids = [] + enqueue = self.queue.workqueue.enqueue + set_option = Host.set_option + + def enqueue_and_record(*args, **kwargs): + job_ids.append(enqueue(*args, **kwargs)) + return job_ids[-1] + + def end_and_set_option(host, name, value): + if name == 'driver': + for job_id in job_ids: + end_now(self.queue, job_id) + set_option(host, name, value) + + def record_driver(job, host, conn): + drivers.append(conn.get_driver().name) + + with ( + mock.patch.object(self.queue.workqueue, 'enqueue', enqueue_and_record), + mock.patch.object(Host, 'set_option', end_and_set_option), + ): + self.queue.run(['dummy://dummy1', 'dummy://dummy2'], record_driver) + self.queue.join() + self.assertEqual(drivers, ['ios', 'ios']) + def testForceRun(self): data = Value('i', 0) hosts = ['dummy://dummy1', 'dummy://dummy2'] From 4efaeaea63dc1078e92212272f44843756707fc9 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 00:38:07 +0100 Subject: [PATCH 154/250] Emit queue_empty_event when the work queue becomes empty The main loop emitted it only when it stopped. Emit it when the removal of an ended job leaves the pipeline empty; the check runs under the lock of the pipeline, so the event comes once. --- CHANGES.md | 3 +++ src/Exscript/workqueue/mainloop.py | 29 ++++++++++++++++------- src/Exscript/workqueue/workqueue.py | 6 +++++ tests/Exscript/workqueue/WorkQueueTest.py | 19 +++++++++++++++ 4 files changed, 48 insertions(+), 9 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 4f39497d..af4d70b6 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -645,6 +645,9 @@ The names below refer to functions callable from templates, not protocol methods - **Jobs Connect With `host_driver`:** The `host_driver` of `Queue` is set on each host before its job is enqueued. Before, it was set after, so a job that started at once could connect without it, with the driver that Exscript guessed. +- **`WorkQueue.queue_empty_event`:** Is emitted when the queue becomes empty, when the + last queued or running job has ended. Before, it was emitted only when the main loop + stopped, on `shutdown()` or `destroy()` of an empty queue. ## Demos diff --git a/src/Exscript/workqueue/mainloop.py b/src/Exscript/workqueue/mainloop.py index 307a017b..0b469657 100644 --- a/src/Exscript/workqueue/mainloop.py +++ b/src/Exscript/workqueue/mainloop.py @@ -49,7 +49,8 @@ class MainLoop(threading.Thread): job_init_event gets the Job before its first attempt. job_started_event, job_succeeded_event and job_aborted_event (after the last failed attempt) get the Thread of the attempt; job_error_event gets the Thread and the - exception information. queue_empty_event gets no arguments. + exception information. queue_empty_event gets no arguments; it is emitted + when the pipeline becomes empty, after the last job in it has ended. """ def __init__(self, collection: 'Pipeline', job_cls: 'type[Thread]') -> None: @@ -186,6 +187,17 @@ def get_queue_length(self) -> int: """ return len(self.collection) + @staticmethod + def _remove(queue: 'Pipeline', job: Job) -> bool: + """ + Removes the given job, which has ended, from the given pipeline. + + :return: Whether the pipeline had the job and is empty now. + """ + removed = job in queue + queue.task_done(job) + return removed and len(queue) == 0 + def _on_job_completed( self, job: Job, @@ -195,10 +207,11 @@ def _on_job_completed( ), ) -> None: """ - Handles the end of an attempt of the given job, with the exception - information of the attempt, or None. Emits the events, then restarts - the job if it failed and has attempts left, else removes it from the - pipeline. The watcher thread of the job calls this. + Handles the end of an attempt of the given job, which failed with the + given exception information, or succeeded if it is None; the watcher + thread of the job calls it. Emits the events, then restarts the job if + it failed and has attempts left, else removes it from the pipeline, + and emits queue_empty_event if the pipeline is empty then. """ # This function is called in a sub-thread, so we need to be # careful that we are not in a lock while sending an event. @@ -227,8 +240,8 @@ def _on_job_completed( if exc_info and job.failures < job.times: self._dbg(3, 'Restarting job "%s"' % job.name) job.start(self.job_cls, self._on_job_completed, self.job_started_event) - else: - self.collection.task_done(job) + elif self.collection.with_lock(self._remove, job): + self.queue_empty_event() def run(self) -> None: """ @@ -239,8 +252,6 @@ def run(self) -> None: # Get the next job from the queue. This blocks until a task # is available or until self.collection.stop() is called. job = next(self.collection) - if len(self.collection) <= 0: - self.queue_empty_event() if job is None: break # self.collection.stop() was called. diff --git a/src/Exscript/workqueue/workqueue.py b/src/Exscript/workqueue/workqueue.py index ec61a34b..71eee63a 100644 --- a/src/Exscript/workqueue/workqueue.py +++ b/src/Exscript/workqueue/workqueue.py @@ -59,6 +59,12 @@ class WorkQueue: """ This class implements the asynchronous workqueue and is the main API for using the workqueue module. + + Its events are those of its main loop, see MainLoop. queue_empty_event + is emitted, without arguments, when the queue becomes empty: when the + last queued or running job has ended. It is emitted in a thread of the + work queue, after the job was removed, so wait_until_done() may return + before it. """ def __init__( diff --git a/tests/Exscript/workqueue/WorkQueueTest.py b/tests/Exscript/workqueue/WorkQueueTest.py index 69995fa7..b9e4b444 100644 --- a/tests/Exscript/workqueue/WorkQueueTest.py +++ b/tests/Exscript/workqueue/WorkQueueTest.py @@ -181,6 +181,25 @@ def testJobHasItsIdWhenItStarts(self): with self.subTest(enqueue=enqueue): self.assertJobHasItsIdWhenItStarts(enqueue, add) + def testQueueEmptyEvent(self): + # Emitted when the last job has ended, not when the queue stops. + self.addCleanup(self.wq.destroy) + lengths = [] + jobs = [] + self.wq.queue_empty_event.connect(lambda: lengths.append(self.wq.get_length())) + self.wq.job_init_event.connect(jobs.append) + self.wq.pause() + for _ in range(3): + self.wq.enqueue(nop) + self.wq.set_max_threads(3) + self.wq.unpause() + self.wq.wait_until_done() + for job in jobs: + job.join() # its thread removes the job and emits the event + self.assertEqual(lengths, [0]) + self.wq.shutdown(True) + self.assertEqual(lengths, [0]) + def testPause(self): pass # See testEnqueue() From 6b8a1cd9f6a68149bbe25f072bf72a6c4d3d72eb Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 00:40:05 +0100 Subject: [PATCH 155/250] Check the running jobs of the work queue in the test thread testGetRunningJobs asserted inside the job, where the work queue catches every exception, so it passed whatever it found. The job now records the running jobs, and the test checks them afterwards. --- tests/Exscript/workqueue/WorkQueueTest.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/tests/Exscript/workqueue/WorkQueueTest.py b/tests/Exscript/workqueue/WorkQueueTest.py index b9e4b444..094dbf4e 100644 --- a/tests/Exscript/workqueue/WorkQueueTest.py +++ b/tests/Exscript/workqueue/WorkQueueTest.py @@ -245,13 +245,19 @@ def testIsPaused(self): self.assertTrue(self.wq.is_paused()) def testGetRunningJobs(self): + # The job records what it sees, and the test checks it afterwards. + self.addCleanup(self.wq.destroy) + running = [] + def function(job): - self.assertEqual(self.wq.get_running_jobs(), [job]) + running.append(self.wq.get_running_jobs()) self.assertEqual(self.wq.get_running_jobs(), []) - self.wq.enqueue(function) - self.wq.shutdown(True) + job_id = self.wq.enqueue(function) + self.wq.wait_for(job_id) self.assertEqual(self.wq.get_running_jobs(), []) + # While the job ran, it was the only running job. + self.assertEqual([[job.id for job in jobs] for jobs in running], [[job_id]]) def testGetLength(self): pass # See testEnqueue() From c5fd0b9c9a4aa128ce79c98811c15388591ceb67 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 00:42:17 +0100 Subject: [PATCH 156/250] Fix the status bar of the queue on narrow terminals The bar cuts the names of the running jobs in the middle to fit, but with a large overflow, the two halves overlapped and made the bar longer. Also pass force by keyword in PipelineTest, and drop an unreachable return of Pipeline.get_from_name(). --- CHANGES.md | 4 ++++ src/Exscript/queue.py | 4 ++-- src/Exscript/workqueue/pipeline.py | 1 - tests/Exscript/QueueTest.py | 22 ++++++++++++++++++++++ tests/Exscript/workqueue/PipelineTest.py | 2 +- 5 files changed, 29 insertions(+), 4 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index af4d70b6..fa975078 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -648,6 +648,10 @@ The names below refer to functions callable from templates, not protocol methods - **`WorkQueue.queue_empty_event`:** Is emitted when the queue becomes empty, when the last queued or running job has ended. Before, it was emitted only when the main loop stopped, on `shutdown()` or `destroy()` of an empty queue. +- **Status Bar on Narrow Terminals:** If the names of the running jobs do not fit into + the status bar of `Queue`, it cuts them in the middle, at most down to `...`. Before, + on a terminal of about 30 columns or less, the cut overlapped and made the bar longer + instead of shorter. ## Demos diff --git a/src/Exscript/queue.py b/src/Exscript/queue.py index 02512cd2..b0d868c3 100644 --- a/src/Exscript/queue.py +++ b/src/Exscript/queue.py @@ -357,9 +357,9 @@ def _print_status_bar(self, exclude: str | None = None) -> None: cont = '...' overflow += len(cont) + 1 strlen = len(running) - partlen = strlen // 2 - overflow // 2 + partlen = max(0, strlen // 2 - overflow // 2) head = running[:partlen] - tail = running[-partlen:] + tail = running[strlen - partlen :] running = head + cont + tail text = 'In progress: [%s] %s' % (running, progress) self._write('status_bar', text) diff --git a/src/Exscript/workqueue/pipeline.py b/src/Exscript/workqueue/pipeline.py index f9d51d2d..8620c893 100644 --- a/src/Exscript/workqueue/pipeline.py +++ b/src/Exscript/workqueue/pipeline.py @@ -101,7 +101,6 @@ def get_from_name(self, name: str | None) -> Any: except KeyError: return None return self.id2item[item_id] - return None def has_id(self, item_id: str) -> bool: """ diff --git a/tests/Exscript/QueueTest.py b/tests/Exscript/QueueTest.py index 97992e76..190b1b34 100644 --- a/tests/Exscript/QueueTest.py +++ b/tests/Exscript/QueueTest.py @@ -12,6 +12,7 @@ from functools import partial from multiprocessing import Value from tempfile import mkdtemp +from types import SimpleNamespace from unittest import mock from Exscript import Account, AccountPool, FileLogger, Host, Queue @@ -479,6 +480,27 @@ def record_driver(job, host, conn): self.queue.join() self.assertEqual(drivers, ['ios', 'ios']) + def testStatusBarShortensTheNamesOfRunningJobs(self): + # The names are cut in the middle to fit the terminal, at most to '...'. + jobs = [SimpleNamespace(name='a' * 30), SimpleNamespace(name='b' * 30)] + expected = { + 40: 'In progress: [aaaaa...bbbbb] 1/3 (33%)', + 30: 'In progress: [...] 1/3 (33%)', + 25: 'In progress: [...] 1/3 (33%)', # cannot fit + } + for cols, bar in expected.items(): + with self.subTest(cols=cols): + self.createQueue(verbose=1, max_threads=2) + self.queue.total = 3 + self.queue.completed = 1 + with ( + mock.patch.object(self.queue.workqueue, 'get_running_jobs', return_value=jobs), + mock.patch('Exscript.queue.get_terminal_size', return_value=(24, cols)), + ): + self.queue._print_status_bar() + self.assertEqual(self.out.read(), bar) + self.assertEqual(self.queue.status_bar_length, len(bar)) + def testForceRun(self): data = Value('i', 0) hosts = ['dummy://dummy1', 'dummy://dummy2'] diff --git a/tests/Exscript/workqueue/PipelineTest.py b/tests/Exscript/workqueue/PipelineTest.py index eee9832a..c6ec2a90 100644 --- a/tests/Exscript/workqueue/PipelineTest.py +++ b/tests/Exscript/workqueue/PipelineTest.py @@ -103,7 +103,7 @@ def testAppendleft(self): self.pipeline.appendleft(item3) self.assertEqual(self.pipeline.try_next(), item3) - self.pipeline.appendleft(item4, True) + self.pipeline.appendleft(item4, force=True) self.assertEqual(self.pipeline.try_next(), item4) def testPrioritize(self): From 10d94acb29221493eca2c6c3af6282d11845be3e Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 00:34:36 +0100 Subject: [PATCH 157/250] Install the command line tools as console entry points The scripts exscript and otp had no file extension, so the shells of Windows did not run them. Their code moves to the new module Exscript.cli, whose main() and otp_main() are installed under the old names; python -m Exscript runs main(). --- CHANGES.md | 5 + docs/cli_tutorial.rst | 3 +- docs/conf.py | 2 +- pyproject.toml | 11 +- scripts/otp | 21 - src/Exscript/__main__.py | 10 + scripts/exscript => src/Exscript/cli.py | 663 ++++++++++++++---------- src/Exscript/util/sigint.py | 2 +- tests/Exscript/ScriptsTest.py | 19 +- tests/Exscript/cliTest.py | 52 ++ tests/scripts/exscriptTest.py | 40 -- 11 files changed, 482 insertions(+), 346 deletions(-) delete mode 100755 scripts/otp create mode 100644 src/Exscript/__main__.py rename scripts/exscript => src/Exscript/cli.py (63%) mode change 100755 => 100644 create mode 100644 tests/Exscript/cliTest.py delete mode 100644 tests/scripts/exscriptTest.py diff --git a/CHANGES.md b/CHANGES.md index fa975078..b87e4995 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -27,6 +27,11 @@ marked with **BREAKING**; a migration guide will accompany the release. - **Dependencies:** `future` and `configparser`, backports for Python 2, are no longer needed. paramiko 3.4 to 5.x is supported (before: 1.17 to 4.x). New: ncclient and lxml 5.0 or later, for NETCONF. +- **Console Commands:** The commands `exscript` and `otp` keep their names, and are now + console entry points of the package: `Exscript.cli.main()` and `otp_main()`. + `python -m Exscript` runs `exscript`, too. On Windows, the commands are `.exe` + launchers that work from any shell. Before, they were Python scripts without a file + extension, which cmd.exe and PowerShell do not run. - **NETCONF Protocol:** The new adapter `Exscript.protocols.Netconf` (scheme `netconf://`, port 830) sends RPCs over SSH with ncclient, and returns the replies as text or as lxml elements; an `` raises `RPCError`. The IOS XR and Junos diff --git a/docs/cli_tutorial.rst b/docs/cli_tutorial.rst index b100fe58..bcf89489 100644 --- a/docs/cli_tutorial.rst +++ b/docs/cli_tutorial.rst @@ -13,7 +13,8 @@ tool. We'll assume that Exscript is already installed. You can confirm that your installation works by typing ``exscript --version`` into a terminal; if this prints the version number, your installation is -complete. +complete. ``python -m Exscript`` runs the tool, too, e.g. if the +``exscript`` command is not on your ``PATH``. Getting started --------------- diff --git a/docs/conf.py b/docs/conf.py index 4e3a1cb4..12fb6ed7 100755 --- a/docs/conf.py +++ b/docs/conf.py @@ -15,7 +15,7 @@ # The command line reference includes the output of "exscript --help". Sphinx # runs this file in the docs directory. _help = subprocess.run( - [sys.executable, "../scripts/exscript", "--help"], + [sys.executable, "-m", "Exscript", "--help"], capture_output=True, check=True, encoding="utf-8", diff --git a/pyproject.toml b/pyproject.toml index f27d8cf7..1c7a0383 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,6 +49,10 @@ otel = [ "opentelemetry-instrumentation>=0.48b0", ] +[project.scripts] +exscript = "Exscript.cli:main" +otp = "Exscript.cli:otp_main" + [project.urls] Homepage = "https://github.com/knipknap/exscript" Documentation = "https://exscript.readthedocs.io" @@ -75,14 +79,9 @@ source = "vcs" [tool.hatch.build.targets.wheel] packages = ["src/Exscript"] -[tool.hatch.build.targets.wheel.shared-scripts] -"scripts/exscript" = "exscript" -"scripts/otp" = "otp" - [tool.ruff] line-length = 100 src = ["src", "tests"] -extend-include = ["scripts/exscript", "scripts/otp"] extend-exclude = ["docs/_build"] [tool.ruff.format] @@ -159,7 +158,7 @@ fail-under = 93 ignore-magic = true ignore-nested-functions = true ignore-setters = true -exclude = ["docs", "tests", "tools", "demos", "scripts"] +exclude = ["docs", "tests", "tools", "demos"] [tool.coverage.run] branch = true diff --git a/scripts/otp b/scripts/otp deleted file mode 100755 index f1974882..00000000 --- a/scripts/otp +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env python3 -import argparse -import getpass - -from Exscript.util.crypt import otp - -# Define the command line syntax. -parser = argparse.ArgumentParser() -parser.add_argument('sequence', help='the sequence number') -parser.add_argument('salt', help='the salt value') -args = parser.parse_args() -seq = args.sequence -salt = args.salt - -try: - seq = int(seq) -except ValueError: - parser.error('first argument must be a number') - -password = getpass.getpass('Password: ') -print(otp(password, salt, seq)) diff --git a/src/Exscript/__main__.py b/src/Exscript/__main__.py new file mode 100644 index 00000000..cc78a72e --- /dev/null +++ b/src/Exscript/__main__.py @@ -0,0 +1,10 @@ +""" +Runs the command line tool ``exscript``: ``python -m Exscript``. +""" + +import sys + +from .cli import main + +if __name__ == '__main__': + sys.exit(main()) diff --git a/scripts/exscript b/src/Exscript/cli.py old mode 100755 new mode 100644 similarity index 63% rename from scripts/exscript rename to src/Exscript/cli.py index 6ea74000..abd41250 --- a/scripts/exscript +++ b/src/Exscript/cli.py @@ -1,6 +1,4 @@ -#!/usr/bin/env python3 -"""TODO: Add missing doctring.""" - +# # Copyright (C) 2007-2010 Samuel Abels # The MIT License (MIT) # @@ -22,6 +20,15 @@ # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" +The command line tools ``exscript`` and ``otp``. + +The package installs both as commands, and ``python -m Exscript`` runs +``exscript``, too. ``exscript`` runs a template on hosts; see the +:doc:`/cli_tutorial` and ``exscript --help``. ``otp`` asks for a password, +and prints the S/Key one-time password for a sequence number and a salt. +""" + import argparse import ast import getpass @@ -30,14 +37,16 @@ import re import sys import time +from collections.abc import Sequence from pathlib import Path import Exscript.protocols.drivers -from Exscript import Account, PrivateKey, Queue -from Exscript.logger import getConsoleLogger, setupLogging -from Exscript.protocols import protocol_map +from Exscript import Account, Host, PrivateKey, Queue +from Exscript.logger import setupLogging +from Exscript.protocols import Protocol, protocol_map from Exscript.util import template from Exscript.util.cast import to_host +from Exscript.util.crypt import otp from Exscript.util.decorator import autologin from Exscript.util.file import ( get_accounts_from_file, @@ -48,8 +57,7 @@ from Exscript.util.interact import get_login from Exscript.util.log import log_to_file from Exscript.util.sigint import SigIntWatcher - -_logger = getConsoleLogger("exscript-cli") +from Exscript.workqueue.job import Thread bracket_expression = r"\{([^\]]*)\}" define_expression = rf"^\S+=(?:\S+|{bracket_expression})$" @@ -59,8 +67,17 @@ duration_units = {"": 1, "s": 1, "m": 60, "h": 3600, "d": 86400} -def duration(text): - """Returns the seconds of a time like the sleep command takes it, e.g. "1h 30m".""" +def duration(text: str) -> float: + """ + Returns the seconds of a time like the sleep command takes it, e.g. "1h 30m": + numbers with an optional suffix s, m, h or d, separated by blanks. + Raises a ValueError for other text. + + :type text: str + :param text: The time. + :rtype: float + :return: The time in seconds. + """ seconds = 0.0 for part in text.split(): match = duration_re.fullmatch(part) @@ -70,8 +87,21 @@ def duration(text): return seconds -def expand_bracket(value_name, value): - """TODO: Add missing doctring.""" +def expand_bracket(value_name: str, value: str) -> object: + """ + Returns the given value of the variable with the given name, or, if the + value is a prompt in braces, asks the user for the value and returns + that. ``{PROMPT}`` reads a line, ``{!PROMPT}`` a password, ``{#PROMPT}`` + a Python literal such as a number or a list (or the line as it is, if it + is no literal), and ``{}`` asks for a value for the variable. + + :type value_name: str + :param value_name: The name of the variable. + :type value: str + :param value: The value, or a prompt in braces. + :rtype: object + :return: The value. + """ match = bracket_expression_re.match(value) if match is None: return value @@ -88,8 +118,15 @@ def expand_bracket(value_name, value): return value -def expand_host_variables(host): - """TODO: Add missing doctring.""" +def expand_host_variables(host: Host) -> None: + """ + Asks the user for the values of the variables of the given host that are + prompts in braces, also in lists, and replaces the prompts by the + values; see :func:`expand_bracket`. + + :type host: Host + :param host: The host. + """ # Define host-specific variables. for key, value in list(host.get_all().items()): if isinstance(value, str): @@ -102,26 +139,35 @@ def expand_host_variables(host): host.set(key, value) -def run(): - """TODO: Add missing doctring.""" - # Parse options. - args = parser.parse_args(sys.argv[1:]) +def run(parser: argparse.ArgumentParser, args: argparse.Namespace) -> int: + """ + Runs the template on the hosts as the given command line arguments say, + and returns the number of hosts on which it failed. Invalid arguments + end the process through ``parser.error()``. + + :type parser: argparse.ArgumentParser + :param parser: The parser of the arguments, see :func:`create_parser`. + :type args: argparse.Namespace + :param args: The parsed arguments. + :rtype: int + :return: The number of failed actions. + """ - def mkhost(name): + def mkhost(name: str) -> Host: return to_host( name, default_protocol=args.protocol, default_domain=args.default_domain, ) - def mkhosts_from_file(filename): + def mkhosts_from_file(filename: Path) -> list[Host]: return get_hosts_from_file( args.hosts, default_protocol=args.protocol, default_domain=args.default_domain, ) - def mkhosts_from_csv(filename): + def mkhosts_from_csv(filename: Path) -> list[Host]: return get_hosts_from_csv( args.csv_hosts, default_protocol=args.protocol, @@ -172,7 +218,7 @@ def mkhosts_from_csv(filename): except OSError as e: parser.error(str(e)) except ValueError as e: - parser.error(e) + parser.error(str(e)) if not txt_hosts: print(f"Warning: {args.hosts} is empty.") hosts += txt_hosts @@ -183,6 +229,8 @@ def mkhosts_from_csv(filename): print(f"Searching drivers in {args.load_driver}...") try: spec = importlib.util.spec_from_file_location(path.stem, path) + if spec is None or spec.loader is None: + raise ImportError(f"{path} is not a Python file") driver_module = importlib.util.module_from_spec(spec) spec.loader.exec_module(driver_module) except OSError as e: @@ -314,7 +362,7 @@ def mkhosts_from_csv(filename): tmpl_vars = args.define if args.no_prompt: - def function(job, host, conn, **kwargs): + def function(job: Thread, host: Host, conn: Protocol, **kwargs: object) -> object: kwargs.update(functions) kwargs.update(tmpl_vars) if args.filename: @@ -326,7 +374,7 @@ def function(job, host, conn, **kwargs): else: - def function(job, host, conn, **kwargs): + def function(job: Thread, host: Host, conn: Protocol, **kwargs: object) -> object: kwargs.update(functions) kwargs.update(tmpl_vars) if args.filename: @@ -377,22 +425,47 @@ def function(job, host, conn, **kwargs): # Define command line option value types. -def assignment(value, pattern=define_expression_re): - """TODO: Add missing doctring.""" +def assignment(value: str, pattern: re.Pattern[str] = define_expression_re) -> str: + """ + Checks a value of the option -d/--define: returns it if it matches the + given pattern, by default that of an assignment NAME=VALUE, and raises + an ArgumentTypeError if not. + + :type value: str + :param value: The value of the option. + :type pattern: re.Pattern + :param pattern: The pattern of valid values. + :rtype: str + :return: The value. + """ if not pattern.match(value): raise argparse.ArgumentTypeError(f"invalid value: {value}") return value class DefineAction(argparse.Action): - """TODO: Add missing doctring.""" - - def __call__(self, parser, namespace, values, option_string=None): - """TODO: Add missing doctring.""" + """ + The action of the option -d/--define: stores the variables of the given + assignments NAME=VALUE in the dictionary of the option. Variables in + double underscores, such as ``__filename__``, are internal. + """ + + def __call__( + self, + parser: argparse.ArgumentParser, + namespace: argparse.Namespace, + values: str | Sequence[str] | None, + option_string: str | None = None, + ) -> None: + """ + Adds the variables of the given assignments to the dictionary of the + option in the given namespace, or reports an error through the parser + if a variable is internal. + """ if type(values) == str: values = [values] d = getattr(namespace, self.dest) - for value in values: + for value in values or (): left, right = value.split("=") if left.startswith('__') and left.endswith('__'): parser.error('Assignment to internal variable %s' % repr(left)) @@ -400,57 +473,65 @@ def __call__(self, parser, namespace, values, option_string=None): setattr(namespace, self.dest, d) -# Define the command line syntax. -parser = argparse.ArgumentParser(epilog="\N{COPYRIGHT SIGN} 2007-2023 by Samuel Abels.") +def create_parser() -> argparse.ArgumentParser: + """ + Returns a parser of the command line arguments of ``exscript``. + + :rtype: argparse.ArgumentParser + :return: The parser. + """ + parser = argparse.ArgumentParser( + prog="exscript", epilog="\N{COPYRIGHT SIGN} 2007-2023 by Samuel Abels." + ) -parser.add_argument( - "filename", - default=None, - nargs="?", - help=""" + parser.add_argument( + "filename", + default=None, + nargs="?", + help=""" Filename of the template to be executed. Ignored with "-e/--execute" and taken as first positional host. """.strip(), -) + ) -parser.add_argument( - "host", - default=list(), - nargs="*", - help=""" + parser.add_argument( + "host", + default=list(), + nargs="*", + help=""" List of hostnames. """.strip(), -) + ) -parser.add_argument( - "--account-pool", - default=None, - metavar="FILE", - help=""" + parser.add_argument( + "--account-pool", + default=None, + metavar="FILE", + help=""" Reads the user/password combination from the given file instead of prompting on the command line. The file may also contain more than one user/password combination, in which case the accounts are used round robin. """.strip(), -) + ) -parser.add_argument( - "-c", - "--connections", - type=int, - metavar="NUM", - default=1, - help=""" + parser.add_argument( + "-c", + "--connections", + type=int, + metavar="NUM", + default=1, + help=""" Maximum number of concurrent connections. NUM is a number between 1 and 20, default is 1. """.strip(), -) + ) -parser.add_argument( - "--csv-hosts", - default=None, - metavar="FILE", - help=""" + parser.add_argument( + "--csv-hosts", + default=None, + metavar="FILE", + help=""" Loads a list of hostnames and definitions from the given file. The first line of the file must have the column headers in the following syntax: "hostname [variable] [variable] ...", where @@ -460,16 +541,16 @@ def __call__(self, parser, namespace, values, option_string=None): The following lines contain the hostname in the first column, and the values of the variables in the following columns. """.strip(), -) + ) -parser.add_argument( - "-d", - "--define", - type=assignment, - action=DefineAction, - default=dict(), - metavar="PAIR", - help=""" + parser.add_argument( + "-d", + "--define", + type=assignment, + action=DefineAction, + default=dict(), + metavar="PAIR", + help=""" Defines a variable that is passed to the script. PAIR has the following syntax: STRING=STRING. Use a prompt with the following syntax: VAR={"PROMPT"}. @@ -477,264 +558,264 @@ def __call__(self, parser, namespace, values, option_string=None): Evaluate the value of a prompt as a Python literal (e.g. a number or a list) with the following syntax: VAR={#"PROMPT"}. """.strip(), -) + ) -parser.add_argument( - "--default-domain", - metavar="STRING", - default="", - help=""" + parser.add_argument( + "--default-domain", + metavar="STRING", + default="", + help=""" The IP domain name that is used if a given hostname has no domain appended. """.strip(), -) + ) -parser.add_argument( - "--delete-logs", - action="store_true", - default=False, - help="Delete logs of successful operations when done.", -) + parser.add_argument( + "--delete-logs", + action="store_true", + default=False, + help="Delete logs of successful operations when done.", + ) -parser.add_argument( - "--no-escaping", - action="store_true", - default=False, - help=""" + parser.add_argument( + "--no-escaping", + action="store_true", + default=False, + help=""" Disable additional evaluation of escaped control characters (NUL/BEL/BS/ESC/FF/VT), octal (\\ooo), hexadecimal (\\xhh) and unicode (\\uhhhh, \\Uhhhhhhhh) characters in the script. Evaluation of escaped CR, LF and TAB characters in strings is always enabled. This is irrelevant for RPC bodies in Netconf as XML does not support escaping (see XML entity syntax instead). """.strip(), -) + ) -parser.add_argument( - "--lib", - default=None, - metavar="FILE", - help=""" + parser.add_argument( + "--lib", + default=None, + metavar="FILE", + help=""" A python file containing a __lib__ dictionary that points to extra functions to be made available in the Exscript template. """.strip(), -) + ) -parser.add_argument( - "--insecure", - action="store_true", - default=False, - help=""" + parser.add_argument( + "--insecure", + action="store_true", + default=False, + help=""" Allow the template to call the functions that run local commands or access local files: sys.exec, sys.env and file.* (except file.exists). """.strip(), -) + ) -parser.add_argument( - "--load-driver", - default=None, - metavar="FILE", - help=""" + parser.add_argument( + "--load-driver", + default=None, + metavar="FILE", + help=""" A python file containing an Exscript driver class. """.strip(), -) + ) -parser.add_argument( - "--use-driver", - default=None, - metavar="NAME", - help=""" + parser.add_argument( + "--use-driver", + default=None, + metavar="NAME", + help=""" The name of an exscript driver to use for all connections (overriding os detection). """.strip(), -) + ) -parser.add_argument( - "-e", - "--execute", - default=None, - metavar="EXSCRIPT", - help=""" + parser.add_argument( + "-e", + "--execute", + default=None, + metavar="EXSCRIPT", + help=""" Interprets the given string as the script. """.strip(), -) + ) -parser.add_argument( - "--hosts", - default=None, - metavar="FILE", - help=""" + parser.add_argument( + "--hosts", + default=None, + metavar="FILE", + help=""" Loads a list of hostnames from the given file (one host per line). """.strip(), -) + ) -parser.add_argument( - "-i", - "--non-interactive", - action="store_true", - default=False, - help=""" + parser.add_argument( + "-i", + "--non-interactive", + action="store_true", + default=False, + help=""" Do not ask for a username or password, but still try to authenticate. May be used if the login credentials are passed as part of a URI formatted hostname. """.strip(), -) + ) -parser.add_argument( - "-l", - "--logdir", - default=None, - metavar="DIR", - help=""" + parser.add_argument( + "-l", + "--logdir", + default=None, + metavar="DIR", + help=""" Logs any communication into the directory with the given name. Each filename consists of the hostname with ".log" appended. Errors are written to a separate file, where the filename consists of the hostname with ".log.error" appended. """.strip(), -) + ) -parser.add_argument( - "-n", - "--no-authentication", - action="store_true", - default=False, - help=""" + parser.add_argument( + "-n", + "--no-authentication", + action="store_true", + default=False, + help=""" When given, the automatic authentication procedure is skipped. Implies -i. """.strip(), -) + ) -parser.add_argument( - "--authorization", - action="store_true", - default=False, - help=""" + parser.add_argument( + "--authorization", + action="store_true", + default=False, + help=""" Ask for an authorization password in addition to the authentication password. """.strip(), -) + ) -parser.add_argument( - "--no-auto-logout", - action="store_true", - default=False, - help=""" + parser.add_argument( + "--no-auto-logout", + action="store_true", + default=False, + help=""" Do not attempt to execute the exit or quit command at the end of a script. """.strip(), -) + ) -parser.add_argument( - "--no-prompt", - action="store_true", - default=False, - help=""" + parser.add_argument( + "--no-prompt", + action="store_true", + default=False, + help=""" Do not wait for a prompt anywhere (except during the authentication procedure). Note that this will also cause Exscript to disable commands that require a prompt, such as "extract". """.strip(), -) + ) -parser.add_argument( - "--no-strip", - action="store_true", - default=False, - help=""" + parser.add_argument( + "--no-strip", + action="store_true", + default=False, + help=""" Do not strip the command echo from each response. This is irrelevant for RPC bodies as Netconf does not echo commands. """.strip(), -) + ) -parser.add_argument( - "--overwrite-logs", - action="store_true", - default=False, - help=""" + parser.add_argument( + "--overwrite-logs", + action="store_true", + default=False, + help=""" Instructs Exscript to overwrite existing logfiles. The default is to append the output if a log already exists. """.strip(), -) + ) -protocols = list(protocol_map.keys()) -protocols.sort() -parser.add_argument( - "-p", - "--protocol", - metavar="STRING", - default="ssh", - choices=protocols, - help=f""" + protocols = list(protocol_map.keys()) + protocols.sort() + parser.add_argument( + "-p", + "--protocol", + metavar="STRING", + default="ssh", + choices=protocols, + help=f""" Specify which protocol to use to connect to the remote host. Allowed values for STRING include: {", ".join(protocols)}. The default protocol is ssh. """.strip(), -) + ) -parser.add_argument( - "--retry", - type=int, - metavar="NUM", - default=0, - help=""" + parser.add_argument( + "--retry", + type=int, + metavar="NUM", + default=0, + help=""" Defines the number of retries per host on failure. Default is 0. """.strip(), -) + ) -parser.add_argument( - "--retry-login", - type=int, - metavar="NUM", - default=0, - help=""" + parser.add_argument( + "--retry-login", + type=int, + metavar="NUM", + default=0, + help=""" Defines the number of retries per host on login failure. Default is 0. """.strip(), -) + ) -parser.add_argument( - "--sleep", - metavar="TIME", - type=duration, - default="", - help=""" + parser.add_argument( + "--sleep", + metavar="TIME", + type=duration, + default="", + help=""" Waits for the specified time before running the script. TIME is a number of seconds, optionally with the suffix m, h or d, like for the 'sleep' Unix command; e.g. "90" or "1h 30m". """.strip(), -) + ) -parser.add_argument( - "--ssh-auto-verify", - action="store_true", - default=False, - help=""" + parser.add_argument( + "--ssh-auto-verify", + action="store_true", + default=False, + help=""" Disables the verification of SSH host keys, so that any key is accepted. Highly insecure and not recommended. """.strip(), -) + ) -parser.add_argument( - "--ssh-host-key-policy", - choices=("reject", "warn", "auto-add"), - default="reject", - help=""" + parser.add_argument( + "--ssh-host-key-policy", + choices=("reject", "warn", "auto-add"), + default="reject", + help=""" What to do with SSH hosts that are not in ~/.ssh/known_hosts: reject the connection (the default), warn and connect, or add the host key to known_hosts and connect. """.strip(), -) + ) -parser.add_argument( - "--ssh-disabled-algs", - default="", - metavar="STRING", - help=""" + parser.add_argument( + "--ssh-disabled-algs", + default="", + metavar="STRING", + help=""" Specify a mapping of disabled SSH algorithms according to Paramiko's SSHClient/Transport constructor. The syntax on the CLI goes like this: "ciphers=aes128-cbc,3des-cbc;pubkeys=ssh-rsa,ssh-dss". This will be parsed into a mapping as expected by the Paramiko constructor. Please refer to the "paramiko.Transport" documentation for details. """.strip(), -) + ) -parser.add_argument( - "--ssh-preferred-auth", - default="", - metavar="STRING", - help=""" + parser.add_argument( + "--ssh-preferred-auth", + default="", + metavar="STRING", + help=""" Specify a comma-separated list of preferred SSH authentication types in the preferred order. This is equivalent to the "-o PreferredAuthentications" option of the openssh command @@ -742,56 +823,104 @@ def __call__(self, parser, namespace, values, option_string=None): The default is "publickey,keyboard-interactive,password" which is derived from the SSH config default value. """.strip(), -) + ) -parser.add_argument( - "--ssh-key", - default=None, - metavar="FILE", - help=""" + parser.add_argument( + "--ssh-key", + default=None, + metavar="FILE", + help=""" Specify a key file that is passed to the SSH client. This is equivalent to using the "-i" parameter of the openssh command line client. """.strip(), -) + ) -parser.add_argument( - "-v", - "--verbose", - type=int, - metavar="NUM", - default=1, - help=""" + parser.add_argument( + "-v", + "--verbose", + type=int, + metavar="NUM", + default=1, + help=""" Print out debug information about the queue. NUM is a number between 0 (min) and 5 (max). Default is 1. """.strip(), -) + ) -parser.add_argument( - "-V", - "--parser-verbose", - type=int, - metavar="NUM", - default=0, - help=""" + parser.add_argument( + "-V", + "--parser-verbose", + type=int, + metavar="NUM", + default=0, + help=""" Print out debug information about the Exscript template parser. NUM is a number between 0 (min) and 5 (max). Default is 0. """.strip(), -) + ) -parser.add_argument( - "--protocol-verbose", - type=int, - metavar="NUM", - default=0, - help=""" + parser.add_argument( + "--protocol-verbose", + type=int, + metavar="NUM", + default=0, + help=""" Print out debug information about the network activity. NUM is a number between 0 (min) and 5 (max). Default is 0. """.strip(), -) + ) + + return parser -if __name__ == "__main__": + +def main(argv: list[str] | None = None) -> int: + """ + Runs the command line tool ``exscript`` with the given arguments. + + :type argv: list[str] | None + :param argv: The arguments without the name of the program; by default + those of the process. + :rtype: int + :return: The exit status: 0 if the template succeeded on all hosts, + else 1. Invalid arguments and ``--help`` end the process through + SystemExit, as usual with argparse. + """ SigIntWatcher() - failed = run() + parser = create_parser() + failed = run(parser, parser.parse_args(argv)) if failed != 0: - sys.exit(f"error: {failed} actions failed.") + print(f"error: {failed} actions failed.", file=sys.stderr) + return 1 + return 0 + + +def otp_main(argv: list[str] | None = None) -> int: + """ + Runs the command line tool ``otp`` with the given arguments: asks for a + password, and prints the S/Key one-time password for the given sequence + number and salt; see :func:`Exscript.util.crypt.otp`. + + :type argv: list[str] | None + :param argv: The arguments without the name of the program; by default + those of the process. + :rtype: int + :return: The exit status, 0. Invalid arguments and ``--help`` end the + process through SystemExit, as usual with argparse. + """ + # Define the command line syntax. + parser = argparse.ArgumentParser(prog='otp') + parser.add_argument('sequence', help='the sequence number') + parser.add_argument('salt', help='the salt value') + args = parser.parse_args(argv) + seq = args.sequence + salt = args.salt + + try: + seq = int(seq) + except ValueError: + parser.error('first argument must be a number') + + password = getpass.getpass('Password: ') + print(otp(password, salt, seq)) + return 0 diff --git a/src/Exscript/util/sigint.py b/src/Exscript/util/sigint.py index f5261322..184b37da 100644 --- a/src/Exscript/util/sigint.py +++ b/src/Exscript/util/sigint.py @@ -46,7 +46,7 @@ class SigIntWatcher: on Windows. """ - def __init__(self): + def __init__(self) -> None: """ Creates a child process, which returns. The parent process waits for a KeyboardInterrupt and then kills diff --git a/tests/Exscript/ScriptsTest.py b/tests/Exscript/ScriptsTest.py index 6bea8bc1..cb55c38f 100644 --- a/tests/Exscript/ScriptsTest.py +++ b/tests/Exscript/ScriptsTest.py @@ -5,13 +5,14 @@ import unittest BASEDIR = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..')) -SCRIPT = os.path.join(BASEDIR, 'scripts', 'exscript') KEYFILE = os.path.join(BASEDIR, 'tests', 'Exscript', 'protocols', 'id_rsa') -# Runs the script, but getpass.getuser() finds no user name, as for a user ID without a passwd +# Runs the CLI, but getpass.getuser() finds no user name, as for a user ID without a passwd # entry: it raises OSError since Python 3.13, and KeyError before. WITHOUT_USER_NAME = ''' -import getpass, runpy, sys +import getpass, sys + +from Exscript.cli import main def getuser(): @@ -19,8 +20,7 @@ def getuser(): getpass.getuser = getuser -del sys.argv[0] -runpy.run_path(sys.argv[0], run_name='__main__') +sys.exit(main()) ''' @@ -31,10 +31,10 @@ def setUp(self): self.tmpdir = tempfile.TemporaryDirectory() self.addCleanup(self.tmpdir.cleanup) - def run_exscript(self, *args, python_args=()): - env = dict(os.environ, HOME=self.tmpdir.name, USERPROFILE=self.tmpdir.name) - proc = subprocess.run( # noqa: S603 - runs this repository's script with fixed arguments - [sys.executable, *python_args, SCRIPT, *args], + def run_exscript(self, *args, python_args=('-m', 'Exscript')): + env = dict(os.environ, HOME=self.tmpdir.name, USERPROFILE=self.tmpdir.name, NO_COLOR='1') + proc = subprocess.run( # noqa: S603 - runs Exscript with fixed arguments + [sys.executable, *python_args, *args], capture_output=True, encoding='utf-8', env=env, @@ -51,6 +51,7 @@ def write_template(self, text): def testHelp(self): code, output = self.run_exscript('--help') self.assertEqual(code, 0, output) + self.assertTrue(output.startswith('usage: exscript '), output) for option in '--load-driver', '--insecure', '--ssh-host-key-policy': self.assertIn(option, output) diff --git a/tests/Exscript/cliTest.py b/tests/Exscript/cliTest.py new file mode 100644 index 00000000..767dc364 --- /dev/null +++ b/tests/Exscript/cliTest.py @@ -0,0 +1,52 @@ +import contextlib +import io +import unittest +from unittest import mock + +from Exscript.cli import duration, expand_bracket, otp_main +from Exscript.util.crypt import otp + + +class cliTest(unittest.TestCase): + def testDuration(self): + self.assertEqual(duration(''), 0) + self.assertEqual(duration('90'), 90) + self.assertEqual(duration('1.5m'), 90) + self.assertEqual(duration('1h 30m'), 5400) + self.assertEqual(duration('2d'), 172800) + for invalid in 'tomorrow', '1x', '-1', '1; rm -rf ~': + self.assertRaises(ValueError, duration, invalid) + + def testPromptValuesAreLiteralsOnly(self): + cases = { + '[1, 2]': [1, 2], + '42': 42, + 'plain text': 'plain text', + '__import__("os").getcwd()': '__import__("os").getcwd()', + } + for typed, expected in cases.items(): + with mock.patch('builtins.input', return_value=typed): + self.assertEqual(expand_bracket('count', '{#count}'), expected) + + def testOtp(self): + stdout = io.StringIO() + with ( + mock.patch('getpass.getpass', return_value='password') as getpass, + contextlib.redirect_stdout(stdout), + ): + self.assertEqual(otp_main(['9999', 'abc123']), 0) + getpass.assert_called_once_with('Password: ') + self.assertEqual(stdout.getvalue(), otp('password', 'abc123', 9999) + '\n') + self.assertEqual(stdout.getvalue(), 'ACTA AMMO CAR WEB BIN YAP\n') + + def testOtpWithoutNumber(self): + stderr = io.StringIO() + with ( + mock.patch('getpass.getpass') as getpass, + contextlib.redirect_stderr(stderr), + self.assertRaises(SystemExit) as cm, + ): + otp_main(['abc', 'abc123']) + self.assertEqual(cm.exception.code, 2) + self.assertIn('first argument must be a number', stderr.getvalue()) + getpass.assert_not_called() diff --git a/tests/scripts/exscriptTest.py b/tests/scripts/exscriptTest.py deleted file mode 100644 index 2e324d23..00000000 --- a/tests/scripts/exscriptTest.py +++ /dev/null @@ -1,40 +0,0 @@ -import importlib.machinery -import importlib.util -import os -import unittest -from unittest import mock - -SCRIPT = os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, 'scripts', 'exscript') - - -def load_cli(): - loader = importlib.machinery.SourceFileLoader('exscript_cli', SCRIPT) - spec = importlib.util.spec_from_file_location('exscript_cli', SCRIPT, loader=loader) - module = importlib.util.module_from_spec(spec) - loader.exec_module(module) - return module - - -class exscriptTest(unittest.TestCase): - cli = load_cli() - - def testDuration(self): - duration = self.cli.duration - self.assertEqual(duration(''), 0) - self.assertEqual(duration('90'), 90) - self.assertEqual(duration('1.5m'), 90) - self.assertEqual(duration('1h 30m'), 5400) - self.assertEqual(duration('2d'), 172800) - for invalid in 'tomorrow', '1x', '-1', '1; rm -rf ~': - self.assertRaises(ValueError, duration, invalid) - - def testPromptValuesAreLiteralsOnly(self): - cases = { - '[1, 2]': [1, 2], - '42': 42, - 'plain text': 'plain text', - '__import__("os").getcwd()': '__import__("os").getcwd()', - } - for typed, expected in cases.items(): - with mock.patch('builtins.input', return_value=typed): - self.assertEqual(self.cli.expand_bracket('count', '{#count}'), expected) From db364d8f9a668e574f5bf0f73663857f910cccb7 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 00:47:12 +0100 Subject: [PATCH 158/250] End the exscript command at once on Ctrl+C on every platform Windows has no fork() for SigIntWatcher, so Ctrl+C took effect only after the running connections ended. Once the template runs, a SIGINT handler now ends the process; the main thread waits in short slices, so that the handler runs on Windows, too. Exscript.util.sigint is removed. --- CHANGES.md | 23 +++++---- CONTRIBUTING.md | 5 +- docs/migration.rst | 9 ++-- docs/platforms.rst | 8 +-- pyproject.toml | 18 ------- src/Exscript/cli.py | 61 +++++++++++++++++----- src/Exscript/util/sigint.py | 96 ----------------------------------- tests/Exscript/ScriptsTest.py | 46 ++++++++++++++++- 8 files changed, 117 insertions(+), 149 deletions(-) delete mode 100644 src/Exscript/util/sigint.py diff --git a/CHANGES.md b/CHANGES.md index b87e4995..dcf26a0c 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -20,10 +20,9 @@ marked with **BREAKING**; a migration guide will accompany the release. versions before 3.12 are no longer supported. - **Type Annotations:** Exscript is annotated and ships the `py.typed` marker (PEP 561), so type checkers such as mypy and pyright check the code that uses it. All modules - pass `mypy --strict`, except `Exscript.util.sigint`, which is annotated only in part. - mypy knows the type of 95.7 % of the expressions; the rest is `Any` on purpose - (callbacks, options that are passed on, template variables) or comes from ncclient, - which is not annotated. CI checks that this share does not drop. + pass `mypy --strict`. mypy knows the type of 95.7 % of the expressions; the rest is + `Any` on purpose (callbacks, options that are passed on, template variables) or comes + from ncclient, which is not annotated. CI checks that this share does not drop. - **Dependencies:** `future` and `configparser`, backports for Python 2, are no longer needed. paramiko 3.4 to 5.x is supported (before: 1.17 to 4.x). New: ncclient and lxml 5.0 or later, for NETCONF. @@ -31,7 +30,11 @@ marked with **BREAKING**; a migration guide will accompany the release. console entry points of the package: `Exscript.cli.main()` and `otp_main()`. `python -m Exscript` runs `exscript`, too. On Windows, the commands are `.exe` launchers that work from any shell. Before, they were Python scripts without a file - extension, which cmd.exe and PowerShell do not run. + extension, which cmd.exe and PowerShell do not run. Ctrl+C stops all connections of + `exscript` at once on every platform: it writes `Interrupted.` and exits with status + 130. Before, on Linux and macOS, it ran the connections in a child process, which it + killed on Ctrl+C before it exited with status 1; on Windows, Ctrl+C took effect only + after the running connections ended. - **NETCONF Protocol:** The new adapter `Exscript.protocols.Netconf` (scheme `netconf://`, port 830) sends RPCs over SSH with ncclient, and returns the replies as text or as lxml elements; an `` raises `RPCError`. The IOS XR and Junos @@ -605,10 +608,12 @@ The names below refer to functions callable from templates, not protocol methods and the copies keep the default factory and the order of the keys. Before, `copy.deepcopy()` failed with `TypeError: cannot pickle 'odict_items' object`, and `pickle.dumps()` with a `PicklingError`. -- **BREAKING - Removed Modules:** `Exscript.util.daemonize`, `Exscript.util.pidutil` - and `Exscript.util.sigintcatcher` are removed, and - `daemonize()` failed on Python 3 with `ValueError: can't have unbuffered text I/O`. - Run long-lived scripts under a service manager such as systemd instead. +- **BREAKING - Removed Modules:** `Exscript.util.daemonize`, `Exscript.util.pidutil`, + `Exscript.util.sigint` and `Exscript.util.sigintcatcher` are removed; `daemonize()` + failed on Python 3 with `ValueError: can't have unbuffered text I/O`. Run long-lived scripts under a service manager such as systemd + instead. To stop the connections of a queue at once on Ctrl+C, as `SigIntWatcher` of + `Exscript.util.sigint` did with a child process, end the process with `os._exit()` in + a handler of `signal.SIGINT`. ## Multi-Threading diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2c07908b..4dcb76a6 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -52,9 +52,8 @@ CI also measures three kinds of coverage, each with a minimum that only goes up: - Document new functions, classes and modules with docstrings in the style of the existing code (`:param:`, `:type:`, `:return:`, `:rtype:`). The API reference is generated from them, and pre-commit fails if the share of documented functions drops. -- Annotate new code with types. mypy checks every module strictly, except those that are - still in the list of `[[tool.mypy.overrides]]` in `pyproject.toml`; remove a module from - that list when you annotate it. Use `Any` only where a type is open on purpose. +- Annotate new code with types; mypy checks every module strictly. Use `Any` only where a + type is open on purpose. - The public API and the template language stay compatible. Please open an issue before you work on a change that breaks them. diff --git a/docs/migration.rst b/docs/migration.rst index 90cd4017..9dc43180 100644 --- a/docs/migration.rst +++ b/docs/migration.rst @@ -200,9 +200,12 @@ Other renamed or removed names: ``check_received_data()`` and ``check_protocol_info()``. - ``Exscript.util.template.test_secure()`` is ``test(secure=True)``. - ``Exscript.util.interact.InputHistory.file`` is ``filename``. -- ``Exscript.util.daemonize``, ``Exscript.util.pidutil`` and - ``Exscript.util.sigintcatcher`` are removed. Run long-lived scripts - under a service manager such as systemd. +- ``Exscript.util.daemonize``, ``Exscript.util.pidutil``, + ``Exscript.util.sigint`` and ``Exscript.util.sigintcatcher`` are + removed. Run long-lived scripts under a service manager such as systemd. + To stop the connections of a queue at once on Ctrl+C, as + ``SigIntWatcher`` did, end the process with ``os._exit()`` in a handler + of ``signal.SIGINT``. - ``WorkQueue`` has no ``debug`` argument, and ``set_debug()`` takes ``level``. Pass the other arguments by keyword. - ``Queue(mode='multiprocessing')`` raises a ``ValueError``: jobs run in diff --git a/docs/platforms.rst b/docs/platforms.rst index 170c8517..bfb9c6ec 100644 --- a/docs/platforms.rst +++ b/docs/platforms.rst @@ -8,8 +8,8 @@ Python 3.14. Most of Exscript works the same on every platform: for example, it reads and writes text files such as templates, host lists and logs as UTF-8 -everywhere, and gives sessions the same terminal size. These are the -differences. +everywhere, gives sessions the same terminal size, and its command line +tool stops all connections at once on Ctrl+C. These are the differences. Windows ------- @@ -19,10 +19,6 @@ Windows completion and control keys therefore do not reach the device, and a typed line may appear twice, because both the console and the device echo it. -- On Linux and macOS, the command line tool runs the connections in a - child process, so that Ctrl+C stops all of them at once. Windows has no - ``fork()``, so there Ctrl+C may take effect only after the running - connections end. - The template function ``sys.exec`` passes the command line to Windows without a shell. Commands that are built into the shell need ``cmd /c``, for example ``sys.exec("cmd /c dir")``. diff --git a/pyproject.toml b/pyproject.toml index 1c7a0383..f6310f5e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -127,24 +127,6 @@ files = ["src"] strict = true ignore_missing_imports = true # ncclient and others ship no type hints -[[tool.mypy.overrides]] -# Modules that are not annotated yet (P8). Remove a module from this list when -# it passes the strict checks; the list only shrinks. -module = [ - "Exscript.util.sigint", -] -disallow_any_generics = false -disallow_subclassing_any = false -disallow_untyped_calls = false -disallow_untyped_defs = false -disallow_incomplete_defs = false -check_untyped_defs = false -disallow_untyped_decorators = false -warn_return_any = false -implicit_reexport = true -strict_equality = false -extra_checks = false - [[tool.mypy.overrides]] # BaseInstrumentor comes from a module that mypy skips (it starts with "# type: ignore"). module = "Exscript.contrib.otel" diff --git a/src/Exscript/cli.py b/src/Exscript/cli.py index abd41250..9dc4a334 100644 --- a/src/Exscript/cli.py +++ b/src/Exscript/cli.py @@ -35,10 +35,13 @@ import importlib.util import os import re +import signal import sys import time from collections.abc import Sequence from pathlib import Path +from types import FrameType +from typing import NoReturn import Exscript.protocols.drivers from Exscript import Account, Host, PrivateKey, Queue @@ -56,7 +59,6 @@ ) from Exscript.util.interact import get_login from Exscript.util.log import log_to_file -from Exscript.util.sigint import SigIntWatcher from Exscript.workqueue.job import Thread bracket_expression = r"\{([^\]]*)\}" @@ -139,6 +141,33 @@ def expand_host_variables(host: Host) -> None: host.set(key, value) +def _exit_on_sigint(signum: int, frame: FrameType | None) -> NoReturn: + """ + The handler of SIGINT (Ctrl+C) while the template runs: writes + "Interrupted." to stderr, and ends the process at once with exit status + 130. A KeyboardInterrupt would end it only after the threads of the + running connections, which cannot be interrupted. + """ + try: + sys.stdout.flush() + sys.stderr.write('Interrupted.\n') + sys.stderr.flush() + finally: + os._exit(130) + + +def _join(queue: Queue) -> None: + """ + Waits until the given queue has completed all jobs, like queue.join(), + but in short slices, so that the main thread handles Ctrl+C meanwhile: + on Windows, Ctrl+C does not interrupt a blocking wait for a lock before + Python 3.14. + """ + while not queue.is_completed(): + time.sleep(0.2) + queue.join() + + def run(parser: argparse.ArgumentParser, args: argparse.Namespace) -> int: """ Runs the template on the hosts as the given command line arguments say, @@ -410,15 +439,22 @@ def function(job: Thread, host: Host, conn: Protocol, **kwargs: object) -> objec decorator = log_to_file(args.logdir, mode, args.delete_logs) function = decorator(function) - # Wait until the specified time. - if args.sleep: - print(f"Waiting for {args.sleep:g} seconds...", end="", flush=True) - time.sleep(args.sleep) - print("time expired, starting script.") - - # Run the template. - queue.run(hosts, function, attempts=args.retry + 1) - queue.join() + # Ctrl+C ends the process at once from here on, see _exit_on_sigint(). + # Not before: while getpass() reads a password, the terminal does not + # echo, and only a KeyboardInterrupt lets getpass() turn the echo on again. + sigint_handler = signal.signal(signal.SIGINT, _exit_on_sigint) + try: + # Wait until the specified time. + if args.sleep: + print(f"Waiting for {args.sleep:g} seconds...", end="", flush=True) + time.sleep(args.sleep) + print("time expired, starting script.") + + # Run the template. + queue.run(hosts, function, attempts=args.retry + 1) + _join(queue) + finally: + signal.signal(signal.SIGINT, sigint_handler) failed = queue.failed queue.destroy(force=True) return failed @@ -876,7 +912,9 @@ def create_parser() -> argparse.ArgumentParser: def main(argv: list[str] | None = None) -> int: """ - Runs the command line tool ``exscript`` with the given arguments. + Runs the command line tool ``exscript`` with the given arguments. Once + the template runs, Ctrl+C ends the process at once with exit status 130, + also while connections are running. :type argv: list[str] | None :param argv: The arguments without the name of the program; by default @@ -886,7 +924,6 @@ def main(argv: list[str] | None = None) -> int: else 1. Invalid arguments and ``--help`` end the process through SystemExit, as usual with argparse. """ - SigIntWatcher() parser = create_parser() failed = run(parser, parser.parse_args(argv)) if failed != 0: diff --git a/src/Exscript/util/sigint.py b/src/Exscript/util/sigint.py deleted file mode 100644 index 184b37da..00000000 --- a/src/Exscript/util/sigint.py +++ /dev/null @@ -1,96 +0,0 @@ -# -# Copyright (C) 2010-2017 Samuel Abels -# The MIT License (MIT) -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files -# (the "Software"), to deal in the Software without restriction, -# including without limitation the rights to use, copy, modify, merge, -# publish, distribute, sublicense, and/or sell copies of the Software, -# and to permit persons to whom the Software is furnished to do so, -# subject to the following conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY -# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, -# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE -# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -""" -A class for catching SIGINT, such that CTRL+c works. -""" - -import os -import signal -import sys -import time - - -class SigIntWatcher: - """ - This class solves two problems with multithreaded programs in Python: - - - A signal might be delivered to any thread and - - if the thread that gets the signal is waiting, the signal - is ignored (which is a bug). - - This class forks and catches sigint for Exscript. - - The watcher is a concurrent process (not thread) that waits for a - signal and the process that contains the threads. - Works on Linux, Solaris, MacOS, and AIX. Known not to work - on Windows. - """ - - def __init__(self) -> None: - """ - Creates a child process, which returns. The parent - process waits for a KeyboardInterrupt and then kills - the child process. - """ - try: - self.child = os.fork() - except AttributeError: # platforms that don't have os.fork - self.child = 0 - except RuntimeError: # prevent "not holding the import lock" on some systems - self.child = 0 - if self.child: - self._watch() - - def _watch(self): - try: - pid, status = os.wait() - except KeyboardInterrupt: - print('********** SIGINT RECEIVED - SHUTTING DOWN! **********') - # Child may also receive SIGINT, so we wait a bit for its cleanup. - deadline = time.monotonic() + 2.0 - while time.monotonic() < deadline: - try: - pid, status = os.waitpid(self.child, os.WNOHANG) - if pid: # child exited on its own - self._exit_from_status(status) - time.sleep(0.05) - except ChildProcessError: - sys.exit(1) # child already gone - except KeyboardInterrupt: - pass # second CTRL+C -> simply continue waiting - self._kill() - sys.exit(1) - self._exit_from_status(status) - - def _kill(self): - try: - os.kill(self.child, signal.SIGKILL) - except OSError: - pass - - def _exit_from_status(self, status): - if os.WIFEXITED(status): - sys.exit(os.WEXITSTATUS(status)) - else: - # Follows shell convention for signal-killed child processes. - sys.exit(128 + os.WTERMSIG(status)) diff --git a/tests/Exscript/ScriptsTest.py b/tests/Exscript/ScriptsTest.py index cb55c38f..8783ee97 100644 --- a/tests/Exscript/ScriptsTest.py +++ b/tests/Exscript/ScriptsTest.py @@ -1,7 +1,10 @@ import os +import selectors +import signal import subprocess import sys import tempfile +import time import unittest BASEDIR = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..')) @@ -30,18 +33,35 @@ class ScriptsTest(unittest.TestCase): def setUp(self): self.tmpdir = tempfile.TemporaryDirectory() self.addCleanup(self.tmpdir.cleanup) + self.env = dict( + os.environ, HOME=self.tmpdir.name, USERPROFILE=self.tmpdir.name, NO_COLOR='1' + ) def run_exscript(self, *args, python_args=('-m', 'Exscript')): - env = dict(os.environ, HOME=self.tmpdir.name, USERPROFILE=self.tmpdir.name, NO_COLOR='1') proc = subprocess.run( # noqa: S603 - runs Exscript with fixed arguments [sys.executable, *python_args, *args], capture_output=True, encoding='utf-8', - env=env, + env=self.env, timeout=60, ) return proc.returncode, proc.stdout + proc.stderr + def read_until(self, pipe, marker, timeout=30): + """Reads from the given pipe until the given marker, and returns what it read.""" + data = b'' + deadline = time.monotonic() + timeout + with selectors.DefaultSelector() as selector: + selector.register(pipe, selectors.EVENT_READ) + while marker not in data: + if not selector.select(deadline - time.monotonic()): + self.fail(f'no {marker!r} within {timeout} seconds: {data!r}') + chunk = os.read(pipe.fileno(), 4096) + if not chunk: + self.fail(f'no {marker!r} before the end of the output: {data!r}') + data += chunk + return data + def write_template(self, text): path = os.path.join(self.tmpdir.name, 'test.exscript') with open(path, 'w', encoding='utf-8') as fp: @@ -82,6 +102,28 @@ def testSshKeyWithoutUserName(self): self.assertEqual(code, 0, output) self.assertIn('hello', output) + @unittest.skipIf(sys.platform == 'win32', 'Windows cannot send SIGINT to a process') + def testCtrlC(self): + # The job waits for a minute in a thread, which Ctrl+C cannot interrupt. + template = self.write_template('{sys.message("started")}{sys.wait(60)}\n') + args = '--non-interactive', '--no-authentication', template, 'dummy://h1' + proc = subprocess.Popen( # noqa: S603 - runs Exscript with fixed arguments + [sys.executable, '-u', '-m', 'Exscript', *args], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=self.env, + ) + self.addCleanup(proc.stderr.close) + self.addCleanup(proc.stdout.close) + self.addCleanup(proc.wait) + self.addCleanup(proc.kill) + output = self.read_until(proc.stdout, b'started') + + proc.send_signal(signal.SIGINT) + stdout, stderr = proc.communicate(timeout=10) + self.assertEqual(proc.returncode, 130, output + stdout + stderr) + self.assertIn(b'Interrupted.', stderr) + def suite(): return unittest.TestLoader().loadTestsFromTestCase(ScriptsTest) From 521fec214ea88d9f8940815dc3575eec279a7c15 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 00:48:45 +0100 Subject: [PATCH 159/250] Restore the --version option of the exscript command The tutorials document it, but it got lost in the port to argparse. --- CHANGES.md | 3 +++ src/Exscript/cli.py | 8 +++++--- tests/Exscript/ScriptsTest.py | 7 +++++++ 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index dcf26a0c..66fdeb6a 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -589,6 +589,9 @@ The names below refer to functions callable from templates, not protocol methods for the current user. If the name of the current user is unknown, it adds none, as without a default user name. Before, the CLI failed with the `OSError` of `getpass.getuser()`, or with a `KeyError` before Python 3.13. +- **`exscript --version`:** Prints the version of Exscript, as in 2.6 and as the + tutorials say. The 2.9 development versions failed with + `exscript: error: unrecognized arguments: --version`. - **`Exscript.util.mail.send()`:** Ends the SMTP session with `QUIT` and closes the connection, also if sending fails; before, it left the connection open. It connects only after the mail and its attachments are ready. diff --git a/src/Exscript/cli.py b/src/Exscript/cli.py index 9dc4a334..1554868d 100644 --- a/src/Exscript/cli.py +++ b/src/Exscript/cli.py @@ -44,7 +44,7 @@ from typing import NoReturn import Exscript.protocols.drivers -from Exscript import Account, Host, PrivateKey, Queue +from Exscript import Account, Host, PrivateKey, Queue, __version__ from Exscript.logger import setupLogging from Exscript.protocols import Protocol, protocol_map from Exscript.util import template @@ -520,6 +520,8 @@ def create_parser() -> argparse.ArgumentParser: prog="exscript", epilog="\N{COPYRIGHT SIGN} 2007-2023 by Samuel Abels." ) + parser.add_argument("--version", action="version", version=__version__) + parser.add_argument( "filename", default=None, @@ -921,8 +923,8 @@ def main(argv: list[str] | None = None) -> int: those of the process. :rtype: int :return: The exit status: 0 if the template succeeded on all hosts, - else 1. Invalid arguments and ``--help`` end the process through - SystemExit, as usual with argparse. + else 1. Invalid arguments, ``--help`` and ``--version`` end the + process through SystemExit, as usual with argparse. """ parser = create_parser() failed = run(parser, parser.parse_args(argv)) diff --git a/tests/Exscript/ScriptsTest.py b/tests/Exscript/ScriptsTest.py index 8783ee97..98e71182 100644 --- a/tests/Exscript/ScriptsTest.py +++ b/tests/Exscript/ScriptsTest.py @@ -7,6 +7,8 @@ import time import unittest +import Exscript + BASEDIR = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..')) KEYFILE = os.path.join(BASEDIR, 'tests', 'Exscript', 'protocols', 'id_rsa') @@ -75,6 +77,11 @@ def testHelp(self): for option in '--load-driver', '--insecure', '--ssh-host-key-policy': self.assertIn(option, output) + def testVersion(self): + code, output = self.run_exscript('--version') + self.assertEqual(code, 0, output) + self.assertEqual(output, Exscript.__version__ + '\n') + def testRunTemplate(self): template = self.write_template('{sys.message("hello")}\n') args = '--non-interactive', '--no-authentication', template, 'dummy://r1' From 7e7b9dcc493b155918967ccfa16f4f94fc1f1a5f Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 01:11:12 +0100 Subject: [PATCH 160/250] Require a docstring for every module, class and function All of them have one now, so the minimum of interrogate rises to 100 %. --- CONTRIBUTING.md | 2 +- pyproject.toml | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4dcb76a6..1df282a2 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -51,7 +51,7 @@ CI also measures three kinds of coverage, each with a minimum that only goes up: **BREAKING**, and say how to migrate. - Document new functions, classes and modules with docstrings in the style of the existing code (`:param:`, `:type:`, `:return:`, `:rtype:`). The API reference is generated from - them, and pre-commit fails if the share of documented functions drops. + them, and pre-commit fails if a module, class or function has none. - Annotate new code with types; mypy checks every module strictly. Use `Any` only where a type is open on purpose. - The public API and the template language stay compatible. Please open an issue before you diff --git a/pyproject.toml b/pyproject.toml index f6310f5e..769f7091 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -133,10 +133,10 @@ module = "Exscript.contrib.otel" disallow_subclassing_any = false [tool.interrogate] -# Docstring coverage (P8). Private functions count; other dunder methods than -# __init__, nested functions and property setters do not. Raise fail-under as -# docstrings are added; the goal is 100. -fail-under = 93 +# Docstring coverage: every module, class and function has a docstring. +# Private functions count; other dunder methods than __init__, nested +# functions and property setters do not. +fail-under = 100 ignore-magic = true ignore-nested-functions = true ignore-setters = true From b8be8b69c55ef6db2d3da0a423523d28fae56a5d Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 01:14:11 +0100 Subject: [PATCH 161/250] Measure the coverage of subprocesses that the tests start pytest-cov 7 no longer measures them, so the tests that run the exscript command as a subprocess added nothing to the coverage. coverage's own patch = ["subprocess"] measures them. --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 769f7091..a4efc47a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -144,6 +144,7 @@ exclude = ["docs", "tests", "tools", "demos"] [tool.coverage.run] branch = true +patch = ["subprocess"] # e.g. python -m Exscript in the tests of the command [tool.pytest.ini_options] testpaths = ["tests"] From 478a57e344ec25d84f5bf9ca89645eaefd8cd0c3 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 01:14:03 +0100 Subject: [PATCH 162/250] Mark the change of queue_empty_event as incompatible The event is emitted when the last job has ended, and no longer when the queue stops. The migration guide says to continue after shutdown() or join() returns instead. --- CHANGES.md | 6 +++--- docs/migration.rst | 4 ++++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 66fdeb6a..5ed59e35 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -658,9 +658,9 @@ The names below refer to functions callable from templates, not protocol methods - **Jobs Connect With `host_driver`:** The `host_driver` of `Queue` is set on each host before its job is enqueued. Before, it was set after, so a job that started at once could connect without it, with the driver that Exscript guessed. -- **`WorkQueue.queue_empty_event`:** Is emitted when the queue becomes empty, when the - last queued or running job has ended. Before, it was emitted only when the main loop - stopped, on `shutdown()` or `destroy()` of an empty queue. +- **BREAKING - `WorkQueue.queue_empty_event`:** Is emitted when the queue becomes empty, + when the last queued or running job has ended. Before, it was emitted only when the + main loop stopped, on `shutdown()` or `destroy()` of an empty queue. - **Status Bar on Narrow Terminals:** If the names of the running jobs do not fit into the status bar of `Queue`, it cuts them in the middle, at most down to `...`. Before, on a terminal of about 30 columns or less, the cut overlapped and made the bar longer diff --git a/docs/migration.rst b/docs/migration.rst index 9dc43180..55bb55f6 100644 --- a/docs/migration.rst +++ b/docs/migration.rst @@ -212,6 +212,10 @@ Other renamed or removed names: threads only. For several CPU cores, see :doc:`platforms`. - ``Exscript.account.AccountProxy`` and ``Exscript.logger.LoggerProxy`` are removed; jobs use the accounts and the loggers directly. +- ``WorkQueue.queue_empty_event`` is emitted when the last job of the queue + has ended, and no longer when the queue stops. Code that used it to learn + that the queue stopped continues after ``shutdown()`` or ``join()`` + returns instead. - ``parselib.Lexer.expect()`` and ``current_is()`` take ``types`` instead of ``type``. - The ``dump1()`` methods of the template interpreter are removed, and so From 2b1b08ec5e537e3e3f77a0f1dd1a37e2606d2194 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 01:16:00 +0100 Subject: [PATCH 163/250] Count only the enqueued jobs in the progress of the queue The queue also counted the hosts that run_or_ignore() and other variants of run() did not enqueue, so the progress stayed below 100 %. It now counts each job as it enqueues it, under the lock of the pipeline. --- CHANGES.md | 6 ++++++ src/Exscript/queue.py | 10 +++++----- tests/Exscript/QueueTest.py | 12 ++++++++++++ 3 files changed, 23 insertions(+), 5 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 5ed59e35..bb44effb 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -661,6 +661,12 @@ The names below refer to functions callable from templates, not protocol methods - **BREAKING - `WorkQueue.queue_empty_event`:** Is emitted when the queue becomes empty, when the last queued or running job has ended. Before, it was emitted only when the main loop stopped, on `shutdown()` or `destroy()` of an empty queue. +- **Progress Counts Only Enqueued Jobs:** `Queue.get_progress()` and the status bar + count only the jobs that the queue enqueued, so the progress reaches 100 % when they + have ended. Before, they also counted the hosts that `run_or_ignore()` and + `priority_run_or_raise()` did not enqueue as they were queued already, and those that + `priority_run()` and `force_run()` did not enqueue as they raised an `AttributeError` + for a queued host. - **Status Bar on Narrow Terminals:** If the names of the running jobs do not fit into the status bar of `Queue`, it cuts them in the middle, at most down to `...`. Before, on a terminal of about 30 columns or less, the cut overlapped and made the bar longer diff --git a/src/Exscript/queue.py b/src/Exscript/queue.py index b0d868c3..ab33de9e 100644 --- a/src/Exscript/queue.py +++ b/src/Exscript/queue.py @@ -613,14 +613,16 @@ def _enqueue( ) -> None: """ Enqueues a job by calling the given function of the work queue with - the given arguments, and adds the id of the job to the given task, - unless the function ignored the job. Holds the lock of the pipeline - meanwhile, so the job cannot start before the task knows its id. + the given arguments; unless the function ignored the job, counts it + in total and adds its id to the given task. Holds the lock of the + pipeline meanwhile, so the job cannot start before it is counted and + the task knows its id. """ def enqueue(pipeline: 'Pipeline') -> None: job_id = queue_function(*args, **kwargs) if job_id is not None: + self.total += 1 task.add_job_id(job_id) self.workqueue.collection.with_lock(enqueue) @@ -642,7 +644,6 @@ def _run( :return: The task of the jobs, or None if no job was enqueued. """ hosts = to_hosts(hosts, default_domain=self.domain) - self.total += len(hosts) callback = _prepare_connection(callback, self.account_manager) task = Task(self.workqueue) for host in hosts: @@ -787,7 +788,6 @@ def enqueue( :rtype: Task :return: An object representing the task. """ - self.total += 1 task = Task(self.workqueue) self._enqueue(task, self.workqueue.enqueue, function, name, attempts) self._dbg(2, 'Function enqueued.') diff --git a/tests/Exscript/QueueTest.py b/tests/Exscript/QueueTest.py index 190b1b34..c6eccd72 100644 --- a/tests/Exscript/QueueTest.py +++ b/tests/Exscript/QueueTest.py @@ -450,6 +450,18 @@ def add_and_end(task, job_id): self.assertIsNone(self.queue.run_or_ignore('dummy://dummy2', do_nothing)) self.queue.workqueue.unpause() + def testProgressCountsOnlyEnqueuedJobs(self): + self.queue.workqueue.pause() + self.queue.run_or_ignore('dummy://dummy1', do_nothing) + self.queue.run_or_ignore('dummy://dummy1', do_nothing) # ignored + self.queue.priority_run_or_raise('dummy://dummy1', do_nothing) # moves it + hosts = ['dummy://dummy2', 'dummy://dummy1', 'dummy://dummy3'] + with self.assertRaises(AttributeError): + self.queue.priority_run(hosts, do_nothing) # enqueues only dummy2 + self.queue.workqueue.unpause() + self.queue.join() + self.assertEqual(self.queue.get_progress(), 100.0) + def testJobsConnectWithTheHostDriver(self): # Each job ends before the driver is set on its host, if it can start # then. From 8272c7f5e096ef3471df154c86749d503655ef5c Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 01:29:22 +0100 Subject: [PATCH 164/250] Count jobs that end at the same time under a lock Task and Queue counted ended jobs without a lock, so two jobs that ended at the same time could lose a count, and their task never completed. Both now count under a lock, so only the last job emits done_event. --- CHANGES.md | 6 ++ src/Exscript/queue.py | 10 +++- src/Exscript/workqueue/task.py | 12 +++- tests/Exscript/QueueTest.py | 79 ++++++++++++++++++++++++-- tests/Exscript/workqueue/TaskTest.py | 84 ++++++++++++++++++++++++++++ 5 files changed, 180 insertions(+), 11 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index bb44effb..266e900d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -667,6 +667,12 @@ The names below refer to functions callable from templates, not protocol methods `priority_run_or_raise()` did not enqueue as they were queued already, and those that `priority_run()` and `force_run()` did not enqueue as they raised an `AttributeError` for a queued host. +- **Jobs That End at the Same Time:** `Task` and `Queue` count the jobs that end at the + same time under a lock. Before, two of them could be counted as one, mostly on the + free-threaded build: the task then never completed, as `Task.is_completed()` stayed + `False` and `done_event` never came, and the progress and the count of failed jobs of + the queue stayed too low. And if the last two jobs of a task ended at the same time, + both could emit `done_event`. - **Status Bar on Narrow Terminals:** If the names of the running jobs do not fit into the status bar of `Queue`, it cuts them in the middle, at most down to `...`. Before, on a terminal of about 30 columns or less, the cut overlapped and made the bar longer diff --git a/src/Exscript/queue.py b/src/Exscript/queue.py index ab33de9e..9c2ed7bb 100644 --- a/src/Exscript/queue.py +++ b/src/Exscript/queue.py @@ -256,6 +256,8 @@ def __init__( self.stderr = stderr self.host_driver = host_driver self.exc_cb = exc_cb + # For the output, completed and failed; _enqueue() counts total under + # the lock of the pipeline. self.lock = threading.RLock() self.devnull = _Discard() self.channel_map = {'fatal_errors': self.stderr} @@ -423,7 +425,8 @@ def _on_job_succeeded(self, job: 'Thread') -> None: Counts the given job as completed, and reports its success on the status bar channel. """ - self.completed += 1 + with self.lock: + self.completed += 1 self._del_status_bar() self._write('status_bar', job.name + ' succeeded.' + '\n') self._print_status_bar(exclude=job.name) @@ -433,8 +436,9 @@ def _on_job_aborted(self, job: 'Thread') -> None: Counts the given job as completed and failed, after its last attempt failed, and reports it on the errors channel. """ - self.completed += 1 - self.failed += 1 + with self.lock: + self.completed += 1 + self.failed += 1 self._del_status_bar() self._write('errors', job.name + ' finally failed.' + '\n') self._print_status_bar(exclude=job.name) diff --git a/src/Exscript/workqueue/task.py b/src/Exscript/workqueue/task.py index 0e9d643a..5fc0717b 100644 --- a/src/Exscript/workqueue/task.py +++ b/src/Exscript/workqueue/task.py @@ -24,6 +24,7 @@ Represents a batch of enqueued actions. """ +import threading from typing import TYPE_CHECKING from Exscript.util.event import Event @@ -50,18 +51,23 @@ def __init__(self, workqueue: 'WorkQueue') -> None: self.workqueue = workqueue self.job_ids: set[str] = set() self.completed = 0 + self.lock = threading.Lock() # for completed; jobs end in several threads self.workqueue.job_succeeded_event.listen(self._on_job_done) self.workqueue.job_aborted_event.listen(self._on_job_done) def _on_job_done(self, job: 'Thread') -> None: """ Counts the given job as completed if it belongs to the task, and - emits done_event when all jobs of the task are completed. + emits done_event when all jobs of the task are completed. Counts and + checks under the lock, so that jobs that end at the same time are + all counted, and only the last one emits done_event. """ if job.id not in self.job_ids: return - self.completed += 1 - if self.is_completed(): + with self.lock: + self.completed += 1 + done = self.is_completed() + if done: self.done_event() def is_completed(self) -> bool: diff --git a/tests/Exscript/QueueTest.py b/tests/Exscript/QueueTest.py index c6eccd72..76482d98 100644 --- a/tests/Exscript/QueueTest.py +++ b/tests/Exscript/QueueTest.py @@ -26,13 +26,13 @@ logger.addHandler(logging.NullHandler()) -def holds_lock(queue): +def holds_lock(lock): """ - Returns whether the current thread holds the lock of the pipeline of the - given queue, which keeps the main loop from taking a job. + Returns whether the current thread holds the given lock, or the lock of + the given condition; a Lock counts as held if any thread holds it. """ try: - queue.workqueue.collection.condition.notify(0) # raises unless the lock is held + threading.Condition(lock).notify(0) # raises unless the lock is held except RuntimeError: return False return True @@ -44,10 +44,37 @@ def end_now(queue, job_id): taken it at once: unless the current thread holds the lock of the pipeline, which keeps the main loop from taking it. """ - if not holds_lock(queue): + if not holds_lock(queue.workqueue.collection.condition): queue.workqueue.wait_for(job_id) +class Interleave: + """ + Replaces the counter with the given name of an object. The first read + of the counter in each of the given threads waits until all of them read + it, unless the thread holds the given lock then. So without the lock, + all threads read the old value before any writes it. + """ + + def __init__(self, name, threads, lock): + self.name = name + self.threads = set(threads) + self.lock = lock + self.barrier = threading.Barrier(len(self.threads), timeout=10) + + def __get__(self, obj, cls=None): + value = obj.__dict__[self.name] + thread = threading.current_thread() + if thread in self.threads: + self.threads.discard(thread) + if not holds_lock(self.lock): + self.barrier.wait() + return value + + def __set__(self, obj, value): + obj.__dict__[self.name] = value + + def count_calls(job, data, **kwargs): assert hasattr(job, 'start') assert 'testarg' in kwargs @@ -462,6 +489,48 @@ def testProgressCountsOnlyEnqueuedJobs(self): self.queue.join() self.assertEqual(self.queue.get_progress(), 100.0) + def testCountsJobsThatEndAtTheSameTime(self): + # Both jobs read each count before either writes it, unless the queue + # locks the counts. + expected = { + 'job_succeeded_event': {'completed': 2, 'failed': 0}, + 'job_aborted_event': {'completed': 2, 'failed': 2}, + } + for event, counts in expected.items(): + with self.subTest(event=event): + self.createQueue(verbose=-1) + emit = getattr(self.queue.workqueue, event) + jobs = [SimpleNamespace(name='job%d' % i) for i in range(2)] + threads = [threading.Thread(target=emit, args=(job,)) for job in jobs] + completed = Interleave('completed', threads, self.queue.lock) + failed = Interleave('failed', threads, self.queue.lock) + with ( + mock.patch.object(Queue, 'completed', completed, create=True), + mock.patch.object(Queue, 'failed', failed, create=True), + ): + for thread in threads: + thread.start() + for thread in threads: + thread.join() + self.assertEqual( + {'completed': self.queue.completed, 'failed': self.queue.failed}, counts + ) + + def testCountsJobsThatAreEnqueuedAtTheSameTime(self): + # Both runs read the count before either writes it, unless they count + # under the lock of the pipeline. + self.queue.workqueue.pause() + hosts = ['dummy://dummy1', 'dummy://dummy2'] + threads = [threading.Thread(target=self.queue.run, args=(h, do_nothing)) for h in hosts] + total = Interleave('total', threads, self.queue.workqueue.collection.condition) + with mock.patch.object(Queue, 'total', total, create=True): + for thread in threads: + thread.start() + for thread in threads: + thread.join() + self.queue.workqueue.unpause() + self.assertEqual(self.queue.total, 2) + def testJobsConnectWithTheHostDriver(self): # Each job ends before the driver is set on its host, if it can start # then. diff --git a/tests/Exscript/workqueue/TaskTest.py b/tests/Exscript/workqueue/TaskTest.py index dc3eea5b..75e79c39 100644 --- a/tests/Exscript/workqueue/TaskTest.py +++ b/tests/Exscript/workqueue/TaskTest.py @@ -1,5 +1,7 @@ +import threading import unittest import warnings +from unittest import mock warnings.simplefilter('ignore', DeprecationWarning) @@ -7,6 +9,54 @@ from Exscript.workqueue.job import Thread +def holds_lock(lock): + """ + Returns whether the current thread holds the given lock; a Lock counts + as held if any thread holds it. + """ + try: + threading.Condition(lock).notify(0) # raises unless the lock is held + except RuntimeError: + return False + return True + + +class Interleave: + """ + Replaces the counter with the given name of an object. The first read, + or write, of the counter in each of the given threads waits until all of + them got there, unless the thread holds the lock of the object then. So + without the lock, all threads read the old value before any writes it, + or all write before any reads it again. + """ + + def __init__(self, name, threads, after='read'): + self.name = name + self.threads = set(threads) + self.after = after + self.barrier = threading.Barrier(len(self.threads), timeout=10) + + def wait(self, obj): + thread = threading.current_thread() + if thread not in self.threads: + return + self.threads.discard(thread) + lock = getattr(obj, 'lock', None) + if lock is None or not holds_lock(lock): + self.barrier.wait() + + def __get__(self, obj, cls=None): + value = obj.__dict__[self.name] + if self.after == 'read': + self.wait(obj) + return value + + def __set__(self, obj, value): + obj.__dict__[self.name] = value + if self.after == 'write': + self.wait(obj) + + class TaskTest(unittest.TestCase): CORRELATE = Task @@ -42,6 +92,40 @@ def testWait(self): def testAddJobId(self): self.testWait() + def endJobsTogether(self, after): + """ + Lets two jobs of a task end at the same time, with the given point of + Interleave, and returns the task and the list of its done events. + """ + task = Task(self.wq) + done = [] + task.done_event.connect(lambda: done.append(True)) + jobs = [Thread(after + str(i), object, 'job%d' % i, None) for i in range(2)] + for job in jobs: + task.add_job_id(job.id) + threads = [ + threading.Thread(target=self.wq.job_succeeded_event, args=(job,)) for job in jobs + ] + with mock.patch.object( + Task, 'completed', Interleave('completed', threads, after), create=True + ): + for thread in threads: + thread.start() + for thread in threads: + thread.join() + return task, done + + def testJobsThatEndAtTheSameTime(self): + # 'read' makes both jobs read the count before either writes it, and + # 'write' makes both write it before either checks it, unless the + # task locks the count. + for after in ('read', 'write'): + with self.subTest(after=after): + task, done = self.endJobsTogether(after) + self.assertEqual(task.completed, 2) + self.assertTrue(task.is_completed()) + self.assertEqual(done, [True]) + def suite(): return unittest.TestLoader().loadTestsFromTestCase(TaskTest) From f6e4057712433e01f8211d667994adc3576a7c67 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 01:42:42 +0100 Subject: [PATCH 165/250] Give the NETCONF test sessions more time on loaded machines The SSH handshake, login, hello and RPC had to finish within 1 s, which is tight under load. Successful sessions do not wait for the timeout, so the tests are not slower. --- tests/Exscript/protocols/NetconfTest.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/Exscript/protocols/NetconfTest.py b/tests/Exscript/protocols/NetconfTest.py index 24607220..645ed89f 100644 --- a/tests/Exscript/protocols/NetconfTest.py +++ b/tests/Exscript/protocols/NetconfTest.py @@ -487,7 +487,8 @@ def operation(rpc): class SessionTest(unittest.TestCase): """Sessions with a NETCONF 1.0 server.""" - def login(self, reply=None, driver=None, timeout=1, **kwargs): + # Sessions that succeed do not wait for the timeout; tests of timeouts pass a short one. + def login(self, reply=None, driver=None, timeout=10, **kwargs): self.server = NetconfServer(self, reply, **kwargs) conn = Netconf( timeout=timeout, @@ -735,7 +736,7 @@ def testHugeTree(self): content = '' + '' * depth + '' * depth + '' server = NetconfServer(self, lambda rpc: content) conn = Netconf( - timeout=1, + timeout=10, connect_rate_limit=0, verify_fingerprint=False, preferred_authentications=['password'], @@ -799,7 +800,7 @@ def setUp(self): self.addCleanup(server.close) self.stdout = io.StringIO() self.conn = Netconf( - timeout=2, + timeout=10, connect_rate_limit=0, verify_fingerprint=False, preferred_authentications=['password'], From bef7daace6af03ee5ad980bca7b646ab908ffc1a Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 01:18:00 +0100 Subject: [PATCH 166/250] Make the helpers of Exscript.cli private Its API is main() and otp_main(), but the helpers kept their public names from the script, so the API reference documented them. _duration() keeps the name "duration", which argparse shows in its errors. --- src/Exscript/cli.py | 58 +++++++++++++++++++---------------- tests/Exscript/ScriptsTest.py | 5 +++ tests/Exscript/cliTest.py | 16 +++++----- 3 files changed, 44 insertions(+), 35 deletions(-) diff --git a/src/Exscript/cli.py b/src/Exscript/cli.py index 1554868d..163d0aba 100644 --- a/src/Exscript/cli.py +++ b/src/Exscript/cli.py @@ -61,15 +61,15 @@ from Exscript.util.log import log_to_file from Exscript.workqueue.job import Thread -bracket_expression = r"\{([^\]]*)\}" -define_expression = rf"^\S+=(?:\S+|{bracket_expression})$" -bracket_expression_re = re.compile(rf"^{bracket_expression}$") -define_expression_re = re.compile(define_expression) -duration_re = re.compile(r"(\d+(?:\.\d*)?|\.\d+)([smhd]?)") -duration_units = {"": 1, "s": 1, "m": 60, "h": 3600, "d": 86400} +_bracket_expression = r"\{([^\]]*)\}" +_define_expression = rf"^\S+=(?:\S+|{_bracket_expression})$" +_bracket_expression_re = re.compile(rf"^{_bracket_expression}$") +_define_expression_re = re.compile(_define_expression) +_duration_re = re.compile(r"(\d+(?:\.\d*)?|\.\d+)([smhd]?)") +_duration_units = {"": 1, "s": 1, "m": 60, "h": 3600, "d": 86400} -def duration(text: str) -> float: +def _duration(text: str) -> float: """ Returns the seconds of a time like the sleep command takes it, e.g. "1h 30m": numbers with an optional suffix s, m, h or d, separated by blanks. @@ -82,14 +82,18 @@ def duration(text: str) -> float: """ seconds = 0.0 for part in text.split(): - match = duration_re.fullmatch(part) + match = _duration_re.fullmatch(part) if match is None: raise ValueError(part) - seconds += float(match.group(1)) * duration_units[match.group(2)] + seconds += float(match.group(1)) * _duration_units[match.group(2)] return seconds -def expand_bracket(value_name: str, value: str) -> object: +# argparse names the type in its errors: "invalid duration value: 'x'". +_duration.__name__ = "duration" + + +def _expand_bracket(value_name: str, value: str) -> object: """ Returns the given value of the variable with the given name, or, if the value is a prompt in braces, asks the user for the value and returns @@ -104,7 +108,7 @@ def expand_bracket(value_name: str, value: str) -> object: :rtype: object :return: The value. """ - match = bracket_expression_re.match(value) + match = _bracket_expression_re.match(value) if match is None: return value prompt = match.group(1) or f"a value for '{value_name}' " @@ -120,11 +124,11 @@ def expand_bracket(value_name: str, value: str) -> object: return value -def expand_host_variables(host: Host) -> None: +def _expand_host_variables(host: Host) -> None: """ Asks the user for the values of the variables of the given host that are prompts in braces, also in lists, and replaces the prompts by the - values; see :func:`expand_bracket`. + values; see :func:`_expand_bracket`. :type host: Host :param host: The host. @@ -132,12 +136,12 @@ def expand_host_variables(host: Host) -> None: # Define host-specific variables. for key, value in list(host.get_all().items()): if isinstance(value, str): - value = expand_bracket(key, value) + value = _expand_bracket(key, value) elif hasattr(value, "__iter__"): for n in range(len(value)): if not isinstance(value[n], str): continue - value[n] = expand_bracket(key, value[n]) + value[n] = _expand_bracket(key, value[n]) host.set(key, value) @@ -168,14 +172,14 @@ def _join(queue: Queue) -> None: queue.join() -def run(parser: argparse.ArgumentParser, args: argparse.Namespace) -> int: +def _run(parser: argparse.ArgumentParser, args: argparse.Namespace) -> int: """ Runs the template on the hosts as the given command line arguments say, and returns the number of hosts on which it failed. Invalid arguments end the process through ``parser.error()``. :type parser: argparse.ArgumentParser - :param parser: The parser of the arguments, see :func:`create_parser`. + :param parser: The parser of the arguments, see :func:`_create_parser`. :type args: argparse.Namespace :param args: The parsed arguments. :rtype: int @@ -383,9 +387,9 @@ def mkhosts_from_csv(filename: Path) -> list[Host]: # Ask for values of marked variables. for host in hosts: - expand_host_variables(host) + _expand_host_variables(host) for key, val in list(args.define.items()): - args.define[key] = expand_bracket(key, val) + args.define[key] = _expand_bracket(key, val) # Choose the template processing type. tmpl_vars = args.define @@ -461,7 +465,7 @@ def function(job: Thread, host: Host, conn: Protocol, **kwargs: object) -> objec # Define command line option value types. -def assignment(value: str, pattern: re.Pattern[str] = define_expression_re) -> str: +def _assignment(value: str, pattern: re.Pattern[str] = _define_expression_re) -> str: """ Checks a value of the option -d/--define: returns it if it matches the given pattern, by default that of an assignment NAME=VALUE, and raises @@ -479,7 +483,7 @@ def assignment(value: str, pattern: re.Pattern[str] = define_expression_re) -> s return value -class DefineAction(argparse.Action): +class _DefineAction(argparse.Action): """ The action of the option -d/--define: stores the variables of the given assignments NAME=VALUE in the dictionary of the option. Variables in @@ -509,7 +513,7 @@ def __call__( setattr(namespace, self.dest, d) -def create_parser() -> argparse.ArgumentParser: +def _create_parser() -> argparse.ArgumentParser: """ Returns a parser of the command line arguments of ``exscript``. @@ -584,8 +588,8 @@ def create_parser() -> argparse.ArgumentParser: parser.add_argument( "-d", "--define", - type=assignment, - action=DefineAction, + type=_assignment, + action=_DefineAction, default=dict(), metavar="PAIR", help=""" @@ -806,7 +810,7 @@ def create_parser() -> argparse.ArgumentParser: parser.add_argument( "--sleep", metavar="TIME", - type=duration, + type=_duration, default="", help=""" Waits for the specified time before running the script. @@ -926,8 +930,8 @@ def main(argv: list[str] | None = None) -> int: else 1. Invalid arguments, ``--help`` and ``--version`` end the process through SystemExit, as usual with argparse. """ - parser = create_parser() - failed = run(parser, parser.parse_args(argv)) + parser = _create_parser() + failed = _run(parser, parser.parse_args(argv)) if failed != 0: print(f"error: {failed} actions failed.", file=sys.stderr) return 1 diff --git a/tests/Exscript/ScriptsTest.py b/tests/Exscript/ScriptsTest.py index 98e71182..3e7507a9 100644 --- a/tests/Exscript/ScriptsTest.py +++ b/tests/Exscript/ScriptsTest.py @@ -82,6 +82,11 @@ def testVersion(self): self.assertEqual(code, 0, output) self.assertEqual(output, Exscript.__version__ + '\n') + def testInvalidSleep(self): + code, output = self.run_exscript('--sleep', 'tomorrow', 'dummy://r1') + self.assertEqual(code, 2, output) + self.assertIn("argument --sleep: invalid duration value: 'tomorrow'", output) + def testRunTemplate(self): template = self.write_template('{sys.message("hello")}\n') args = '--non-interactive', '--no-authentication', template, 'dummy://r1' diff --git a/tests/Exscript/cliTest.py b/tests/Exscript/cliTest.py index 767dc364..cc90d687 100644 --- a/tests/Exscript/cliTest.py +++ b/tests/Exscript/cliTest.py @@ -3,19 +3,19 @@ import unittest from unittest import mock -from Exscript.cli import duration, expand_bracket, otp_main +from Exscript.cli import _duration, _expand_bracket, otp_main from Exscript.util.crypt import otp class cliTest(unittest.TestCase): def testDuration(self): - self.assertEqual(duration(''), 0) - self.assertEqual(duration('90'), 90) - self.assertEqual(duration('1.5m'), 90) - self.assertEqual(duration('1h 30m'), 5400) - self.assertEqual(duration('2d'), 172800) + self.assertEqual(_duration(''), 0) + self.assertEqual(_duration('90'), 90) + self.assertEqual(_duration('1.5m'), 90) + self.assertEqual(_duration('1h 30m'), 5400) + self.assertEqual(_duration('2d'), 172800) for invalid in 'tomorrow', '1x', '-1', '1; rm -rf ~': - self.assertRaises(ValueError, duration, invalid) + self.assertRaises(ValueError, _duration, invalid) def testPromptValuesAreLiteralsOnly(self): cases = { @@ -26,7 +26,7 @@ def testPromptValuesAreLiteralsOnly(self): } for typed, expected in cases.items(): with mock.patch('builtins.input', return_value=typed): - self.assertEqual(expand_bracket('count', '{#count}'), expected) + self.assertEqual(_expand_bracket('count', '{#count}'), expected) def testOtp(self): stdout = io.StringIO() From 3cbae4aee836f71a256d001ec3807db1a0fa7cd4 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 01:18:55 +0100 Subject: [PATCH 167/250] Drop the threading mode argument of the exscript command The queue runs its jobs in threads only, so mode="threading", added for upstream issue #230, only repeated the default. --- src/Exscript/cli.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/Exscript/cli.py b/src/Exscript/cli.py index 163d0aba..8269a407 100644 --- a/src/Exscript/cli.py +++ b/src/Exscript/cli.py @@ -327,11 +327,8 @@ def mkhosts_from_csv(filename: Path) -> list[Host]: template.debug = 0 # Create Exscript. - # Switch to 'threading' mode to match API default. - # See also: https://github.com/knipknap/exscript/issues/230 queue = Queue( domain=args.default_domain, - mode="threading", host_driver=args.use_driver, verbose=args.verbose, max_threads=args.connections, From e285ac6fc9ef44effffe7df13bd5687fb68def10 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 01:23:29 +0100 Subject: [PATCH 168/250] Expand ~ and variables in the host files of the exscript command The command expanded the names of the files of --hosts and --csv-hosts, but passed the original names to the readers, so a ~ that the shell left alone, e.g. on Windows, failed. --- CHANGES.md | 4 +++ src/Exscript/cli.py | 4 +-- tests/Exscript/cliTest.py | 54 ++++++++++++++++++++++++++++++++++++++- 3 files changed, 59 insertions(+), 3 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 266e900d..1f914293 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -592,6 +592,10 @@ The names below refer to functions callable from templates, not protocol methods - **`exscript --version`:** Prints the version of Exscript, as in 2.6 and as the tutorials say. The 2.9 development versions failed with `exscript: error: unrecognized arguments: --version`. +- **`exscript --hosts` and `--csv-hosts` With `~` and Variables:** Expand `~` and + environment variables in the name of the file, as `--account-pool` and + `--load-driver` do, e.g. in `--hosts '~/$SITE/hosts.txt'`. Before, they read the + name as given, so such a name failed with `No such file`. - **`Exscript.util.mail.send()`:** Ends the SMTP session with `QUIT` and closes the connection, also if sending fails; before, it left the connection open. It connects only after the mail and its attachments are ready. diff --git a/src/Exscript/cli.py b/src/Exscript/cli.py index 8269a407..3aebe400 100644 --- a/src/Exscript/cli.py +++ b/src/Exscript/cli.py @@ -195,14 +195,14 @@ def mkhost(name: str) -> Host: def mkhosts_from_file(filename: Path) -> list[Host]: return get_hosts_from_file( - args.hosts, + filename, default_protocol=args.protocol, default_domain=args.default_domain, ) def mkhosts_from_csv(filename: Path) -> list[Host]: return get_hosts_from_csv( - args.csv_hosts, + filename, default_protocol=args.protocol, default_domain=args.default_domain, ) diff --git a/tests/Exscript/cliTest.py b/tests/Exscript/cliTest.py index cc90d687..69bc1adb 100644 --- a/tests/Exscript/cliTest.py +++ b/tests/Exscript/cliTest.py @@ -1,13 +1,65 @@ import contextlib import io +import os +import tempfile import unittest from unittest import mock -from Exscript.cli import _duration, _expand_bracket, otp_main +from Exscript.cli import _duration, _expand_bracket, main, otp_main +from Exscript.util import template from Exscript.util.crypt import otp class cliTest(unittest.TestCase): + def setUp(self): + self.tmpdir = tempfile.TemporaryDirectory() + self.addCleanup(self.tmpdir.cleanup) + # main() sets up the logging of the process and the debug level of the templates. + for patcher in ( + mock.patch('Exscript.cli.setupLogging'), + mock.patch.object(template, 'debug', 0), + ): + patcher.start() + self.addCleanup(patcher.stop) + + def path(self, *names): + """Returns the path of the given file in the temporary directory.""" + return os.path.join(self.tmpdir.name, *names) + + def write(self, name, text): + """Writes the given text to the given file in the temporary directory.""" + with open(self.path(name), 'w', encoding='utf-8') as fp: + fp.write(text) + return self.path(name) + + def run_main(self, *args): + """Runs main() with the given arguments, and returns the exit status and the output.""" + stdout, stderr = io.StringIO(), io.StringIO() + with contextlib.redirect_stdout(stdout), contextlib.redirect_stderr(stderr): + try: + code = main(list(args)) + except SystemExit as e: + code = e.code + return code, stdout.getvalue(), stderr.getvalue() + + def testHostFilesWithUserAndVariables(self): + # The shell expands neither in quotes, e.g. --hosts '~/$SITE/hosts.txt'. + os.mkdir(self.path('hosts')) + self.write(os.path.join('hosts', 'hosts.txt'), 'dummy://r1\n') + self.write(os.path.join('hosts', 'hosts.csv'), 'address\tgreeting\ndummy://r2\thello\n') + env = {'HOME': self.tmpdir.name, 'USERPROFILE': self.tmpdir.name, 'SITE': 'hosts'} + cases = ( + ('--hosts', 'hosts.txt', '{sys.message("hello")}'), + ('--csv-hosts', 'hosts.csv', '{sys.message(greeting)}'), + ) + with mock.patch.dict(os.environ, env): + for option, name, script in cases: + code, stdout, stderr = self.run_main( + '-n', '-i', '-e', script, option, '~/$SITE/' + name + ) + self.assertEqual(code, 0, stderr) + self.assertEqual(stdout, 'hello\n') + def testDuration(self): self.assertEqual(_duration(''), 0) self.assertEqual(_duration('90'), 90) From fa71e6b8d3c723b21f12b1f4d68874d6dc05cb5a Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 01:24:38 +0100 Subject: [PATCH 169/250] Split -d assignments of the exscript command at the first = An assignment with a second =, e.g. -d a=b=c, failed with a ValueError. --- CHANGES.md | 2 ++ src/Exscript/cli.py | 2 +- tests/Exscript/cliTest.py | 6 ++++++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 1f914293..a3cfd7a1 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -596,6 +596,8 @@ The names below refer to functions callable from templates, not protocol methods environment variables in the name of the file, as `--account-pool` and `--load-driver` do, e.g. in `--hosts '~/$SITE/hosts.txt'`. Before, they read the name as given, so such a name failed with `No such file`. +- **`exscript -d` With `=` in the Value:** `-d a=b=c` gives the variable `a` the value + `b=c`. Before, the command failed with `ValueError: too many values to unpack`. - **`Exscript.util.mail.send()`:** Ends the SMTP session with `QUIT` and closes the connection, also if sending fails; before, it left the connection open. It connects only after the mail and its attachments are ready. diff --git a/src/Exscript/cli.py b/src/Exscript/cli.py index 3aebe400..91695ced 100644 --- a/src/Exscript/cli.py +++ b/src/Exscript/cli.py @@ -503,7 +503,7 @@ def __call__( values = [values] d = getattr(namespace, self.dest) for value in values or (): - left, right = value.split("=") + left, right = value.split("=", 1) if left.startswith('__') and left.endswith('__'): parser.error('Assignment to internal variable %s' % repr(left)) d[left] = right diff --git a/tests/Exscript/cliTest.py b/tests/Exscript/cliTest.py index 69bc1adb..1a0bc509 100644 --- a/tests/Exscript/cliTest.py +++ b/tests/Exscript/cliTest.py @@ -60,6 +60,12 @@ def testHostFilesWithUserAndVariables(self): self.assertEqual(code, 0, stderr) self.assertEqual(stdout, 'hello\n') + def testDefineValueWithEqualSign(self): + script = '{sys.message(a)}' + code, stdout, stderr = self.run_main('-n', '-i', '-d', 'a=b=c', '-e', script, 'dummy://r1') + self.assertEqual(code, 0, stderr) + self.assertEqual(stdout, 'b=c\n') + def testDuration(self): self.assertEqual(_duration(''), 0) self.assertEqual(_duration('90'), 90) From eedfef26476f9f21850e1805fc3bb954105bcbdc Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 01:42:24 +0100 Subject: [PATCH 170/250] Test the options of the exscript command in-process Most of its options, prompts and error paths had no tests; the few tests ran it in a subprocess. --- tests/Exscript/ScriptsTest.py | 48 +--- tests/Exscript/cliTest.py | 449 +++++++++++++++++++++++++++++++++- 2 files changed, 444 insertions(+), 53 deletions(-) diff --git a/tests/Exscript/ScriptsTest.py b/tests/Exscript/ScriptsTest.py index 3e7507a9..fcedbd73 100644 --- a/tests/Exscript/ScriptsTest.py +++ b/tests/Exscript/ScriptsTest.py @@ -9,25 +9,6 @@ import Exscript -BASEDIR = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..')) -KEYFILE = os.path.join(BASEDIR, 'tests', 'Exscript', 'protocols', 'id_rsa') - -# Runs the CLI, but getpass.getuser() finds no user name, as for a user ID without a passwd -# entry: it raises OSError since Python 3.13, and KeyError before. -WITHOUT_USER_NAME = ''' -import getpass, sys - -from Exscript.cli import main - - -def getuser(): - raise %s('no user name') - - -getpass.getuser = getuser -sys.exit(main()) -''' - class ScriptsTest(unittest.TestCase): """Runs the exscript command line tool like a user.""" @@ -39,9 +20,9 @@ def setUp(self): os.environ, HOME=self.tmpdir.name, USERPROFILE=self.tmpdir.name, NO_COLOR='1' ) - def run_exscript(self, *args, python_args=('-m', 'Exscript')): + def run_exscript(self, *args): proc = subprocess.run( # noqa: S603 - runs Exscript with fixed arguments - [sys.executable, *python_args, *args], + [sys.executable, '-m', 'Exscript', *args], capture_output=True, encoding='utf-8', env=self.env, @@ -82,11 +63,6 @@ def testVersion(self): self.assertEqual(code, 0, output) self.assertEqual(output, Exscript.__version__ + '\n') - def testInvalidSleep(self): - code, output = self.run_exscript('--sleep', 'tomorrow', 'dummy://r1') - self.assertEqual(code, 2, output) - self.assertIn("argument --sleep: invalid duration value: 'tomorrow'", output) - def testRunTemplate(self): template = self.write_template('{sys.message("hello")}\n') args = '--non-interactive', '--no-authentication', template, 'dummy://r1' @@ -94,26 +70,6 @@ def testRunTemplate(self): self.assertEqual(code, 0, output) self.assertIn('hello', output) - def testSecureModeByDefault(self): - template = self.write_template('{sys.env("PATH")}\n') - args = '--non-interactive', '--no-authentication', template, 'dummy://r1' - code, output = self.run_exscript(*args) - self.assertEqual(code, 2, output) - self.assertIn("insecure function 'sys.env' is not permitted", output) - - code, output = self.run_exscript('--insecure', *args) - self.assertEqual(code, 0, output) - - def testSshKeyWithoutUserName(self): - # Without a login, the key is for the current user; without a name, it gets no account. - template = self.write_template('{sys.message("hello")}\n') - args = '--non-interactive', '--no-authentication', '--ssh-key', KEYFILE, template - for error in 'OSError', 'KeyError': - python_args = '-c', WITHOUT_USER_NAME % error - code, output = self.run_exscript(*args, 'dummy://r1', python_args=python_args) - self.assertEqual(code, 0, output) - self.assertIn('hello', output) - @unittest.skipIf(sys.platform == 'win32', 'Windows cannot send SIGINT to a process') def testCtrlC(self): # The job waits for a minute in a thread, which Ctrl+C cannot interrupt. diff --git a/tests/Exscript/cliTest.py b/tests/Exscript/cliTest.py index 1a0bc509..4c453640 100644 --- a/tests/Exscript/cliTest.py +++ b/tests/Exscript/cliTest.py @@ -1,26 +1,56 @@ import contextlib import io import os +import runpy +import signal import tempfile import unittest from unittest import mock -from Exscript.cli import _duration, _expand_bracket, main, otp_main +from Exscript import Account, Host, PrivateKey +from Exscript.cli import _duration, _exit_on_sigint, _expand_bracket, main, otp_main +from Exscript.parselib.exception import CompileError +from Exscript.protocols import Dummy, drivers from Exscript.util import template from Exscript.util.crypt import otp +KEYFILE = os.path.join(os.path.dirname(__file__), 'protocols', 'id_rsa') +HELLO = '{sys.message("hello")}' +POOL = '[account-pool]\nalice = c2VjcmV0\n' # the password is "secret" + +LIB = ''' +def hello(scope, name): + print('hello', name[0]) + return True + + +__lib__ = {'hello': hello} +''' + +DRIVER = ''' +from Exscript.protocols.drivers import Driver + + +class CliTestDriver(Driver): + def __init__(self): + Driver.__init__(self, 'cli_test') +''' + class cliTest(unittest.TestCase): def setUp(self): self.tmpdir = tempfile.TemporaryDirectory() self.addCleanup(self.tmpdir.cleanup) # main() sets up the logging of the process and the debug level of the templates. - for patcher in ( - mock.patch('Exscript.cli.setupLogging'), - mock.patch.object(template, 'debug', 0), - ): - patcher.start() - self.addCleanup(patcher.stop) + self.setup_logging = self.start(mock.patch('Exscript.cli.setupLogging')) + self.start(mock.patch.object(template, 'debug', 0)) + self.start(mock.patch.dict(os.environ, NO_COLOR='1')) + + def start(self, patcher): + """Starts the given patcher until the end of the test, and returns its mock.""" + result = patcher.start() + self.addCleanup(patcher.stop) + return result def path(self, *names): """Returns the path of the given file in the temporary directory.""" @@ -32,6 +62,11 @@ def write(self, name, text): fp.write(text) return self.path(name) + def read(self, *names): + """Returns the content of the given file in the temporary directory.""" + with open(self.path(*names), encoding='utf-8') as fp: + return fp.read() + def run_main(self, *args): """Runs main() with the given arguments, and returns the exit status and the output.""" stdout, stderr = io.StringIO(), io.StringIO() @@ -42,6 +77,64 @@ def run_main(self, *args): code = e.code return code, stdout.getvalue(), stderr.getvalue() + def unload_driver(self, name): + """Unregisters the driver with the given name if it is registered.""" + if name in drivers.driver_map: + drivers.disable_driver(name) + + def testTemplateFile(self): + path = self.write('test.exscript', '{sys.message(__filename__)}\n') + code, stdout, stderr = self.run_main('-n', '-i', path, 'dummy://r1') + self.assertEqual(code, 0, stderr) + self.assertEqual(stdout, path + '\n') + + def testExecuteTakesTheFilenameAsHost(self): + script = '{sys.message(__hostname__)}' + args = '-n', '-i', '-c', '2', '-e', script, 'dummy://r1', 'dummy://r2' + code, stdout, stderr = self.run_main(*args) + self.assertEqual(code, 0, stderr) + self.assertEqual(sorted(stdout.splitlines()), ['r1', 'r2']) + + def testProtocolAndDefaultDomain(self): + args = '-n', '-i', '-p', 'dummy', '--default-domain', 'example.com' + code, stdout, stderr = self.run_main(*args, '-e', '{sys.message(__hostname__)}', 'r1') + self.assertEqual(code, 0, stderr) + self.assertEqual(stdout, 'r1.example.com\n') + + def testTemplateErrors(self): + cases = ( + ((), 'Missing template filename'), + ((self.path('missing.exscript'), 'dummy://r1'), 'No such file or directory'), + (('-e', HELLO), 'No hosts to connect to'), + (('-e', '{sys.message(}', 'dummy://r1'), 'Expected term but got close_curly_bracket'), + (('-e', '{sys.env("PATH")}', 'dummy://r1'), "function 'sys.env' is not permitted"), + ) + for args, error in cases: + code, stdout, stderr = self.run_main('-n', '-i', *args) + self.assertEqual(code, 2, stderr) + self.assertIn(error, stderr) + + def testInsecure(self): + code, stdout, stderr = self.run_main( + '-n', '-i', '--insecure', '-e', '{sys.env("PATH")}', 'dummy://r1' + ) + self.assertEqual(code, 0, stderr) + + def testParserDebugging(self): + code, stdout, stderr = self.run_main('-n', '-i', '-V', '1', '-e', HELLO, 'dummy://r1') + self.assertEqual(code, 0, stderr) + self.assertTrue(stdout.endswith('hello\n'), stdout) + # With -V, an error in the template raises the exception with its traceback. + with self.assertRaises(CompileError): + self.run_main('-n', '-i', '-V', '1', '-e', '{sys.message(}', 'dummy://r1') + + def testVerbosity(self): + self.run_main('-n', '-i', '-e', HELLO, 'dummy://r1') + self.setup_logging.assert_called_once_with(package=True, parser=False, protocol=False) + args = '-v', '0', '-V', '1', '--protocol-verbose', '1' + self.run_main('-n', '-i', *args, '-e', HELLO, 'dummy://r1') + self.setup_logging.assert_called_with(package=False, parser=True, protocol=True) + def testHostFilesWithUserAndVariables(self): # The shell expands neither in quotes, e.g. --hosts '~/$SITE/hosts.txt'. os.mkdir(self.path('hosts')) @@ -60,12 +153,354 @@ def testHostFilesWithUserAndVariables(self): self.assertEqual(code, 0, stderr) self.assertEqual(stdout, 'hello\n') + def testEmptyHostFiles(self): + for option, name, text in ( + ('--hosts', 'hosts.txt', ''), + ('--csv-hosts', 'hosts.csv', 'address\n'), + ): + path = self.write(name, text) + code, stdout, stderr = self.run_main( + '-n', '-i', '-e', HELLO, option, path, 'dummy://r1' + ) + self.assertEqual(code, 0, stderr) + self.assertEqual(stdout, f'Warning: {path} is empty.\nhello\n') + + def testHostFileErrors(self): + cases = ( + ('--hosts', self.path('missing.txt'), 'No such file'), + ('--csv-hosts', self.path('missing.csv'), 'No such file'), + ('--hosts', self.write('hosts.txt', 'dummy://r1:port\n'), 'Hostname parse error'), + ) + for option, path, error in cases: + code, stdout, stderr = self.run_main('-n', '-i', '-e', HELLO, option, path) + self.assertEqual(code, 2, stderr) + self.assertIn(error, stderr) + + def testHostVariablePrompt(self): + path = self.write('hosts.csv', 'address\tgreeting\ndummy://r1\t{Greeting}\n') + script = '{sys.message(greeting)}' + with mock.patch('builtins.input', return_value='hello') as input: + code, stdout, stderr = self.run_main('-n', '-i', '-e', script, '--csv-hosts', path) + self.assertEqual(code, 0, stderr) + self.assertEqual(stdout, 'hello\n') + input.assert_called_once_with('Please enter Greeting: ') + + def testDefinePrompts(self): + args = '-d', 'a={A}', '-d', 'b={!B}', '-d', 'c={#C}', '-d', 'd={}' + script = '{sys.message(a)}{sys.message(b)}{sys.message(c)}{sys.message(d)}' + with ( + mock.patch('builtins.input', side_effect=['x', '"y"', 'z']) as input, + mock.patch('getpass.getpass', return_value='secret') as getpass, + ): + code, stdout, stderr = self.run_main('-n', '-i', *args, '-e', script, 'dummy://r1') + self.assertEqual(code, 0, stderr) + self.assertEqual(stdout, 'x\nsecret\ny\nz\n') + prompts = [call.args[0] for call in input.call_args_list] + self.assertEqual( + prompts, ['Please enter A: ', 'Please enter C: ', "Please enter a value for 'd' : "] + ) + getpass.assert_called_once_with('Please enter B: ') + + def testDefineErrors(self): + cases = ( + ('a', 'invalid value: a'), + ('__filename__=x', "Assignment to internal variable '__filename__'"), + ) + for value, error in cases: + code, stdout, stderr = self.run_main('-n', '-i', '-d', value, '-e', HELLO, 'dummy://r1') + self.assertEqual(code, 2, stderr) + self.assertIn(error, stderr) + def testDefineValueWithEqualSign(self): script = '{sys.message(a)}' code, stdout, stderr = self.run_main('-n', '-i', '-d', 'a=b=c', '-e', script, 'dummy://r1') self.assertEqual(code, 0, stderr) self.assertEqual(stdout, 'b=c\n') + def testLoginPrompt(self): + with ( + mock.patch('builtins.input', return_value='joe') as input, + mock.patch('getpass.getpass', return_value='secret') as getpass, + ): + code, stdout, stderr = self.run_main('-e', '{sys.message(__username__)}', 'dummy://r1') + self.assertEqual(code, 0, stderr) + self.assertEqual(stdout, 'joe\n') + self.assertTrue(input.call_args.args[0].startswith('Please enter your user name')) + getpass.assert_called_once_with('Please enter your password: ') + + def testLoginPromptInterrupted(self): + with mock.patch('builtins.input', side_effect=KeyboardInterrupt): + code, stdout, stderr = self.run_main('-e', HELLO, 'dummy://r1') + self.assertEqual(code, 1) + self.assertEqual(stdout, '') + + def testAuthorizationPrompt(self): + with ( + mock.patch('builtins.input', return_value='joe'), + mock.patch('getpass.getpass', side_effect=['secret', 'enable']) as getpass, + ): + code, stdout, stderr = self.run_main('--authorization', '-e', HELLO, 'dummy://r1') + self.assertEqual(code, 0, stderr) + prompts = [call.args[0] for call in getpass.call_args_list] + expected = ['Please enter your password: ', 'Please enter your authorization password: '] + self.assertEqual(prompts, expected) + + def testAccountPool(self): + pool = self.write('pool.cfg', POOL) + script = '{sys.message(__username__)}' + with mock.patch('builtins.input') as input, mock.patch('getpass.getpass') as getpass: + code, stdout, stderr = self.run_main('--account-pool', pool, '-e', script, 'dummy://r1') + self.assertEqual(code, 0, stderr) + self.assertEqual(stdout, 'alice\n') + input.assert_not_called() + getpass.assert_not_called() + + def testEmptyAccountPool(self): + pool = self.write('pool.cfg', '[account-pool]\n') + code, stdout, stderr = self.run_main( + '-n', '-i', '--account-pool', pool, '-e', HELLO, 'dummy://r1' + ) + self.assertEqual(code, 0, stderr) + self.assertEqual( + stdout, f'WARNING: No accounts found in account pool file ({pool})!\nhello\n' + ) + + def testSshKey(self): + with ( + mock.patch('builtins.input', return_value='joe'), + mock.patch('getpass.getpass', return_value='secret'), + ): + script = '{sys.message(__username__)}' + code, stdout, stderr = self.run_main('--ssh-key', KEYFILE, '-e', script, 'dummy://r1') + self.assertEqual(code, 0, stderr) + self.assertEqual(stdout, f'Reading key from {KEYFILE}\njoe\n') + + def testSshKeyWithoutLoginPrompt(self): + # The key is for the current user; without a user name, there is no account. + cases = ( + (mock.patch('getpass.getuser', return_value='jane'), [('jane', None, None)]), + (mock.patch('getpass.getuser', side_effect=OSError('no user name')), []), # 3.13+ + (mock.patch('getpass.getuser', side_effect=KeyError('no user name')), []), + ) + for getuser, expected in cases: + with getuser, mock.patch('Exscript.cli.Account', wraps=Account) as account: + code, stdout, stderr = self.run_main( + '-n', '-i', '--ssh-key', KEYFILE, '-e', HELLO, 'dummy://r1' + ) + self.assertEqual(code, 0, stderr) + self.assertEqual(stdout, f'Reading key from {KEYFILE}\nhello\n') + self.assertEqual([call.args for call in account.call_args_list], expected) + for call in account.call_args_list: + self.assertIsInstance(call.kwargs['key'], PrivateKey) + + def testInvalidSshKey(self): + key = self.write('id_invalid', 'not a key\n') + code, stdout, stderr = self.run_main( + '-n', '-i', '--ssh-key', key, '-e', HELLO, 'dummy://r1' + ) + self.assertEqual(code, 2, stderr) + self.assertIn('not a recognized private key', stderr) + + def testSshOptions(self): + args = ( + ('--ssh-disabled-algs', 'ciphers=aes128-cbc,3des-cbc;pubkeys=ssh-rsa'), + ('--ssh-preferred-auth', 'publickey,password'), + ('--ssh-auto-verify',), + ('--ssh-host-key-policy', 'warn'), + ) + expected = { + 'disabled_algorithms': {'ciphers': ['aes128-cbc', '3des-cbc'], 'pubkeys': ['ssh-rsa']}, + 'preferred_authentications': ['publickey', 'password'], + 'verify_fingerprint': False, + 'host_key_policy': 'warn', + } + defaults = {'verify_fingerprint': True, 'host_key_policy': 'reject'} + for options, want in ((), defaults), (sum(args, ()), expected): + with mock.patch.object( + Host, 'set_option', autospec=True, side_effect=Host.set_option + ) as set_option: + code, stdout, stderr = self.run_main( + '-n', '-i', *options, '-e', HELLO, 'dummy://r1' + ) + self.assertEqual(code, 0, stderr) + got = {call.args[1]: call.args[2] for call in set_option.call_args_list} + got.pop('debug') + self.assertEqual(got, want) + + def testNoPrompt(self): + # The commands are sent without waiting for a prompt; the device asks for the login. + template = self.write('test.exscript', 'ls\n{sys.message(__filename__)}\n') + code, stdout, stderr = self.run_main('-n', '-i', '--no-prompt', template, 'dummy://r1') + self.assertEqual(code, 0, stderr) + self.assertEqual(stdout, template + '\n') + + hosts = self.write('hosts.txt', 'dummy://r1\n') + script = 'ls\n{sys.message("sent")}' + code, stdout, stderr = self.run_main( + '-n', '-i', '--no-prompt', '-e', script, '--hosts', hosts + ) + self.assertEqual(code, 0, stderr) + self.assertEqual(stdout, 'sent\n') + + def testNoStrip(self): + pool = self.write('pool.cfg', POOL) + script = 'ls\n{sys.message(__response__)}' + for args, expected in ((), '\n'), (('--no-strip',), 'ls\n'): + code, stdout, stderr = self.run_main( + '--account-pool', pool, *args, '-e', script, 'dummy://r1' + ) + self.assertEqual(code, 0, stderr) + self.assertEqual(stdout, expected) + + def testNoEscaping(self): + script = r'{sys.message("A\x42C")}' + for args, expected in ((), 'ABC\n'), (('--no-escaping',), 'Ax42C\n'): + code, stdout, stderr = self.run_main('-n', '-i', *args, '-e', script, 'dummy://r1') + self.assertEqual(code, 0, stderr) + self.assertEqual(stdout, expected) + + def testNoAutoLogout(self): + # The template logs out (close without force) before the queue closes the connection. + for args, forced in ((), [False, True]), (('--no-auto-logout',), [True]): + with mock.patch.object(Dummy, 'close', autospec=True, side_effect=Dummy.close) as close: + code, stdout, stderr = self.run_main('-n', '-i', *args, '-e', HELLO, 'dummy://r1') + self.assertEqual(code, 0, stderr) + self.assertEqual([call.kwargs['force'] for call in close.call_args_list], forced) + + def testLib(self): + lib = self.write('mylib.py', LIB) + script = '{mylib.hello("world")}' + code, stdout, stderr = self.run_main('-n', '-i', '--lib', lib, '-e', script, 'dummy://r1') + self.assertEqual(code, 0, stderr) + self.assertEqual(stdout, 'hello world\n') + + missing = self.path('missing.py') + code, stdout, stderr = self.run_main( + '-n', '-i', '--lib', missing, '-e', HELLO, 'dummy://r1' + ) + self.assertEqual(code, 2, stderr) + self.assertIn('No such file', stderr) + + def testLoadDriver(self): + path = self.write('cli_test_driver.py', DRIVER) + self.addCleanup(self.unload_driver, 'cli_test') + args = '--load-driver', path, '--use-driver', 'cli_test' + code, stdout, stderr = self.run_main('-n', '-i', *args, '-e', HELLO, 'dummy://r1') + self.assertEqual(code, 0, stderr) + self.assertEqual( + stdout, f"Searching drivers in {path}...\nDriver 'CliTestDriver' added.\nhello\n" + ) + + def testLoadDriverErrors(self): + missing = self.path('missing.py') + code, stdout, stderr = self.run_main( + '-n', '-i', '--load-driver', missing, '-e', HELLO, 'dummy://r1' + ) + self.assertEqual(code, 2, stderr) + self.assertIn('No such file or directory', stderr) + + for name, text, error in ( + ('driver.txt', 'x = 1\n', ImportError), + ('broken.py', 'x = \n', SyntaxError), + ): + path = self.write(name, text) + stderr = io.StringIO() + with ( + contextlib.redirect_stdout(io.StringIO()), + contextlib.redirect_stderr(stderr), + self.assertRaises(error), + ): + main(['-n', '-i', '--load-driver', path, '-e', HELLO, 'dummy://r1']) + self.assertEqual(stderr.getvalue(), f'Error in driver {path}\n') + + def testUnknownDriver(self): + code, stdout, stderr = self.run_main( + '-n', '-i', '--use-driver', 'bogus', '-e', HELLO, 'dummy://r1' + ) + self.assertEqual(code, 1) + self.assertEqual(stderr, 'error: 1 actions failed.\n') + + def testLogdir(self): + pool = self.write('pool.cfg', POOL) + logdir = self.path('logs') + args = '--account-pool', pool, '-l', logdir + code, stdout, stderr = self.run_main(*args, '-e', HELLO, 'dummy://r1') + self.assertEqual(code, 0, stderr) + self.assertEqual(stdout, f'Creating log directory ({logdir})...\nhello\n') + session = self.read('logs', 'r1.log') + self.assertIn('User: alice', session) + + # New sessions are appended, unless --overwrite-logs is given. + self.run_main(*args, '-e', HELLO, 'dummy://r1') + self.assertEqual(self.read('logs', 'r1.log'), session * 2) + self.run_main(*args, '--overwrite-logs', '-e', HELLO, 'dummy://r1') + self.assertLessEqual(len(self.read('logs', 'r1.log')), len(session)) + + # --delete-logs deletes the logs of the hosts that succeeded. + self.run_main(*args, '--delete-logs', '-e', HELLO, 'dummy://r1') + self.assertEqual(os.listdir(logdir), []) + + # An error is also written to a file of its own. + code, stdout, stderr = self.run_main(*args, '-e', '{fail "boom"}', 'dummy://r1') + self.assertEqual(code, 1) + self.assertEqual(sorted(os.listdir(logdir)), ['r1.log', 'r1.log.error']) + self.assertIn('boom', self.read('logs', 'r1.log.error')) + + def testLogdirError(self): + logdir = os.path.join(self.write('file', ''), 'logs') + code, stdout, stderr = self.run_main('-n', '-i', '-l', logdir, '-e', HELLO, 'dummy://r1') + self.assertEqual(code, 2, stderr) + self.assertEqual(stdout, f'Creating log directory ({logdir})...\n') + + def testSleep(self): + code, stdout, stderr = self.run_main( + '-n', '-i', '--sleep', '0.001', '-e', HELLO, 'dummy://r1' + ) + self.assertEqual(code, 0, stderr) + self.assertEqual( + stdout, 'Waiting for 0.001 seconds...time expired, starting script.\nhello\n' + ) + + def testInvalidSleep(self): + code, stdout, stderr = self.run_main('--sleep', 'tomorrow', 'dummy://r1') + self.assertEqual(code, 2) + self.assertIn("argument --sleep: invalid duration value: 'tomorrow'", stderr) + + def testFailures(self): + # -i without an account: the login fails. + code, stdout, stderr = self.run_main('-i', '-e', HELLO, 'dummy://r1') + self.assertEqual(code, 1) + self.assertEqual(stderr, 'error: 1 actions failed.\n') + + code, stdout, stderr = self.run_main( + '-n', '-i', '-e', '{fail "boom"}', 'dummy://r1', 'dummy://r2' + ) + self.assertEqual(code, 1) + self.assertEqual(stderr, 'error: 2 actions failed.\n') + + def testRetry(self): + script = '{sys.message("attempt")}{fail "boom"}' + code, stdout, stderr = self.run_main('-n', '-i', '--retry', '1', '-e', script, 'dummy://r1') + self.assertEqual(code, 1) + self.assertEqual(stdout, 'attempt\nattempt\n') + + def testExitOnSigint(self): + stdout, stderr = io.StringIO(), io.StringIO() + with ( + mock.patch('os._exit') as exit, + contextlib.redirect_stdout(stdout), + contextlib.redirect_stderr(stderr), + ): + _exit_on_sigint(signal.SIGINT, None) + exit.assert_called_once_with(130) + self.assertEqual(stderr.getvalue(), 'Interrupted.\n') + + def testImportOfMainModule(self): + # The API reference imports Exscript.__main__, which must not run the command. + with mock.patch('Exscript.cli.main') as main: + runpy.run_module('Exscript.__main__', run_name='Exscript.__main__') + main.assert_not_called() + def testDuration(self): self.assertEqual(_duration(''), 0) self.assertEqual(_duration('90'), 90) From cf0601b1faf63548430b77dd9d83a58904c64b39 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 01:47:37 +0100 Subject: [PATCH 171/250] Unpause the queue before tearDown() of QueueTest destroys it destroy() waits for the queued jobs, so a test that failed while the work queue was paused hung instead of failing. --- tests/Exscript/QueueTest.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/Exscript/QueueTest.py b/tests/Exscript/QueueTest.py index 76482d98..87189b0a 100644 --- a/tests/Exscript/QueueTest.py +++ b/tests/Exscript/QueueTest.py @@ -166,6 +166,9 @@ def setUp(self): def tearDown(self): shutil.rmtree(self.tempdir) + # destroy() waits for the queued jobs, so a test that paused the queue + # and failed would hang otherwise. + self.queue.workqueue.unpause() try: self.queue.destroy() except: @@ -475,7 +478,6 @@ def add_and_end(task, job_id): self.queue.workqueue.pause() self.assertIsNotNone(self.queue.run_or_ignore('dummy://dummy2', do_nothing)) self.assertIsNone(self.queue.run_or_ignore('dummy://dummy2', do_nothing)) - self.queue.workqueue.unpause() def testProgressCountsOnlyEnqueuedJobs(self): self.queue.workqueue.pause() @@ -528,7 +530,6 @@ def testCountsJobsThatAreEnqueuedAtTheSameTime(self): thread.start() for thread in threads: thread.join() - self.queue.workqueue.unpause() self.assertEqual(self.queue.total, 2) def testJobsConnectWithTheHostDriver(self): From 95840faa9eaa06adee64d19776d45f3e9172465e Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 01:51:12 +0100 Subject: [PATCH 172/250] Draw the status bar of the queue in one step per job The thread of each job erases the bar, writes its line and draws the bar again, but the lock covered only each write, so two jobs could mix their output on one line. The lock now covers all steps. --- CHANGES.md | 5 +++ src/Exscript/queue.py | 29 +++++++------ tests/Exscript/QueueTest.py | 85 +++++++++++++++++++++++++++++++++++++ 3 files changed, 106 insertions(+), 13 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index a3cfd7a1..a66f3952 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -683,6 +683,11 @@ The names below refer to functions callable from templates, not protocol methods the status bar of `Queue`, it cuts them in the middle, at most down to `...`. Before, on a terminal of about 30 columns or less, the cut overlapped and made the bar longer instead of shorter. +- **Status Bar of Jobs That End at the Same Time:** The queue erases the status bar, + writes the line of a job and draws the bar again as one step, which the threads of + other jobs cannot interrupt. Before, jobs that started, failed or ended at the same + time could mix these steps: a line such as `job1 succeeded.` could follow a bar on the + same line, or two bars could end up on one line. ## Demos diff --git a/src/Exscript/queue.py b/src/Exscript/queue.py index 9c2ed7bb..7df36d00 100644 --- a/src/Exscript/queue.py +++ b/src/Exscript/queue.py @@ -256,8 +256,9 @@ def __init__( self.stderr = stderr self.host_driver = host_driver self.exc_cb = exc_cb - # For the output, completed and failed; _enqueue() counts total under - # the lock of the pipeline. + # Held across each erasing, writing and redrawing of the status bar + # and the counting of completed and failed, which the threads of the + # jobs do; _enqueue() counts total under the lock of the pipeline. self.lock = threading.RLock() self.devnull = _Discard() self.channel_map = {'fatal_errors': self.stderr} @@ -372,9 +373,10 @@ def _print(self, channel: str, msg: str) -> None: Writes the given message and a newline to the given output channel, and redraws the status bar after it. """ - self._del_status_bar() - self._write(channel, msg + '\n') - self._print_status_bar() + with self.lock: + self._del_status_bar() + self._write(channel, msg + '\n') + self._print_status_bar() def _dbg(self, level: int, msg: str, exc_info: bool | BaseException | None = None) -> None: """ @@ -397,8 +399,9 @@ def _on_job_started(self, job: 'Thread') -> None: """ Redraws the status bar when an attempt of a job starts. """ - self._del_status_bar() - self._print_status_bar() + with self.lock: + self._del_status_bar() + self._print_status_bar() def _on_job_error( self, @@ -427,9 +430,9 @@ def _on_job_succeeded(self, job: 'Thread') -> None: """ with self.lock: self.completed += 1 - self._del_status_bar() - self._write('status_bar', job.name + ' succeeded.' + '\n') - self._print_status_bar(exclude=job.name) + self._del_status_bar() + self._write('status_bar', job.name + ' succeeded.' + '\n') + self._print_status_bar(exclude=job.name) def _on_job_aborted(self, job: 'Thread') -> None: """ @@ -439,9 +442,9 @@ def _on_job_aborted(self, job: 'Thread') -> None: with self.lock: self.completed += 1 self.failed += 1 - self._del_status_bar() - self._write('errors', job.name + ' finally failed.' + '\n') - self._print_status_bar(exclude=job.name) + self._del_status_bar() + self._write('errors', job.name + ' finally failed.' + '\n') + self._print_status_bar(exclude=job.name) def set_max_threads(self, n_connections: int) -> None: """ diff --git a/tests/Exscript/QueueTest.py b/tests/Exscript/QueueTest.py index 87189b0a..7b0478d5 100644 --- a/tests/Exscript/QueueTest.py +++ b/tests/Exscript/QueueTest.py @@ -75,6 +75,48 @@ def __set__(self, obj, value): obj.__dict__[self.name] = value +def erase_together(threads): + """ + Returns a replacement for Queue._del_status_bar(): after its first call + in each of the given threads, the thread waits until all of them got + there, unless it holds the lock of the queue then. So without the lock, + all threads erase the status bar before any writes a line or draws it. + """ + erase = Queue._del_status_bar + waiting = set(threads) + barrier = threading.Barrier(len(waiting), timeout=10) + + def del_status_bar(queue): + erase(queue) + thread = threading.current_thread() + if thread in waiting: + waiting.discard(thread) + if not holds_lock(queue.lock): + barrier.wait() + + return del_status_bar + + +def screen(output): + """ + Returns the lines that the given output leaves on a terminal, where a + backspace moves the cursor back and a character replaces the one under + the cursor; without the blanks at the end of the lines. + """ + lines = [[]] + column = 0 + for char in output: + if char == '\n': + lines.append([]) + column = 0 + elif char == '\b': + column = max(column - 1, 0) + else: + lines[-1][column : column + 1] = [char] + column += 1 + return [''.join(line).rstrip() for line in lines] + + def count_calls(job, data, **kwargs): assert hasattr(job, 'start') assert 'testarg' in kwargs @@ -583,6 +625,49 @@ def testStatusBarShortensTheNamesOfRunningJobs(self): self.assertEqual(self.out.read(), bar) self.assertEqual(self.queue.status_bar_length, len(bar)) + def testStatusBarOfJobsThatEndAtTheSameTime(self): + # Both threads erase the bar before either writes its line or draws + # the bar again, unless the queue holds its lock across all of it. + error = (ValueError, ValueError('intentional error'), 'Traceback (fake)') + succeeded = ('job_succeeded_event', ()) + cases = { + 'succeeded': ([succeeded, succeeded], ['job0 succeeded.', 'job1 succeeded.']), + 'aborted': ( + [('job_aborted_event', ())] * 2, + ['job0 finally failed.', 'job1 finally failed.'], + ), + 'started': ([('job_started_event', ()), succeeded], ['job1 succeeded.']), + 'error': ( + [('job_error_event', (error,)), succeeded], + ['Traceback (fake)', 'job0 error: intentional error', 'job1 succeeded.'], + ), + } + jobs = [SimpleNamespace(name='job0'), SimpleNamespace(name='job1')] + for case, (events, lines) in cases.items(): + with self.subTest(case=case): + self.queue.destroy() + output = Log() # stdout and stderr, like on a terminal + self.queue = Queue(verbose=1, max_threads=2, stdout=output, stderr=output) + self.queue.total = 2 + workqueue = self.queue.workqueue + threads = [ + threading.Thread(target=getattr(workqueue, event), args=(job, *args)) + for job, (event, args) in zip(jobs, events) + ] + with ( + mock.patch.object(workqueue, 'get_running_jobs', return_value=jobs), + mock.patch('Exscript.queue.get_terminal_size', return_value=(24, 80)), + mock.patch.object(Queue, '_del_status_bar', erase_together(threads)), + ): + self.queue._print_status_bar() + for thread in threads: + thread.start() + for thread in threads: + thread.join() + *rest, bar = screen(output.read()) + self.assertEqual(sorted(rest), lines) + self.assertRegex(bar, r'^In progress: \[[\w|]+\] \d/2 \(\d+%\)$') + def testForceRun(self): data = Value('i', 0) hosts = ['dummy://dummy1', 'dummy://dummy2'] From 503bb3b7149b98684be05ad3e8da944e08eafec1 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 01:58:13 +0100 Subject: [PATCH 173/250] Correct the help of -n of the exscript command It said "Implies -i", but -n never skipped the prompt for the account. It skips the automatic login, so that the template can log in itself. --- CHANGES.md | 4 ++++ src/Exscript/cli.py | 4 +++- tests/Exscript/cliTest.py | 25 +++++++++++++++++++++++++ 3 files changed, 32 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index a66f3952..32785820 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -598,6 +598,10 @@ The names below refer to functions callable from templates, not protocol methods name as given, so such a name failed with `No such file`. - **`exscript -d` With `=` in the Value:** `-d a=b=c` gives the variable `a` the value `b=c`. Before, the command failed with `ValueError: too many values to unpack`. +- **`exscript --no-authentication` Help:** The help no longer says that `-n` implies + `-i`; it never did. `-n` skips the automatic login, and unless `-i` is given, the + command still asks for a username and password, so that the template can log in + itself with them. - **`Exscript.util.mail.send()`:** Ends the SMTP session with `QUIT` and closes the connection, also if sending fails; before, it left the connection open. It connects only after the mail and its attachments are ready. diff --git a/src/Exscript/cli.py b/src/Exscript/cli.py index 91695ced..4c34b20e 100644 --- a/src/Exscript/cli.py +++ b/src/Exscript/cli.py @@ -716,7 +716,9 @@ def _create_parser() -> argparse.ArgumentParser: action="store_true", default=False, help=""" -When given, the automatic authentication procedure is skipped. Implies -i. +Skip the automatic login. Unless -i is given, the command still asks for a +username and password, so that the template can log in itself with them, +e.g. with connection.authenticate(). """.strip(), ) diff --git a/tests/Exscript/cliTest.py b/tests/Exscript/cliTest.py index 4c453640..0316b54c 100644 --- a/tests/Exscript/cliTest.py +++ b/tests/Exscript/cliTest.py @@ -234,6 +234,31 @@ def testLoginPromptInterrupted(self): self.assertEqual(code, 1) self.assertEqual(stdout, '') + def testNoAuthentication(self): + # As the help says, -n skips only the automatic login: without -i, the command asks + # for the login, and the template can log in with it. + code, stdout, stderr = self.run_main('--help') + words = ' '.join(stdout.split()) + help = words[words.index('-n, --no-authentication') : words.index('--authorization ')] + expected = ( + '-n, --no-authentication Skip the automatic login. Unless -i is given, the command' + ' still asks for a username and password, so that the template can log in itself' + ' with them, e.g. with connection.authenticate(). ' + ) + self.assertEqual(help, expected) + + script = '{connection.authenticate()}{sys.message("logged in")}' + with ( + mock.patch('builtins.input', return_value='joe'), + mock.patch('getpass.getpass', return_value='secret'), + ): + code, stdout, stderr = self.run_main( + '-n', '-l', self.path('logs'), '-e', script, 'dummy://r1' + ) + self.assertEqual(code, 0, stderr) + self.assertTrue(stdout.endswith('logged in\n'), stdout) + self.assertIn('User: joe', self.read('logs', 'r1.log')) + def testAuthorizationPrompt(self): with ( mock.patch('builtins.input', return_value='joe'), From 4e49bc454f7f92d410eb552374079ce1a5b00b9c Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 01:59:53 +0100 Subject: [PATCH 174/250] Leave __filename__ undefined for -e of the exscript command With -e, the first argument is a host, but the template got it as __filename__. --- CHANGES.md | 3 +++ src/Exscript/cli.py | 1 + tests/Exscript/cliTest.py | 8 ++++++++ 3 files changed, 12 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 32785820..58125a1f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -602,6 +602,9 @@ The names below refer to functions callable from templates, not protocol methods `-i`; it never did. `-n` skips the automatic login, and unless `-i` is given, the command still asks for a username and password, so that the template can log in itself with them. +- **`exscript -e` and `__filename__`:** With `-e`, the template variable `__filename__` + is `undefined`, as for other templates that are not read from a file. Before, it was + the first host, which `-e` takes from the place of the file name. - **`Exscript.util.mail.send()`:** Ends the SMTP session with `QUIT` and closes the connection, also if sending fails; before, it left the connection open. It connects only after the mail and its attachments are ready. diff --git a/src/Exscript/cli.py b/src/Exscript/cli.py index 4c34b20e..74573c79 100644 --- a/src/Exscript/cli.py +++ b/src/Exscript/cli.py @@ -230,6 +230,7 @@ def mkhosts_from_csv(filename: Path) -> list[Host]: # Extract the hostnames out of the command line arguments. if args.execute and args.filename: args.host = [args.filename] + args.host + args.filename = None hosts = [mkhost(h) for h in args.host] # If a filename containing hostnames AND VARIABLES was given, read it. diff --git a/tests/Exscript/cliTest.py b/tests/Exscript/cliTest.py index 0316b54c..56620470 100644 --- a/tests/Exscript/cliTest.py +++ b/tests/Exscript/cliTest.py @@ -95,6 +95,14 @@ def testExecuteTakesTheFilenameAsHost(self): self.assertEqual(code, 0, stderr) self.assertEqual(sorted(stdout.splitlines()), ['r1', 'r2']) + def testExecuteHasNoFilename(self): + for args in ('-n', '-i'), ('-n', '-i', '--no-prompt'): + code, stdout, stderr = self.run_main( + *args, '-e', '{sys.message(__filename__)}', 'dummy://r1' + ) + self.assertEqual(code, 0, stderr) + self.assertEqual(stdout, 'undefined\n') + def testProtocolAndDefaultDomain(self): args = '-n', '-i', '-p', 'dummy', '--default-domain', 'example.com' code, stdout, stderr = self.run_main(*args, '-e', '{sys.message(__hostname__)}', 'r1') From aac325cab6c29468265b9cc24a9e5be613fe8198 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 02:14:04 +0100 Subject: [PATCH 175/250] Raise LoginFailure at a password prompt for an account without one An account may have no password, e.g. one with only a key. If the device asked for a password or sent an S/Key challenge during the app-level login, the login failed with a TypeError. --- CHANGES.md | 6 ++++++ src/Exscript/protocols/protocol.py | 11 +++++++++-- tests/Exscript/cliTest.py | 12 ++++++++++++ tests/Exscript/protocols/ExpectTest.py | 27 ++++++++++++++++++++++++++ 4 files changed, 54 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 58125a1f..bbcf8d0e 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -187,6 +187,12 @@ marked with **BREAKING**; a migration guide will accompany the release. Before, it raised paramiko's `AuthenticationException` or `SSHException` when the server rejected the key, and an `OSError` when the key file could not be read, so `autologin(attempts=...)` did not retry key logins. +- **App-Level Logins Without a Password:** If the device asks for a password or an + S/Key response, but the account has no password, e.g. an account with only a key + like that of `exscript -i --ssh-key`, `login()`, `app_authenticate()` and + `app_authorize()` raise `LoginFailure`, as for a wrong password, with a message that + says so. Before, they failed with + `TypeError: unsupported operand type(s) for +: 'NoneType' and 'str'`. - **SSH Keys Without Files:** `PrivateKey.from_string()` creates a key from its text, in the format of a key file, e.g. a key from a secret store, which then never needs to be written to a file (upstream issue #209). The type is detected unless it is given. diff --git a/src/Exscript/protocols/protocol.py b/src/Exscript/protocols/protocol.py index 681a9560..1b8a42d4 100644 --- a/src/Exscript/protocols/protocol.py +++ b/src/Exscript/protocols/protocol.py @@ -1263,7 +1263,7 @@ def is_protocol_authenticated(self) -> bool: def _app_authenticate( self, account: _Context, - password: str, + password: str | None, flush: bool = True, bailout: bool = False, ) -> None: @@ -1277,7 +1277,9 @@ def _app_authenticate( :param account: The context of the account whose name is sent at a username prompt. :param password: The password that is sent at a password prompt, or - from which the response to an S/Key challenge is computed. + from which the response to an S/Key challenge is computed. If it + is None, e.g. for an account with only a key, these prompts raise + a LoginFailure. :param flush: Whether to consume the CLI prompt from the buffer. :param bailout: Whether to return after sending the password, without waiting for a CLI prompt. @@ -1335,6 +1337,9 @@ def _app_authenticate( seed = match.group(2) self.otp_requested_event(account, seq, seed) self._dbg(3, "Seq: %s, Seed: %s" % (seq, seed)) + if password is None: + msg = 'The device asks for an S/Key response, but the account has no password' + raise LoginFailure(msg) phrase = otp(password, seed, seq) # A password prompt is now required. @@ -1349,6 +1354,8 @@ def _app_authenticate( elif section == 'password': self._dbg(2, "Password prompt received.") self.expect(prompt) # consume the prompt from the buffer + if password is None: + raise LoginFailure('The device asks for a password, but the account has none') with self._secret(): self.send(password + '\r') if bailout: diff --git a/tests/Exscript/cliTest.py b/tests/Exscript/cliTest.py index 56620470..c57f0047 100644 --- a/tests/Exscript/cliTest.py +++ b/tests/Exscript/cliTest.py @@ -326,6 +326,18 @@ def testSshKeyWithoutLoginPrompt(self): for call in account.call_args_list: self.assertIsInstance(call.kwargs['key'], PrivateKey) + def testSshKeyAtAPasswordPrompt(self): + # The dummy device asks for a password, which the account with the key lacks. + args = '-i', '--ssh-key', KEYFILE, '-l', self.path('logs'), '-e', HELLO, 'dummy://r1' + with mock.patch('getpass.getuser', return_value='jane'): + code, stdout, stderr = self.run_main(*args) + self.assertEqual(code, 1) + self.assertEqual(stderr, 'error: 1 actions failed.\n') + error = self.read('logs', 'r1.log.error') + self.assertIn( + 'LoginFailure: The device asks for a password, but the account has none', error + ) + def testInvalidSshKey(self): key = self.write('id_invalid', 'not a key\n') code, stdout, stderr = self.run_main( diff --git a/tests/Exscript/protocols/ExpectTest.py b/tests/Exscript/protocols/ExpectTest.py index 7f4c15e3..77e0faca 100644 --- a/tests/Exscript/protocols/ExpectTest.py +++ b/tests/Exscript/protocols/ExpectTest.py @@ -308,6 +308,33 @@ def testAuthorizeWithoutPassword(self): self.assertTrue(conn.is_app_authorized()) self.assertEqual(conn.sent, []) + def testPasswordPromptWithoutPassword(self): + # E.g. an account with only a key: the login fails, and the next account may follow. + conn = Dummy(device=VirtualDevice('router', strict=False)) + conn.connect('router') + self.addCleanup(conn.close, force=True) + with self.assertRaises(LoginFailure) as cm: + conn.login(Account('admin', None, needs_lock=False)) + self.assertEqual( + str(cm.exception), 'The device asks for a password, but the account has none' + ) + + conn = self.connect(['Password: ']) + with self.assertRaises(LoginFailure) as cm: + conn.app_authorize(Account('admin', None, needs_lock=False)) + self.assertEqual( + str(cm.exception), 'The device asks for a password, but the account has none' + ) + self.assertEqual(conn.sent, []) + + def testSKeyWithoutPassword(self): + conn = self.connect(['User: ', 'otp-md4 9999 abc123 ext\r\nPassword: ']) + with self.assertRaises(LoginFailure) as cm: + conn.login(Account('admin', None, needs_lock=False)) + expected = 'The device asks for an S/Key response, but the account has no password' + self.assertEqual(str(cm.exception), expected) + self.assertEqual(conn.sent, ['admin\r']) + def testAccountForTheDevice(self): # The app account logs into the device; the other account is for the protocol. conn = self.connect(['User: ', 'Password: ', '\r\nrouter> ']) From 2130cb73e30cbccdde47926dfa7a32e275620abd Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 00:58:41 +0100 Subject: [PATCH 176/250] Leave if __name__ == "__main__" blocks out of the coverage The tests would have to run a module as a script to reach such a block. --- pyproject.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index a4efc47a..a3dfd8c6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -146,6 +146,9 @@ exclude = ["docs", "tests", "tools", "demos"] branch = true patch = ["subprocess"] # e.g. python -m Exscript in the tests of the command +[tool.coverage.report] +exclude_also = ['if __name__ == .__main__.:'] + [tool.pytest.ini_options] testpaths = ["tests"] python_files = ["*Test.py"] From 1687d75478cb3b3a51f1b78a5a75f527a47b7fdb Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 01:04:29 +0100 Subject: [PATCH 177/250] Test the logging setup, and the errors and options of log files --- tests/Exscript/FileLoggerTest.py | 31 +++++++ tests/Exscript/LogTest.py | 21 +++++ tests/Exscript/LogfileTest.py | 48 ++++++++++ tests/Exscript/LoggerTest.py | 153 ++++++++++++++++++++++++++++++- 4 files changed, 251 insertions(+), 2 deletions(-) diff --git a/tests/Exscript/FileLoggerTest.py b/tests/Exscript/FileLoggerTest.py index 64b06518..24a5d683 100644 --- a/tests/Exscript/FileLoggerTest.py +++ b/tests/Exscript/FileLoggerTest.py @@ -1,4 +1,5 @@ import os.path +import sys import unittest from shutil import rmtree from tempfile import mkdtemp @@ -86,6 +87,36 @@ def testLogSucceeded2(self): self.assertTrue(os.path.isfile(self.logfile)) self.assertFalse(os.path.exists(self.errfile)) + def _runJobs(self, logger): + # Logs one job that succeeds and one that fails. + job1, job2 = FakeJob('job1'), FakeJob('job2') + for job in job1, job2: + logger.add_log(id(job), job.name, 1) + logger.log(id(job), 'hello') + logger.log_succeeded(id(job1)) + try: + raise FakeError('Connection refused') + except FakeError: + logger.log_aborted(id(job2), sys.exc_info()) + + def testClearmem(self): + # The logger forgets the logs of finished jobs, but keeps the files. + logger = FileLogger(self.logdir) + self._runJobs(logger) + self.assertEqual(logger.get_logs(), []) + self.assertEqual(logger.get_succeeded_actions(), 1) + self.assertEqual(logger.get_aborted_actions(), 1) + files = ['job1.log', 'job2.log', 'job2.log.error'] + self.assertEqual(sorted(os.listdir(self.logdir)), files) + + def testDelete(self): + # The log of a job that succeeds is deleted, the one of a failed job is kept. + logger = FileLogger(self.logdir, delete=True) + self._runJobs(logger) + self.assertEqual(sorted(os.listdir(self.logdir)), ['job2.log', 'job2.log.error']) + with open(os.path.join(self.logdir, 'job2.log'), encoding='utf-8') as fp: + self.assertEqual(fp.read(), 'helloERROR: Connection refused \n') + def suite(): return unittest.TestLoader().loadTestsFromTestCase(FileLoggerTest) diff --git a/tests/Exscript/LogTest.py b/tests/Exscript/LogTest.py index 33d2b423..07f7a1b1 100644 --- a/tests/Exscript/LogTest.py +++ b/tests/Exscript/LogTest.py @@ -25,6 +25,27 @@ def testGetError(self): self.log.aborted(sys.exc_info()) self.assertIn('FakeError', self.log.get_error()) + def testGetErrorWithoutTraceback(self): + try: + raise FakeError('Connection refused') + except FakeError: + self.log.aborted(sys.exc_info()) + self.assertEqual(self.log.get_error(include_tb=False), 'Connection refused') + self.assertIn('Traceback (most recent call last)', self.log.get_error()) + + def testGetErrorWithoutTracebackAndMessage(self): + # Without a message, the name of the exception class is the error. + try: + raise FakeError() + except FakeError: + self.log.aborted(sys.exc_info()) + self.assertEqual(self.log.get_error(include_tb=False), 'FakeError') + + def testLen(self): + self.assertEqual(len(self.log), 0) + self.log.write('hello', 'world') + self.assertEqual(len(self.log), len('hello world')) + def testGetName(self): self.assertEqual(self.log.get_name(), 'testme') self.log.started() diff --git a/tests/Exscript/LogfileTest.py b/tests/Exscript/LogfileTest.py index de0f0b6c..68e2416d 100644 --- a/tests/Exscript/LogfileTest.py +++ b/tests/Exscript/LogfileTest.py @@ -1,9 +1,13 @@ +import io import os.path +import sys import unittest +from contextlib import redirect_stdout from shutil import rmtree from tempfile import mkdtemp from LogTest import LogTest +from util.reportTest import FakeError from Exscript.logger import Logfile @@ -41,6 +45,50 @@ def testSucceeded(self): self.assertTrue(os.path.exists(self.logfile)) self.assertFalse(os.path.exists(self.errorfile)) + def testCreatesTheDirectoryOfTheFile(self): + # Only the directory of the file is created, not its parents. + logfile = os.path.join(self.tempdir, 'logs', 'test.log') + Logfile('testme', logfile).started() + self.assertTrue(os.path.isfile(logfile)) + logfile = os.path.join(self.tempdir, 'missing', 'logs', 'test.log') + self.assertRaises(FileNotFoundError, Logfile, 'testme', logfile) + + def testFileInTheWorkingDirectory(self): + cwd = os.getcwd() + os.chdir(self.tempdir) + try: + log = Logfile('testme', 'test.log') + log.started() + log.write('hello') + finally: + os.chdir(cwd) + with open(self.logfile, encoding='utf-8') as fp: + self.assertEqual(fp.read(), 'hello') + + def testWriteError(self): + # The error is shown and raised, and the log writes nothing more. + os.mkdir(self.logfile) # a directory where the file should be + output = io.StringIO() + with redirect_stdout(output): + self.assertRaises(OSError, self.log.started) + try: + raise FakeError('Connection refused') + except FakeError: + self.log.aborted(sys.exc_info()) + lines = output.getvalue().splitlines() + self.assertEqual(len(lines), 1, lines) + self.assertTrue(lines[0].startswith('Error writing to %s: ' % self.logfile), lines) + self.assertEqual(self.log.get_error(False), 'Connection refused') + self.assertFalse(os.path.exists(self.errorfile)) + + def testDeleteWhenSucceeded(self): + log = Logfile('testme', self.logfile, delete=True) + log.started() + log.write('hello') + log.succeeded() + self.assertFalse(os.path.exists(self.logfile)) + self.assertFalse(os.path.exists(self.errorfile)) + def suite(): return unittest.TestLoader().loadTestsFromTestCase(LogfileTest) diff --git a/tests/Exscript/LoggerTest.py b/tests/Exscript/LoggerTest.py index b621cc00..3a1fbd68 100644 --- a/tests/Exscript/LoggerTest.py +++ b/tests/Exscript/LoggerTest.py @@ -1,11 +1,17 @@ +import io +import logging +import os import sys +import tempfile +import time import unittest +from contextlib import redirect_stderr from itertools import islice from LogTest import FakeError from util.reportTest import FakeJob -from Exscript.logger import Log, Logger +from Exscript.logger import Log, Logger, getConsoleLogger, setupLogging def count(iterable): @@ -171,8 +177,151 @@ def testLogSucceeded(self): self._logSucceeded() +class ListHandler(logging.Handler): + """Collects the records that pass its filters.""" + + def __init__(self): + logging.Handler.__init__(self) + self.records = [] + + def emit(self, record): + self.records.append(record) + + +class LoggingSetupTest(unittest.TestCase): + """getConsoleLogger() and setupLogging(), the logging of the Python loggers.""" + + def getConsoleLogger(self, name, **kwargs): + # The handler writes to the stream that is sys.stderr when it is created. + stderr = io.StringIO() + with redirect_stderr(stderr): + logger = getConsoleLogger(name, **kwargs) + self.addCleanup(self.resetLogger, logger) + return logger, stderr + + def resetLogger(self, logger): + for handler in logger.handlers[:]: + logger.removeHandler(handler) + handler.close() + logger.setLevel(logging.NOTSET) + logger.propagate = True + + def testGetConsoleLogger(self): + logger, stderr = self.getConsoleLogger('LoggerTest.console', level=logging.INFO) + self.assertIs(logger, logging.getLogger('LoggerTest.console')) + self.assertFalse(logger.propagate) + logger.debug('below the level of the handler') + logger.info('connected') + self.assertEqual(stderr.getvalue(), 'INFO: LoggerTest.console (MainThread) connected\n') + + # A logger that has a console handler is returned unchanged. + again, _ = self.getConsoleLogger('LoggerTest.console', level=logging.DEBUG) + self.assertIs(again, logger) + self.assertEqual(len(logger.handlers), 1) + logger.debug('below the level of the handler') + self.assertEqual(stderr.getvalue(), 'INFO: LoggerTest.console (MainThread) connected\n') + + def testGetConsoleLoggerWithFileHandler(self): + # A FileHandler is a StreamHandler, but it does not write to the console. + tmpdir = self.enterContext(tempfile.TemporaryDirectory()) + filename = os.path.join(tmpdir, 'test.log') + logging.getLogger('LoggerTest.file').addHandler( + logging.FileHandler(filename, encoding='utf-8', delay=True) + ) + logger, stderr = self.getConsoleLogger('LoggerTest.file') + self.assertEqual(len(logger.handlers), 2) + logger.info('connected') + self.assertEqual(stderr.getvalue(), 'INFO: LoggerTest.file (MainThread) connected\n') + logger.handlers[0].close() + with open(filename, encoding='utf-8') as fp: + self.assertEqual(fp.read(), 'connected\n') + + def testGetConsoleLoggerTimestamps(self): + record = logging.makeLogRecord({'msg': 'hello', 'created': 0.0, 'msecs': 0.0}) + fmt = '%(asctime)s %(message)s' + utc, _ = self.getConsoleLogger('LoggerTest.utc', fmt=fmt) + local, _ = self.getConsoleLogger('LoggerTest.local', fmt=fmt, utc=False) + self.assertEqual(utc.handlers[0].format(record), '1970-01-01 00:00:00,000 hello') + expected = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(0)) + ',000 hello' + self.assertEqual(local.handlers[0].format(record), expected) + + def setupLogging(self, **kwargs): + # Calls setupLogging() with loggers that are not registered in the + # logging module, so that the test changes no global logger. + loggers = {} + + def init(name): + logger = logging.Logger(name) + logger.addHandler(ListHandler()) + loggers[name] = logger + return logger + + setupLogging(init=init, **kwargs) + return loggers + + def passes(self, logger, name): + # Whether the handler of the logger passes a record of the logger with the given name. + handler = logger.handlers[0] + handler.records.clear() + logger.handle(logging.LogRecord(name, logging.INFO, __file__, 1, 'message', None, None)) + return len(handler.records) == 1 + + def testSetupLogging(self): + loggers = self.setupLogging() + names = { + 'Exscript', + 'Exscript.interpreter.parser', + 'Exscript.parselib.lexer', + 'Exscript.protocols', + 'Exscript.servers', + 'ncclient.capabilities', + 'ncclient.devices', + 'ncclient.manager', + 'ncclient.operations', + 'ncclient.transport.session', + 'ncclient.transport.ssh', + 'paramiko.transport', + } + self.assertEqual(set(loggers), names) + for name, logger in loggers.items(): + self.assertFalse(self.passes(logger, name), name) + + def testSetupLoggingPackage(self): + loggers = self.setupLogging(package=True) + self.assertTrue(self.passes(loggers['Exscript'], 'Exscript')) + self.assertTrue(self.passes(loggers['Exscript'], 'Exscript.queue')) + self.assertFalse(self.passes(loggers['Exscript'], 'ExscriptX')) + self.assertFalse(self.passes(loggers['Exscript.protocols'], 'Exscript.protocols.ssh2')) + self.assertFalse(self.passes(loggers['paramiko.transport'], 'paramiko.transport')) + + def testSetupLoggingParser(self): + loggers = self.setupLogging(parser=True) + self.assertTrue(self.passes(loggers['Exscript.parselib.lexer'], 'Exscript.parselib.lexer')) + logger = loggers['Exscript.interpreter.parser'] + self.assertTrue(self.passes(logger, 'Exscript.interpreter.parser')) + self.assertFalse(self.passes(loggers['Exscript'], 'Exscript.queue')) + + def testSetupLoggingProtocol(self): + loggers = self.setupLogging(protocol=True) + logger = loggers['Exscript.protocols'] + self.assertTrue(self.passes(logger, 'Exscript.protocols')) + self.assertTrue(self.passes(logger, 'Exscript.protocols.ssh2')) + self.assertFalse(self.passes(logger, 'Exscript.protocolsX')) + self.assertTrue(self.passes(loggers['Exscript.servers'], 'Exscript.servers.sshd')) + self.assertTrue(self.passes(loggers['paramiko.transport'], 'paramiko.transport')) + logger = loggers['ncclient.transport.ssh'] + self.assertTrue(self.passes(logger, 'ncclient.transport.ssh')) + self.assertFalse(self.passes(loggers['Exscript'], 'Exscript.queue')) + + def suite(): - return unittest.TestLoader().loadTestsFromTestCase(LoggerTest) + loader = unittest.TestLoader() + return unittest.TestSuite( + [ + loader.loadTestsFromTestCase(LoggerTest), + loader.loadTestsFromTestCase(LoggingSetupTest), + ] + ) if __name__ == '__main__': From f7319481a9e0060336e07c846848e7c1029ffb14 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 01:05:56 +0100 Subject: [PATCH 178/250] Keep all output of a job in a log file that is opened with mode w Logfile opened the file in the given mode for every write, so with 'w' each write truncated it, e.g. with exscript --overwrite-logs. The mode now applies to the first write only; later writes append. --- CHANGES.md | 4 ++++ src/Exscript/logger.py | 15 ++++++++++----- tests/Exscript/FileLoggerTest.py | 30 ++++++++++++++++++++++++++++++ 3 files changed, 44 insertions(+), 5 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index bbcf8d0e..62fc0144 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -630,6 +630,10 @@ The names below refer to functions callable from templates, not protocol methods and the copies keep the default factory and the order of the keys. Before, `copy.deepcopy()` failed with `TypeError: cannot pickle 'odict_items' object`, and `pickle.dumps()` with a `PicklingError`. +- **Overwritten Log Files:** `FileLogger(mode='w')`, `log_to_file(mode='w')` and + `exscript --overwrite-logs` overwrite the log file of an earlier run and keep all + output of the job. Before, every write overwrote the file, so it kept only the last + piece of the output, e.g. the last prompt. - **BREAKING - Removed Modules:** `Exscript.util.daemonize`, `Exscript.util.pidutil`, `Exscript.util.sigint` and `Exscript.util.sigintcatcher` are removed; `daemonize()` failed on Python 3 with `ValueError: can't have unbuffered text I/O`. Run long-lived scripts under a service manager such as systemd diff --git a/src/Exscript/logger.py b/src/Exscript/logger.py index 3e99baa5..9a90eec9 100644 --- a/src/Exscript/logger.py +++ b/src/Exscript/logger.py @@ -253,7 +253,9 @@ def __init__(self, name: str, filename: str, mode: str = 'a', delete: bool = Fal traceback to the same file name with the suffix '.error'. The directory of the file is created if it does not exist. - :param mode: The mode in which the files are opened for each write. + :param mode: The mode in which each file is opened for the first + write, e.g. 'w' to overwrite the file of an earlier run; the + later writes append to it. :param delete: Whether to delete the file when the log succeeds without an error. """ @@ -263,6 +265,7 @@ def __init__(self, name: str, filename: str, mode: str = 'a', delete: bool = Fal self.mode = mode self.delete = delete self.do_log = True + self._written: set[str] = set() # the files that later writes append to dirname = os.path.dirname(filename) if dirname: try: @@ -283,19 +286,21 @@ def __str__(self) -> str: def _write_file(self, filename: str, *data: str) -> None: """ - Writes the given strings to the given file, separated by spaces. - If writing fails, the error is raised, and later writes are - skipped. + Writes the given strings to the given file, separated by spaces, in + the mode of the log on the first write and appending after that. If + writing fails, the error is raised, and later writes are skipped. """ if not self.do_log: return + mode = 'a' if filename in self._written else self.mode try: - with open(filename, self.mode, encoding='utf-8') as thefile: + with open(filename, mode, encoding='utf-8') as thefile: thefile.write(' '.join(data)) except Exception as e: print('Error writing to %s: %s' % (filename, e)) self.do_log = False raise + self._written.add(filename) def write(self, *data: str) -> None: """ diff --git a/tests/Exscript/FileLoggerTest.py b/tests/Exscript/FileLoggerTest.py index 24a5d683..25063c8b 100644 --- a/tests/Exscript/FileLoggerTest.py +++ b/tests/Exscript/FileLoggerTest.py @@ -117,6 +117,36 @@ def testDelete(self): with open(os.path.join(self.logdir, 'job2.log'), encoding='utf-8') as fp: self.assertEqual(fp.read(), 'helloERROR: Connection refused \n') + def testMode(self): + # 'a' appends to the log of an earlier run, 'w' overwrites it; both + # keep all output of the job. + for mode, expected in ('a', 'earlier run\nhello\nworld\n'), ('w', 'hello\nworld\n'): + with self.subTest(mode=mode): + with open(self.logfile, 'w', encoding='utf-8') as fp: + fp.write('earlier run\n') + logger = FileLogger(self.logdir, mode=mode) + logger.add_log(id(self.job), self.job.name, 1) + logger.log(id(self.job), 'hello\n') + logger.log(id(self.job), 'world\n') + logger.log_succeeded(id(self.job)) + with open(self.logfile, encoding='utf-8') as fp: + self.assertEqual(fp.read(), expected) + + def testModeOfTheErrorFile(self): + with open(self.errfile, 'w', encoding='utf-8') as fp: + fp.write('error of an earlier run\n') + logger = FileLogger(self.logdir, mode='w') + logger.add_log(id(self.job), self.job.name, 1) + logger.log(id(self.job), 'hello\n') + try: + raise FakeError('Connection refused') + except FakeError: + logger.log_aborted(id(self.job), sys.exc_info()) + with open(self.logfile, encoding='utf-8') as fp: + self.assertEqual(fp.read(), 'hello\nERROR: Connection refused \n') + with open(self.errfile, encoding='utf-8') as fp: + self.assertTrue(fp.read().startswith('Traceback (most recent call last):')) + def suite(): return unittest.TestLoader().loadTestsFromTestCase(FileLoggerTest) From 0d8d343a9ddd6443bbd17d6177afdb770cc689e8 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 01:06:51 +0100 Subject: [PATCH 179/250] Release the accounts of owners in AccountPool.reset() reset() failed with a RuntimeError if an owner held an account: it iterated over the owners while release_accounts() removed them. --- CHANGES.md | 4 ++++ src/Exscript/account.py | 2 +- tests/Exscript/AccountPoolTest.py | 11 +++++++++++ 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 62fc0144..c1b199aa 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -655,6 +655,10 @@ The names below refer to functions callable from templates, not protocol methods `Exscript.account.LoggerProxy`, `Exscript.workqueue.job.Process` and `Queue._create_pipe()`. The queue, accounts and account pools use the locks of `threading` instead of those of `multiprocessing`. +- **`AccountPool.reset()` With Owners:** Releases the accounts that owners acquired, + then removes all accounts. Before, it failed with `RuntimeError: dictionary changed + size during iteration` if an account was acquired with an owner, as in + `pool.acquire_account(owner=job)`. - **Order of Sleeping Items:** `next()` and `try_next()` of `Exscript.workqueue.Pipeline` leave the sleeping items at the start of the queue in their order, so they come next in that order after `wake()`. Before, each call diff --git a/src/Exscript/account.py b/src/Exscript/account.py index 4b981469..1c65b70c 100644 --- a/src/Exscript/account.py +++ b/src/Exscript/account.py @@ -337,7 +337,7 @@ def reset(self) -> None: Removes all accounts. """ with self.unlock_cond: - for owner in self.owner2account: + for owner in list(self.owner2account): self.release_accounts(owner) self._remove_account(self.accounts.copy()) self.unlock_cond.notify_all() diff --git a/tests/Exscript/AccountPoolTest.py b/tests/Exscript/AccountPoolTest.py index 3aba0f86..fd5d8062 100644 --- a/tests/Exscript/AccountPoolTest.py +++ b/tests/Exscript/AccountPoolTest.py @@ -38,6 +38,17 @@ def testReset(self): self.accm.reset() self.assertEqual(self.accm.n_accounts(), 0) + def testResetReleasesTheAccountsOfOwners(self): + self.accm.add_account([self.account1, self.account2]) + self.accm.acquire_account(self.account1, owner='job1') + self.accm.acquire_account(owner='job2') + self.accm.reset() + self.assertEqual(self.accm.n_accounts(), 0) + for account in self.account1, self.account2: + self.assertFalse(account.lock.locked()) + account.acquire() # the pool no longer follows the account + account.release() + def testHasAccount(self): self.assertEqual(self.accm.has_account(self.account1), False) self.accm.add_account(self.account1) From 757c3fcf2f398172379fed12609c0246e5c119c0 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 01:09:31 +0100 Subject: [PATCH 180/250] Test the locking of accounts and the account pools --- tests/Exscript/AccountManagerTest.py | 25 ++++++++++ tests/Exscript/AccountPoolTest.py | 75 ++++++++++++++++++++++++++++ tests/Exscript/AccountTest.py | 42 ++++++++++++++++ 3 files changed, 142 insertions(+) diff --git a/tests/Exscript/AccountManagerTest.py b/tests/Exscript/AccountManagerTest.py index f48a37e4..0618528d 100644 --- a/tests/Exscript/AccountManagerTest.py +++ b/tests/Exscript/AccountManagerTest.py @@ -1,5 +1,6 @@ import unittest +from Exscript import Host from Exscript.account import Account, AccountManager, AccountPool @@ -99,6 +100,30 @@ def start_cb(data, conn): self.assertEqual(self.data, {'match-called': True, 'host': 'myhost'}) self.assertEqual(self.account, account) + def testPoolsWithMatchFunctions(self): + router_account = Account('router-admin') + switch_account = Account('switch-admin') + routers = AccountPool(router_account) + switches = AccountPool(switch_account) + self.am.add_pool(routers, lambda host: host.get_name().startswith('router')) + self.am.add_pool(switches, lambda host: host.get_name().startswith('switch')) + self.am.add_account(self.account) + + # The first pool whose match function returns True is used, else the default pool. + self.assertIs(self.am.acquire_account_for(Host('switch1'), 'job1'), switch_account) + self.assertIs(self.am.acquire_account_for(Host('server1'), 'job1'), self.account) + self.am.release_accounts('job1') + + # A given account is acquired from its pool. + self.assertIs(self.am.acquire_account(router_account, 'job2'), router_account) + self.assertNotIn(router_account, routers.unlocked_accounts) + self.am.release_accounts('job2') + self.assertIn(router_account, routers.unlocked_accounts) + + for account in router_account, switch_account, self.account: + self.assertIs(self.am.get_account_from_hash(account.__hash__()), account) + self.assertIsNone(self.am.get_account_from_hash(Account('other').__hash__())) + def testReleaseAccounts(self): account1 = Account('foo') pool = AccountPool() diff --git a/tests/Exscript/AccountPoolTest.py b/tests/Exscript/AccountPoolTest.py index fd5d8062..1233655f 100644 --- a/tests/Exscript/AccountPoolTest.py +++ b/tests/Exscript/AccountPoolTest.py @@ -1,6 +1,9 @@ import os.path +import threading import unittest +from AccountTest import signal_wait + from Exscript import Account from Exscript.account import AccountPool from Exscript.util.file import get_accounts_from_file @@ -49,6 +52,12 @@ def testResetReleasesTheAccountsOfOwners(self): account.acquire() # the pool no longer follows the account account.release() + def testResetRefusesAnAccountWithoutOwner(self): + self.accm.add_account(self.account1) + self.accm.acquire_account() + self.assertRaisesRegex(Exception, 'should be unlocked', self.accm.reset) + self.assertTrue(self.accm.has_account(self.account1)) + def testHasAccount(self): self.assertEqual(self.accm.has_account(self.account1), False) self.accm.add_account(self.account1) @@ -59,10 +68,12 @@ def testGetAccountFromHash(self): thehash = account.__hash__() self.accm.add_account(account) self.assertEqual(self.accm.get_account_from_hash(thehash), account) + self.assertIsNone(self.accm.get_account_from_hash(self.account1.__hash__())) def testGetAccountFromName(self): self.testAddAccount() self.assertEqual(self.account2, self.accm.get_account_from_name(self.user2)) + self.assertIsNone(self.accm.get_account_from_name('admin')) def testNAccounts(self): self.testAddAccount() @@ -100,6 +111,70 @@ def testAcquireAccount(self): for account in list(acquired.values()): account.release() + def acquireInThread(self, account=None): + # Calls acquire_account() in a thread, and returns when it waits. + waiting = signal_wait(self.accm.unlock_cond) + result = [] + thread = threading.Thread( + target=lambda: result.append(self.accm.acquire_account(account)), daemon=True + ) + thread.start() + self.assertTrue(waiting.wait(5)) + return thread, result + + def testAcquireAccountWaitsForTheAccount(self): + self.accm.add_account([self.account1, self.account2]) + self.accm.acquire_account(self.account1) + thread, result = self.acquireInThread(self.account1) + self.assertEqual(result, []) # though account2 is free + self.account1.release() + thread.join(5) + self.assertEqual(result, [self.account1]) + self.assertTrue(self.account1.lock.locked()) + self.assertEqual(list(self.accm.unlocked_accounts), [self.account2]) + + def testAcquireAccountWaitsForAnyAccount(self): + self.accm.add_account([self.account1, self.account2]) + self.accm.acquire_account() + self.accm.acquire_account() + thread, result = self.acquireInThread() + self.assertEqual(result, []) + self.account2.release() + thread.join(5) + self.assertEqual(result, [self.account2]) + self.assertTrue(self.account2.lock.locked()) + + def testAccountAcquiredOutsideThePool(self): + # The pool does not lend an account that was acquired directly. + self.accm.add_account([self.account1, self.account2]) + self.account1.acquire() + self.assertIs(self.accm.acquire_account(), self.account2) + self.account1.release() + self.assertIs(self.accm.acquire_account(), self.account1) + + def testAccountOfAnOwnerReleasedDirectly(self): + # The account no longer belongs to the owner, so release_accounts() + # does not release it when someone else has acquired it since. + self.accm.add_account(self.account1) + self.accm.acquire_account(owner='job1') + self.account1.release() + self.accm.acquire_account(owner='job2') + self.accm.release_accounts('job1') + self.assertTrue(self.account1.lock.locked()) + self.accm.release_accounts('job2') + self.assertFalse(self.account1.lock.locked()) + + def testSignalsThatDoNotMatchThePool(self): + # Locking an account of the pool without a signal, and unlocking it + # with one, or the other way around, is detected. + self.accm.add_account(self.account1) + self.account1.acquire(signal=False) + self.assertRaisesRegex(Exception, 'should be locked', self.account1.release) + self.account1.acquire() + self.account1.release(signal=False) + self.assertRaisesRegex(Exception, 'is already locked', self.account1.acquire) + self.account1.release(signal=False) + def testReleaseAccounts(self): account1 = Account('foo') account2 = Account('bar') diff --git a/tests/Exscript/AccountTest.py b/tests/Exscript/AccountTest.py index baf3fbe3..b1343c40 100644 --- a/tests/Exscript/AccountTest.py +++ b/tests/Exscript/AccountTest.py @@ -1,8 +1,22 @@ +import threading import unittest from Exscript import Account, PrivateKey +def signal_wait(condition): + """Returns an event that is set when a thread waits for the given condition.""" + waiting = threading.Event() + wait = condition.wait + + def wait_and_signal(timeout=None): + waiting.set() + return wait(timeout) + + condition.wait = wait_and_signal + return waiting + + class AccountTest(unittest.TestCase): CORRELATE = Account @@ -47,6 +61,34 @@ def testAcquire(self): self.account.acquire() self.account.release() + def testAcquireWaitsForTheRelease(self): + self.account.acquire() + waiting = signal_wait(self.account.synclock) + thread = threading.Thread(target=self.account.acquire, daemon=True) + thread.start() + self.assertTrue(waiting.wait(5)) + acquired = [] + self.account.acquired_event.connect(acquired.append) + self.account.release() + thread.join(5) + self.assertFalse(thread.is_alive()) + self.assertEqual(acquired, [self.account]) + self.assertTrue(self.account.lock.locked()) + self.account.release() + + def testWithoutLock(self): + account = Account(self.user, self.password1, needs_lock=False) + events = [] + account.acquired_event.connect(events.append) + account.released_event.connect(events.append) + with account: + self.assertFalse(account.lock.locked()) + account.acquire() # does not wait + account.release() + account.release() + self.assertFalse(account.lock.locked()) + self.assertEqual(events, []) + def testRelease(self): self.assertRaises(Exception, self.account.release) From 6355d2a5a0b88bbbdaed633e90c3325004fa9ede Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 01:10:59 +0100 Subject: [PATCH 181/250] Test the errors and options of the readers of host and account files --- tests/Exscript/util/fileTest.py | 94 +++++++++++++++++++++++++++++++++ 1 file changed, 94 insertions(+) diff --git a/tests/Exscript/util/fileTest.py b/tests/Exscript/util/fileTest.py index 3d6186a8..33191059 100644 --- a/tests/Exscript/util/fileTest.py +++ b/tests/Exscript/util/fileTest.py @@ -1,5 +1,7 @@ import base64 import os.path +import re +import tempfile import unittest from tempfile import NamedTemporaryFile @@ -69,6 +71,98 @@ def testLoadLib(self): name = os.path.splitext(os.path.basename(self.lib_file.name))[0] self.assertEqual({name + '.test': object}, functions) + def testMissingFiles(self): + from Exscript.util.file import get_hosts_from_csv, get_hosts_from_file, load_lib + + tmpdir = self.enterContext(tempfile.TemporaryDirectory()) + filename = os.path.join(tmpdir, 'missing') + for func in get_hosts_from_file, get_hosts_from_csv, load_lib: + with self.subTest(func=func.__name__): + with self.assertRaisesRegex(OSError, 'No such file: ' + re.escape(filename)): + func(filename) + + def testGetAccountsFromFileWithOtherSections(self): + from Exscript.util.file import get_accounts_from_file + + data = '[other]\nuser9 = cGFzc3dvcmQ=\n[account-pool]\nAdmin = cGFzc3dvcmQ=\n' + accounts = get_accounts_from_file(self.write_tempfile(data.encode('utf8')).name) + self.assertEqual( + [(a.get_name(), a.get_password()) for a in accounts], [('Admin', 'password')] + ) + + def testGetAccountsFromFileWithInvalidPasswords(self): + from Exscript.util.file import get_accounts_from_file + + for password in 'not base64!', base64.b64encode(b'\xffpassword').decode('ascii'): + with self.subTest(password=password): + data = '[account-pool]\nuser1 = %s\n' % password + filename = self.write_tempfile(data.encode('utf8')).name + with self.assertRaisesRegex(Exception, 'Could not decode password for user: user1'): + get_accounts_from_file(filename) + + def testGetAccountsFromFileWithPasswordEncoding(self): + from Exscript.util.file import get_accounts_from_file + + password = base64.b64encode('pässword'.encode('latin-1')).decode('ascii') + data = '[account-pool]\nuser1 = %s\n' % password + filename = self.write_tempfile(data.encode('utf8')).name + accounts = get_accounts_from_file(filename, password_encoding='latin-1') + self.assertEqual(accounts[0].get_password(), 'pässword') + + def testGetHostsFromFileWithCommentsAndDuplicates(self): + from Exscript.util.file import get_hosts_from_file + + data = '# core routers\nrouter1\n\nrouter2 # backup\nrouter1\ntelnet://switch1\n' + filename = self.write_tempfile(data.encode('utf8')).name + hosts = get_hosts_from_file(filename, default_protocol='telnet') + self.assertEqual( + [h.get_name() for h in hosts], ['router1', 'router2', 'router1', 'switch1'] + ) + self.assertEqual({h.get_protocol() for h in hosts}, {'telnet'}) + hosts = get_hosts_from_file(filename, remove_duplicates=True, default_domain='example.com') + self.assertEqual( + [(h.get_protocol(), h.get_address()) for h in hosts], + [ + ('ssh', 'router1.example.com'), + ('ssh', 'router2.example.com'), + ('telnet', 'switch1.example.com'), + ], + ) + + def testGetHostsFromCsvWithInvalidHeader(self): + from Exscript.util.file import get_hosts_from_csv + + for header, error in ( + ('name\tvar', 'File does not start with "hostname" or "address".'), + ('address var', 'Make sure to separate columns by tabs.'), + ('address\t\tvar', 'Make sure to separate columns by tabs.'), + ): + with self.subTest(header=header): + filename = self.write_tempfile((header + '\n1.2.3.4\tvalue\n').encode('utf8')).name + with self.assertRaisesRegex(Exception, re.escape(error)): + get_hosts_from_csv(filename) + + def testGetHostsFromCsvWithSeveralLinesPerHost(self): + # The example of the documentation, with an empty line and a missing value. + from Exscript.util.file import get_hosts_from_csv + + data = ( + 'address\thostname\ttestvar1\ttestvar2\n' + '10.0.0.1\tmyhost\tvalue1\tothervalue\n' + '\n' + '10.0.0.1\tmyhost\tvalue2\tothervalue2\n' + '10.0.0.2\totherhost\tfoo\n' + ) + filename = self.write_tempfile(data.encode('utf8')).name + hosts = get_hosts_from_csv(filename, default_protocol='telnet') + self.assertEqual([h.get_address() for h in hosts], ['10.0.0.1', '10.0.0.2']) + self.assertEqual([h.get_name() for h in hosts], ['myhost', 'otherhost']) + self.assertEqual([h.get_protocol() for h in hosts], ['telnet', 'telnet']) + self.assertEqual(hosts[0].get('testvar1'), ['value1', 'value2']) + self.assertEqual(hosts[0].get('testvar2'), ['othervalue', 'othervalue2']) + self.assertEqual(hosts[1].get('testvar1'), ['foo']) + self.assertEqual(hosts[1].get('testvar2'), ['']) + def suite(): return unittest.TestLoader().loadTestsFromTestCase(fileTest) From 455be7c3c9c29d0d62ef99a43768b142409cf9b9 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 01:12:48 +0100 Subject: [PATCH 182/250] Test the placeholders, header errors and attachments of mail templates --- tests/Exscript/util/mailTest.py | 65 +++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/tests/Exscript/util/mailTest.py b/tests/Exscript/util/mailTest.py index dbcc2b5b..63f9e837 100644 --- a/tests/Exscript/util/mailTest.py +++ b/tests/Exscript/util/mailTest.py @@ -207,6 +207,71 @@ def testSend(self): self.assertEqual(message.get_payload()[0].get_payload(decode=True), 'Grüße'.encode()) smtp.return_value.__exit__.assert_called_once() # QUIT, and the connection is closed + def testFromTemplateStringPlaceholders(self): + from Exscript.util.mail import from_template_string + + # A line that ends with a space is joined with the next one, and the + # items of a list are put on separate lines. + tmpl = 'From: me@example.com\nSubject: {count} hosts \nfailed\n\nHosts:\n{hosts}' + mail = from_template_string(tmpl, count=2, hosts=['router1', 'router2']) + self.assertEqual(mail.get_subject(), '2 hosts failed') + self.assertEqual(mail.get_body(), 'Hosts:\nrouter1\nrouter2') + + def testFromTemplateStringErrors(self): + from Exscript.util.mail import from_template_string + + for tmpl, error in ( + ('From: me@example.com\n\n{hosts}', 'Undefined value for hosts'), + ('From: me@example.com\n\n{1st}', '1st is not a variable name'), + ('From: me@example.com\nPriority: high\n\nbody', 'Invalid header field "priority"'), + ): + with self.subTest(tmpl=tmpl): + self.assertRaisesRegex(Exception, error, from_template_string, tmpl) + + def testFromTemplateStringWithoutBody(self): + from Exscript.util.mail import from_template_string + + # Also with a blank at the end of the last line, which joins no line. + for tmpl in 'To: you@example.com\nSubject: Report', 'To: you@example.com\nSubject: Report ': + with self.subTest(tmpl=tmpl): + with mock.patch('Exscript.util.mail.getuser', return_value='operator'): + mail = from_template_string(tmpl) + self.assertTrue(mail.get_sender().startswith('operator@'), mail.get_sender()) + self.assertEqual(mail.get_to(), ['you@example.com']) + self.assertEqual(mail.get_subject(), 'Report') + self.assertEqual(mail.get_body(), '') + + def testSendAttachments(self): + # The parts get a content type from the file name, application/octet-stream + # for unknown and compressed files. + tmpdir = self.enterContext(tempfile.TemporaryDirectory()) + files = { + 'report.txt': ('text/plain', 'Grüße\n'.encode()), + 'graph.png': ('image/png', b'\x89PNG\r\n\x1a\n'), + 'alarm.wav': ('audio', b'RIFF\x00\x00\x00\x00WAVE'), + 'config.txt.gz': ('application/octet-stream', b'\x1f\x8b\x08\x00'), + 'dump.unknown-type': ('application/octet-stream', b'\x00\x01'), + } + mail = Mail(sender='me@example.com', to='you@example.com') + for name, (_, data) in files.items(): + filename = os.path.join(tmpdir, name) + with open(filename, 'wb') as fp: + fp.write(data) + mail.add_attachment(filename) + with mock.patch('smtplib.SMTP') as smtp: + Exscript.util.mail.send(mail) + text = smtp.return_value.__enter__.return_value.sendmail.call_args.args[2] + body, *parts = email.message_from_string(text).get_payload() + self.assertEqual(body.get_content_disposition(), 'inline') + result = {} + for part in parts: + self.assertEqual(part.get_content_disposition(), 'attachment') + content_type = part.get_content_type() + if part.get_content_maintype() == 'audio': + content_type = 'audio' # audio/wav or audio/x-wav, depending on the system + result[part.get_filename()] = content_type, part.get_payload(decode=True) + self.assertEqual(result, files) + def testSendDoesNotConnectIfTheMailIsIncomplete(self): mail = Mail(to='you@example.com') mail.add_attachment(os.path.join(tempfile.gettempdir(), 'missing-attachment')) From 12fdaacc0595271052f8998376c42a9b8b1cc8f6 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 01:21:31 +0100 Subject: [PATCH 183/250] Test the end of the replication and the errors of NETCONF sessions --- tests/Exscript/protocols/NetconfTest.py | 75 +++++++++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/tests/Exscript/protocols/NetconfTest.py b/tests/Exscript/protocols/NetconfTest.py index 645ed89f..6dac51da 100644 --- a/tests/Exscript/protocols/NetconfTest.py +++ b/tests/Exscript/protocols/NetconfTest.py @@ -22,6 +22,7 @@ from hypothesis import strategies as st from lxml import etree from ncclient.manager import make_device_handler +from ncclient.operations.errors import OperationError from ncclient.operations.rpc import RPC, RPCReply from ncclient.transport.errors import NetconfFramingError from ncclient.transport.session import HelloHandler, NetconfBase @@ -33,6 +34,7 @@ from Exscript.protocols.exception import ( ConnectionClosedException, LoginFailure, + ProtocolException, RPCError, TimeoutException, ) @@ -71,6 +73,14 @@ def testChunks(self): def testWithoutChunks(self): self.assertEqual(Netconf.reassemble_chunks(''), '') + def testNoChunkHeader(self): + # Text that starts like a chunk header, but is none, is returned as it is. + self.assertEqual(Netconf.reassemble_chunks('\n# comment'), '\n# comment') + + def testWithoutEndOfChunks(self): + # E.g. if the prompt consumed the end of the chunks. + self.assertEqual(Netconf.reassemble_chunks('\n#5\n\n#2\nok'), 'ok') + def testTrailingBytes(self): self.assertRaises(ValueError, Netconf.reassemble_chunks, '\n#2\nok\n##\nx') self.assertRaises(ValueError, Netconf.reassemble_chunks, '\n#2\nokx') @@ -160,6 +170,11 @@ def testWhitespaceBetweenMessages(self): data = b'\r\n \n' + encode_message(b'', True) self.assertEqual(self.feed(data, base=NetconfBase.BASE_11), ['']) + def testEmptyMessages(self): + # Messages without content are not dispatched. + data = b']]>]]>]]>]]>\r\n]]>]]>]]>]]>' + self.assertEqual(self.feed(data), ['', '']) + def testSwitchToFraming11(self): # ncclient selects NETCONF 1.1 after the messages. session = FakeSession() @@ -303,6 +318,42 @@ def testCharactersSplitAcrossWrites(self): self.assertEqual(''.join(sent), 'Zürich €') self.assertEqual(self.conn.stdout.getvalue(), 'Zürich €') + def testWriteWithinACharacter(self): + sent = [] + self.conn.data_sent_event.connect(sent.append) + channel = FakeChannel() + self.conn._wrap_channel(channel) + self.assertEqual(channel.send('€'.encode()), 2) # two of its three bytes + self.assertEqual(sent, []) + channel.send(b'\xac') + self.assertEqual(sent, ['€']) + + def testEndOfTheReceivedData(self): + # After the end of the received data, nothing more is replicated. + received, sent = [], [] + self.conn.data_received_event.connect(received.append) + self.conn.data_sent_event.connect(sent.append) + channel = FakeChannel([b'']) + self.conn._wrap_channel(channel) + while channel.recv(4096): + pass + self.conn._receive_message('') + self.assertEqual(channel.send(b''), 2) + self.assertEqual(received, ['']) + self.assertEqual(sent, []) + self.assertEqual(str(self.conn._buffer), '') + + def testWriteToAClosedChannel(self): + # Then nothing more is replicated, as after the end of the received data. + received, sent = [], [] + self.conn.data_received_event.connect(received.append) + self.conn.data_sent_event.connect(sent.append) + channel = ClosedChannel([b'']) + self.conn._wrap_channel(channel) + self.assertEqual(channel.send(b''), 0) + self.assertEqual(channel.recv(4096), b'') + self.assertEqual((received, sent), ([], [])) + class FakeChannel: """Returns the given pieces from recv(), then b''; send() writes two bytes at most.""" @@ -319,6 +370,13 @@ def send(self, s): return len(s[:2]) +class ClosedChannel(FakeChannel): + """Like FakeChannel, but send() returns 0, like paramiko's after the channel was closed.""" + + def send(self, s): + return 0 + + class NetconfServer: """ A NETCONF server on the netconf SSH subsystem, on 127.0.0.1 at a free port. @@ -681,6 +739,23 @@ def testTimeout(self): self.assertIsNone(conn.get_response_element()) self.assertRaises(TimeoutException, conn.expect, 'never') + def testReplyThatIsNoXML(self): + # lxml's error is raised, and the session goes on. + replies = iter(['', '']) + conn = self.login(lambda rpc: next(replies)) + with self.assertRaisesRegex(etree.XMLSyntaxError, 'Opening and ending tag mismatch'): + conn.execute(GET_CONFIG) + conn.execute(LOCK) + self.assertEqual(etree.QName(conn.get_response_element()[0]).localname, 'ok') + + def testOtherErrorsOfNcclient(self): + conn = self.login() + error = OperationError("Unknown 'message-id': 7") + with mock.patch.object(RPC, '_request', side_effect=error): + with self.assertRaises(ProtocolException) as cm: + conn.execute(GET_CONFIG) + self.assertEqual(str(cm.exception), "ncclient related error (Unknown 'message-id': 7)") + def testSetTimeoutAfterLogin(self): # Each RPC waits as long as the current timeout says, also close-session. timeouts = [] From 2046bb30a61f02b54394b6c3df27396f9b5e1d1b Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 01:33:11 +0100 Subject: [PATCH 184/250] Accept only complete IPv4 addresses in is_ip() is_ip() also accepted text that starts with an address, so Host failed with a ValueError for names such as 10.0.0.1.nip.io. --- CHANGES.md | 4 ++++ src/Exscript/util/ipv4.py | 2 +- tests/Exscript/HostTest.py | 7 +++++++ tests/Exscript/util/ipv4Test.py | 7 +++++++ 4 files changed, 19 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index c1b199aa..18953e14 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -630,6 +630,10 @@ The names below refer to functions callable from templates, not protocol methods and the copies keep the default factory and the order of the keys. Before, `copy.deepcopy()` failed with `TypeError: cannot pickle 'odict_items' object`, and `pickle.dumps()` with a `PicklingError`. +- **`Exscript.util.ipv4.is_ip()` and `Exscript.util.ip.is_ip()`:** Return True only + for a complete address. Before, they also returned True for text that starts with + an IPv4 address, so a `Host` with a name such as `10.0.0.1.nip.io` failed with a + `ValueError`. - **Overwritten Log Files:** `FileLogger(mode='w')`, `log_to_file(mode='w')` and `exscript --overwrite-logs` overwrite the log file of an earlier run and keep all output of the job. Before, every write overwrote the file, so it kept only the last diff --git a/src/Exscript/util/ipv4.py b/src/Exscript/util/ipv4.py index 25351976..d9bbb498 100644 --- a/src/Exscript/util/ipv4.py +++ b/src/Exscript/util/ipv4.py @@ -69,7 +69,7 @@ def is_ip(string: str) -> bool: :rtype: bool :return: True if the string is an IP address, False otherwise. """ - mo = re.match(r'(\d+)\.(\d+)\.(\d+)\.(\d+)', string) + mo = re.match(r'(\d+)\.(\d+)\.(\d+)\.(\d+)\Z', string) if mo is None: return False for group in mo.groups(): diff --git a/tests/Exscript/HostTest.py b/tests/Exscript/HostTest.py index 8e6085d3..6fc00493 100644 --- a/tests/Exscript/HostTest.py +++ b/tests/Exscript/HostTest.py @@ -58,6 +58,13 @@ def testSetAddress(self): self.assertEqual(self.host.get_name(), 'localhost') self.assertEqual(self.host.get_address(), '1.2.3.4') + def testNameThatStartsWithAnIp(self): + # E.g. the host names of wildcard DNS services such as nip.io. + for name in '10.0.0.1.nip.io', '192.168.1.10-router': + host = Host(name) + self.assertEqual(host.get_name(), name) + self.assertEqual(host.get_address(), name) + def testGetAddress(self): self.assertEqual(self.host.get_address(), 'localhost') # Additional tests are in testSetAddress(). diff --git a/tests/Exscript/util/ipv4Test.py b/tests/Exscript/util/ipv4Test.py index deffc7b3..82bf6bdc 100644 --- a/tests/Exscript/util/ipv4Test.py +++ b/tests/Exscript/util/ipv4Test.py @@ -19,6 +19,13 @@ def testIsIp(self): self.assertTrue(not is_ip('1.23.4')) self.assertTrue(not is_ip('1..3.4')) + def testIsIpWithMoreText(self): + # E.g. host names of wildcard DNS services such as nip.io. + from Exscript.util.ipv4 import is_ip + + for string in '10.0.0.1.nip.io', '1.2.3.4.5', '1.2.3.4a', '1.2.3.4/24', '1.2.3.4\n': + self.assertFalse(is_ip(string), repr(string)) + def testNormalizeIp(self): from Exscript.util.ipv4 import normalize_ip From 21b2f50c5a8a25b80329c69b8f56138df983c473 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 01:35:36 +0100 Subject: [PATCH 185/250] Test the errors of the IP address helpers --- tests/Exscript/util/ipTest.py | 2 ++ tests/Exscript/util/ipv4Test.py | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/tests/Exscript/util/ipTest.py b/tests/Exscript/util/ipTest.py index 9110ded7..8d2f798f 100644 --- a/tests/Exscript/util/ipTest.py +++ b/tests/Exscript/util/ipTest.py @@ -18,6 +18,8 @@ def testNormalizeIp(self): self.assertEqual(normalize_ip('0.128.255.0'), '000.128.255.000') self.assertEqual(normalize_ip('1234:0:01:02::'), '1234:0000:0001:0002:0000:0000:0000:0000') + for ip in 'router1', '1.2.3.4.5': + self.assertRaisesRegex(ValueError, 'neither ipv4 nor ipv6', normalize_ip, ip) def testCleanIp(self): from Exscript.util.ip import clean_ip diff --git a/tests/Exscript/util/ipv4Test.py b/tests/Exscript/util/ipv4Test.py index 82bf6bdc..6771cd96 100644 --- a/tests/Exscript/util/ipv4Test.py +++ b/tests/Exscript/util/ipv4Test.py @@ -18,6 +18,8 @@ def testIsIp(self): self.assertTrue(not is_ip('.1.2.3')) self.assertTrue(not is_ip('1.23.4')) self.assertTrue(not is_ip('1..3.4')) + self.assertFalse(is_ip('1.2.3.256')) + self.assertFalse(is_ip('300.2.3.4')) def testIsIpWithMoreText(self): # E.g. host names of wildcard DNS services such as nip.io. @@ -34,6 +36,8 @@ def testNormalizeIp(self): self.assertEqual(normalize_ip('001.002.003.004'), '001.002.003.004') self.assertEqual(normalize_ip('192.168.010.001'), '192.168.010.001') self.assertEqual(normalize_ip('0.128.255.0'), '000.128.255.000') + for ip in '1.2.3', '1.2.3.4.5': + self.assertRaisesRegex(ValueError, 'ip should be 4 tuples', normalize_ip, ip) def testCleanIp(self): from Exscript.util.ipv4 import clean_ip From 010ddf1c8e9a87569f9b85feb02985013df8c178 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 01:46:52 +0100 Subject: [PATCH 186/250] Test the hp_pro_curve driver with a ProCurve banner and prompt The tests of the drivers get a virtual device, which shows a banner and prompts and records the commands. --- tests/Exscript/protocols/drivers/__init__.py | 0 tests/Exscript/protocols/drivers/device.py | 85 ++++++++++++++ .../protocols/drivers/hp_pro_curveTest.py | 104 ++++++++++++++++++ 3 files changed, 189 insertions(+) create mode 100644 tests/Exscript/protocols/drivers/__init__.py create mode 100644 tests/Exscript/protocols/drivers/device.py create mode 100644 tests/Exscript/protocols/drivers/hp_pro_curveTest.py diff --git a/tests/Exscript/protocols/drivers/__init__.py b/tests/Exscript/protocols/drivers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/Exscript/protocols/drivers/device.py b/tests/Exscript/protocols/drivers/device.py new file mode 100644 index 00000000..68e25fcd --- /dev/null +++ b/tests/Exscript/protocols/drivers/device.py @@ -0,0 +1,85 @@ +""" +A virtual device with the prompts of a real one, for the tests of the drivers. +""" + +import os + +from Exscript import Account +from Exscript.emulators import VirtualDevice +from Exscript.protocols import Dummy + +BANNER_DIR = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'banners') + +# The authorization password differs from the login password. +ACCOUNT = Account('admin', 'login-secret', 'enable-secret', needs_lock=False) + + +def banner(name): + """Returns the banner of the corpus with the given name (tests/Exscript/protocols/banners).""" + with open(os.path.join(BANNER_DIR, name), encoding='utf-8', newline='') as fp: + return fp.read() + + +def search(patterns, text): + """Returns whether one of the given patterns matches the text.""" + return any(pattern.search(text) for pattern in patterns) + + +class Device(VirtualDevice): + """ + Shows the banner, the login prompts if given, and the prompt; answers the + commands that were added with add_command(), and others with the prompt + only; and records the commands without their line breaks. With an + enabled prompt, the enable command asks for a password, and the device + shows the enabled prompt afterwards. + + :param login: The user name prompt and the password prompt of the login. + """ + + def __init__( + self, + prompt, + banner='\r\n', + login=None, + enabled_prompt=None, + enable_command='enable', + enable_password_prompt='Password: ', + ): + login_type = VirtualDevice.LOGIN_TYPE_BOTH if login else VirtualDevice.LOGIN_TYPE_NONE + super().__init__('r1', login_type=login_type, strict=False, banner=banner) + if login: + self.user_prompt, self.password_prompt = login + self.prompt = prompt + self.enabled_prompt = enabled_prompt + self.enable_command = enable_command + self.enable_password_prompt = enable_password_prompt + self.received = [] + self._enabling = False + + def do(self, command): + command = command.strip('\r\n') + self.received.append(command) + if self._enabling: # the password, which is not echoed + self._enabling = False + self.prompt = self.enabled_prompt + return '\r\n' + self.prompt + if command == self.enable_command and self.enabled_prompt is not None: + self._enabling = True + return command + '\r\n' + self.enable_password_prompt + return super().do(command) + + +def connect(test, device, driver, login=None, prompt=True): + """ + Returns a Dummy connection with the given driver to the device, after the + login with the given account, or else after the banner and the first + prompt unless prompt is False. + """ + conn = Dummy(device=device, driver=driver, timeout=1) + conn.connect(device.hostname) + test.addCleanup(conn.close, force=True) + if login: + conn.login(login) + elif prompt: + conn.expect_prompt() + return conn diff --git a/tests/Exscript/protocols/drivers/hp_pro_curveTest.py b/tests/Exscript/protocols/drivers/hp_pro_curveTest.py new file mode 100644 index 00000000..4687400e --- /dev/null +++ b/tests/Exscript/protocols/drivers/hp_pro_curveTest.py @@ -0,0 +1,104 @@ +""" +The hp_pro_curve driver, with the banner and the escape sequences of a +ProCurve 5406zl switch. +""" + +import unittest + +from Exscript.protocols.drivers import driver_map +from Exscript.protocols.exception import InvalidCommandException + +from .device import ACCOUNT, Device, banner, connect, search + +# The switch ends the line with ESC E (next line), moves the cursor to the last +# line of the screen and clears it before the prompt, and moves the cursor +# behind the prompt afterwards. +PROMPT = '\x1bE\x1b[24;1H\x1b[2K\x1b[24;1H\x1b[?25h\x1b[24;1HHP-5406zl# \x1b[24;1H\x1b[24;13H' +VERSION = '\r\nImage stamp: /sw/code/build/btm(K_15)\r\n Jan 26 2011 12:54:37\r\n' + + +class ProCurve(Device): + """Shows its banner, which ends with "Press any key to continue", and waits for a key.""" + + def init(self): + super().init() + self.logged_in = False # the next input is the key + return banner('hp_pro_curve.1') + + +class HPProCurveDriverTest(unittest.TestCase): + def setUp(self): + self.driver = driver_map['hp_pro_curve'] + + def testCheckHeadForOs(self): + self.assertEqual(self.driver.check_head_for_os(banner('hp_pro_curve.1')), 95) + # The banner of a later model that does not mention ProCurve. + head = ( + 'HP J9728A 2920-48G Switch\r\nSoftware revision WB.15.12.0012\r\n\r\n' + 'Copyright (C) 1991-2013 Hewlett-Packard Development Company, L.P.\r\n' + ) + self.assertEqual(self.driver.check_head_for_os(head), 50) + self.assertEqual(self.driver.check_head_for_os('Welcome to router1\r\n'), 0) + + def testLoginPatterns(self): + self.assertTrue(search(self.driver.user_re, 'Press any key to continue\r\nUsername: ')) + self.assertTrue(search(self.driver.password_re, 'Username: admin\r\nPassword: ')) + self.assertTrue(search(self.driver.login_error_re, 'Password: \r\nInvalid password\r\n')) + self.assertFalse(search(self.driver.login_error_re, banner('hp_pro_curve.1'))) + + def testCleanResponseForReMatch(self): + clean = self.driver.clean_response_for_re_match + expected = ('Uptime: 2 days\r\nHP-5406zl# ', '') + self.assertEqual(clean('Uptime: 2 days' + PROMPT), expected) + # CSI is also the character U+009B. + self.assertEqual(clean('Uptime: 2 days\x1bEHP-5406zl# \x9b24;13H'), expected) + # The rest from an incomplete escape sequence on is returned separately. + self.assertEqual( + clean('Uptime: 2 days\x1bEHP-5406zl# \x1b[24;1'), (expected[0], '\x1b[24;1') + ) + # The data before an escape character in the first ten characters is + # dropped, e.g. the echo of a short command. + self.assertEqual(clean('no page' + PROMPT), ('\r\nHP-5406zl# ', '')) + + def testPrompts(self): + for prompt in 'HP-5406zl# ', 'HP-5406zl> ', 'sw-core.1#': + self.assertTrue(search(self.driver.prompt_re, 'Uptime\r\n' + prompt), prompt) + for line in 'Uptime 2 days', ' Status and Counters - General System Information': + self.assertFalse(search(self.driver.prompt_re, 'show system\r\n' + line), line) + + def testInitTerminalPressesAKey(self): + device = ProCurve(PROMPT) + device.add_command('show version', VERSION) + conn = connect_after_banner(self, device) + conn.autoinit() + self.assertEqual(device.received, ['']) + conn.execute('show version') + self.assertIn('Image stamp:', conn.response) + + def testLogin(self): + device = Device(PROMPT, login=('Username: ', 'Password: ')) + conn = connect(self, device, 'hp_pro_curve', login=ACCOUNT) + self.assertEqual(device.received, ['admin', 'login-secret']) + self.assertTrue(conn.is_app_authenticated()) + + def testError(self): + device = Device(PROMPT) + device.add_command('shw', '\r\nInvalid input: shw') + conn = connect(self, device, 'hp_pro_curve') + with self.assertRaisesRegex(InvalidCommandException, 'Invalid input: shw'): + conn.execute('shw version') + + +def connect_after_banner(test, device): + """Returns a connection to the device after its banner.""" + conn = connect(test, device, 'hp_pro_curve', prompt=False) + conn.expect('Press any key to continue') + return conn + + +def suite(): + return unittest.TestLoader().loadTestsFromTestCase(HPProCurveDriverTest) + + +if __name__ == '__main__': + unittest.TextTestRunner(verbosity=2).run(suite()) From a86e76540811f9c18db40e44013bfc3ce2b96cbe Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 01:48:38 +0100 Subject: [PATCH 187/250] Recognize the configuration prompts of HP ProCurve switches The prompt pattern did not allow the context in parentheses, e.g. HP-2920(config)#, so a command that entered one timed out. --- CHANGES.md | 4 ++++ .../protocols/drivers/hp_pro_curve.py | 7 ++++--- .../protocols/drivers/hp_pro_curveTest.py | 19 +++++++++++++++++++ 3 files changed, 27 insertions(+), 3 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 18953e14..0f83e9e6 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -406,6 +406,10 @@ development versions; "before" refers to them. banner is now part of the test corpus. - **NX-OS Banners:** The `nxos` driver also recognizes the banner "Cisco NX-OS Software" of newer versions, e.g. nx-osv9000 (upstream issue #200). +- **HP ProCurve Configuration Prompts:** The `hp_pro_curve` driver recognizes the + prompts of the configuration contexts, e.g. `HP-2920(config)# ` or + `HP-2920(vlan-10)# `. Before, a command that entered such a context, e.g. + `configure`, waited until the timeout. - **Login Errors of `ericsson_ban`:** A login over the `ericsson_ban` driver no longer fails with `AttributeError: 'str' object has no attribute 'search'`; its login error pattern was a string instead of a compiled pattern. A test checks the diff --git a/src/Exscript/protocols/drivers/hp_pro_curve.py b/src/Exscript/protocols/drivers/hp_pro_curve.py index 69358ed3..9dc72bda 100644 --- a/src/Exscript/protocols/drivers/hp_pro_curve.py +++ b/src/Exscript/protocols/drivers/hp_pro_curve.py @@ -34,7 +34,7 @@ _user_re = [re.compile(r'(?:\r\n|[\r\n])Username: ?$')] _password_re = [re.compile(r'(?:\r\n|[\r\n])Password: ?$')] -_prompt_re = [re.compile(r'(?:\r\n|[\r\n])[\-\w+\.:/]+[>#] ?$')] +_prompt_re = [re.compile(r'(?:\r\n|[\r\n])[\-\w+\.:/]+(?:\([^\)]+\))?[>#] ?$')] _error_re = [re.compile(r'(?:invalid|incomplete|ambiguous) input:', re.I)] _login_fail_re = [ re.compile(r'(?:\r\n|[\r\n])invalid password', re.I), @@ -50,8 +50,9 @@ class HPProCurveDriver(Driver): """ The ``hp_pro_curve`` driver for HP ProCurve switches. Its prompt is a - name followed by ">" or "#"; the escape sequences around it are removed - before the prompt is matched. + name with an optional context in parentheses, followed by ">" or "#", + e.g. "HP-2920#" or "HP-2920(config)#"; the escape sequences around it are + removed before the prompt is matched. """ def __init__(self) -> None: diff --git a/tests/Exscript/protocols/drivers/hp_pro_curveTest.py b/tests/Exscript/protocols/drivers/hp_pro_curveTest.py index 4687400e..5c54bb9a 100644 --- a/tests/Exscript/protocols/drivers/hp_pro_curveTest.py +++ b/tests/Exscript/protocols/drivers/hp_pro_curveTest.py @@ -66,6 +66,25 @@ def testPrompts(self): for line in 'Uptime 2 days', ' Status and Counters - General System Information': self.assertFalse(search(self.driver.prompt_re, 'show system\r\n' + line), line) + def testConfigurationContexts(self): + device = Device(PROMPT) + + def enter(context): + def handler(command): + device.prompt = PROMPT.replace('# ', '(%s)# ' % context) + return '' + + return handler + + commands = {'configure': 'config', 'vlan 10': 'vlan-10', 'interface A1': 'eth-A1'} + for command, context in commands.items(): + device.add_command(command, enter(context)) + conn = connect(self, device, 'hp_pro_curve') + for command in commands: + index, match = conn.execute(command) + self.assertEqual(device.received, list(commands)) + self.assertEqual(match.group().lstrip(), 'HP-5406zl(eth-A1)# ') + def testInitTerminalPressesAKey(self): device = ProCurve(PROMPT) device.add_command('show version', VERSION) From 19c867f17dd723f0b4873604439046dc773bfdd7 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 01:51:33 +0100 Subject: [PATCH 188/250] Test the Cisco drivers with the prompts and messages of the devices --- tests/Exscript/protocols/drivers/aceTest.py | 59 ++++++++++++++ .../Exscript/protocols/drivers/aironetTest.py | 46 +++++++++++ tests/Exscript/protocols/drivers/iosTest.py | 79 +++++++++++++++++++ .../Exscript/protocols/drivers/ios_xrTest.py | 63 +++++++++++++++ tests/Exscript/protocols/drivers/nxosTest.py | 57 +++++++++++++ 5 files changed, 304 insertions(+) create mode 100644 tests/Exscript/protocols/drivers/aceTest.py create mode 100644 tests/Exscript/protocols/drivers/aironetTest.py create mode 100644 tests/Exscript/protocols/drivers/iosTest.py create mode 100644 tests/Exscript/protocols/drivers/ios_xrTest.py create mode 100644 tests/Exscript/protocols/drivers/nxosTest.py diff --git a/tests/Exscript/protocols/drivers/aceTest.py b/tests/Exscript/protocols/drivers/aceTest.py new file mode 100644 index 00000000..576fdaad --- /dev/null +++ b/tests/Exscript/protocols/drivers/aceTest.py @@ -0,0 +1,59 @@ +""" +The ace driver, with the banner and the prompts of the Cisco Application +Control Engine (ACE). +""" + +import unittest + +from Exscript.protocols.drivers import driver_map +from Exscript.protocols.exception import InvalidCommandException + +from .device import ACCOUNT, Device, banner, connect, search + + +class ACEDriverTest(unittest.TestCase): + def setUp(self): + self.driver = driver_map['ace'] + + def testCheckHeadForOs(self): + self.assertEqual(self.driver.check_head_for_os(banner('ace.1')), 90) + self.assertEqual(self.driver.check_head_for_os('\r\nUser Access Verification\r\n'), 0) + + def testPrompts(self): + # The name of the device and the context. + for prompt in ( + 'switch/Admin#', + 'ace-1/ctx1>', + 'ace-1/ctx1(config)#', + 'ace-1/ctx1(config-if)# ', + ): + self.assertTrue(search(self.driver.prompt_re, 'Admin\r\n' + prompt), prompt) + self.assertFalse(search(self.driver.prompt_re, '\r\nACE module/context')) + + def testLoginAndEnable(self): + device = Device( + 'ace-1/Admin>', login=('Username: ', 'Password: '), enabled_prompt='ace-1/Admin#' + ) + conn = connect(self, device, 'ace', login=ACCOUNT) + self.assertEqual(device.received, ['admin', 'login-secret', 'enable', 'enable-secret']) + self.assertTrue(conn.is_app_authorized()) + + def testInitTerminal(self): + device = Device('ace-1/Admin#') + conn = connect(self, device, 'ace') + conn.autoinit() + self.assertEqual(device.received, ['term len 0']) + + def testError(self): + device = Device('ace-1/Admin#') + device.add_command('show rserver', '\r\nError: rserver rs1 not found') + conn = connect(self, device, 'ace') + self.assertRaises(InvalidCommandException, conn.execute, 'show rserver rs1') + + +def suite(): + return unittest.TestLoader().loadTestsFromTestCase(ACEDriverTest) + + +if __name__ == '__main__': + unittest.TextTestRunner(verbosity=2).run(suite()) diff --git a/tests/Exscript/protocols/drivers/aironetTest.py b/tests/Exscript/protocols/drivers/aironetTest.py new file mode 100644 index 00000000..f1201383 --- /dev/null +++ b/tests/Exscript/protocols/drivers/aironetTest.py @@ -0,0 +1,46 @@ +""" +The aironet driver, with the banner and the prompt of a Cisco Wireless LAN +Controller (AireOS). +""" + +import unittest + +from Exscript.protocols.drivers import driver_map + +from .device import ACCOUNT, Device, banner, connect, search + + +class AironetDriverTest(unittest.TestCase): + def setUp(self): + self.driver = driver_map['aironet'] + + def testCheckHeadForOs(self): + self.assertEqual(self.driver.check_head_for_os(banner('aironet.1')), 90) + self.assertEqual(self.driver.check_head_for_os('(WiSM-slot2-1)\r\nUser: '), 90) + self.assertEqual(self.driver.check_head_for_os('(WLC-5508) >'), 87) + self.assertEqual(self.driver.check_head_for_os('\r\nUser Access Verification\r\n'), 0) + + def testPrompts(self): + for prompt in '(Cisco Controller) >', '(WLC-5508) >': + self.assertTrue(search(self.driver.prompt_re, 'Last login\r\n' + prompt), prompt) + self.assertFalse(search(self.driver.prompt_re, '\r\nWLC-5508>')) + + def testLogin(self): + device = Device('(Cisco Controller) >', login=('User: ', 'Password:')) + conn = connect(self, device, 'aironet', login=ACCOUNT) + self.assertEqual(device.received, ['admin', 'login-secret']) + self.assertTrue(conn.is_app_authorized()) + + def testInitTerminal(self): + device = Device('(Cisco Controller) >') + conn = connect(self, device, 'aironet') + conn.autoinit() + self.assertEqual(device.received, ['config paging disable']) + + +def suite(): + return unittest.TestLoader().loadTestsFromTestCase(AironetDriverTest) + + +if __name__ == '__main__': + unittest.TextTestRunner(verbosity=2).run(suite()) diff --git a/tests/Exscript/protocols/drivers/iosTest.py b/tests/Exscript/protocols/drivers/iosTest.py new file mode 100644 index 00000000..f749bcaf --- /dev/null +++ b/tests/Exscript/protocols/drivers/iosTest.py @@ -0,0 +1,79 @@ +""" +The ios driver, with the prompts and messages of Cisco IOS and of the Cisco ASA. +""" + +import unittest + +from Exscript.protocols.drivers import driver_map +from Exscript.protocols.exception import InvalidCommandException + +from .device import ACCOUNT, Device, banner, connect, search + +INVALID = "\r\n ^\r\n% Invalid input detected at '^' marker.\r\n" + + +class IOSDriverTest(unittest.TestCase): + def setUp(self): + self.driver = driver_map['ios'] + + def testCheckProtocolForOs(self): + self.assertEqual(self.driver.check_protocol_for_os('SSH-2.0-Cisco-1.25'), 80) + self.assertEqual(self.driver.check_protocol_for_os('SSH-2.0-OpenSSH_9.6'), 0) + self.assertEqual(self.driver.check_protocol_for_os(''), 0) # e.g. Telnet + + def testCheckHeadForOs(self): + self.assertEqual(self.driver.check_head_for_os(banner('ios.1')), 60) + # An S/Key challenge of a TACACS+ server, or only the user name prompt. + challenge = '\r\nUsername: admin\r\ns/key 98 lo4521\r\nPassword: ' + self.assertEqual(self.driver.check_head_for_os(challenge), 50) + self.assertEqual(self.driver.check_head_for_os('\r\nUsername: '), 30) + self.assertEqual(self.driver.check_head_for_os('\r\nlogin: '), 0) + + def testPrompts(self): + for prompt in 'Router>', 'Router#', 'Router(config)#', 'core-sw.1(config-if)#': + self.assertTrue(search(self.driver.prompt_re, 'Building configuration...\r\n' + prompt)) + for line in '!', ' description uplink #2', 'interface GigabitEthernet0/1': + self.assertFalse(search(self.driver.prompt_re, 'show run\r\n' + line), line) + + def testLoginAndEnable(self): + device = Device( + 'Router>', + banner=banner('ios.1').split('*')[0], + login=('Username: ', 'Password: '), + enabled_prompt='Router#', + ) + conn = connect(self, device, 'ios', login=ACCOUNT) + self.assertEqual(device.received, ['admin', 'login-secret', 'enable', 'enable-secret']) + self.assertTrue(conn.is_app_authorized()) + + def testInitTerminal(self): + device = Device('Router#') + conn = connect(self, device, 'ios') + conn.autoinit() + self.assertEqual(device.received, ['term len 0', 'term width 0']) + + def testInitTerminalOfAnASA(self): + # The ASA rejects "term len 0" and pages with "term pager 0". + device = Device('ciscoasa#') + device.add_command('term len', '\r\nERROR: % Invalid input detected at \'^\' marker.') + conn = connect(self, device, 'ios') + conn.autoinit() + self.assertEqual(device.received, ['term len 0', 'term pager 0']) + + def testErrors(self): + device = Device('Router#') + device.add_command('show ip rout', INVALID) + device.add_command('show ip', "\r\n% Incomplete command.\r\n") + device.add_command('sh', '\r\n% Ambiguous command: "sh"\r\n') + conn = connect(self, device, 'ios') + for command in 'show ip routing', 'show ip', 'sh': + with self.subTest(command=command): + self.assertRaises(InvalidCommandException, conn.execute, command) + + +def suite(): + return unittest.TestLoader().loadTestsFromTestCase(IOSDriverTest) + + +if __name__ == '__main__': + unittest.TextTestRunner(verbosity=2).run(suite()) diff --git a/tests/Exscript/protocols/drivers/ios_xrTest.py b/tests/Exscript/protocols/drivers/ios_xrTest.py new file mode 100644 index 00000000..6131f0ce --- /dev/null +++ b/tests/Exscript/protocols/drivers/ios_xrTest.py @@ -0,0 +1,63 @@ +""" +The ios_xr driver, with the prompts of Cisco IOS XR. +""" + +import unittest + +from Exscript.protocols.drivers import driver_map + +from .device import ACCOUNT, Device, banner, connect, search + +PROMPT = 'RP/0/RSP0/CPU0:xr-pe1#' + + +class IOSXRDriverTest(unittest.TestCase): + def setUp(self): + self.driver = driver_map['ios_xr'] + + def testCheckResponseForOs(self): + self.assertEqual(self.driver.check_response_for_os(banner('ios_xr')), 95) + self.assertEqual(self.driver.check_response_for_os('\r\nRouter#'), 0) + + def testCheckNetconfCapabilities(self): + capabilities = ( + 'urn:ietf:params:netconf:base:1.1\n' + 'http://cisco.com/ns/yang/Cisco-IOS-XR-ifmgr-cfg?module=Cisco-IOS-XR-ifmgr-cfg' + '&revision=2017-09-07\n' + ) + self.assertEqual(self.driver.check_netconf_capabilities(capabilities), 100) + self.assertEqual(self.driver.check_netconf_capabilities(capabilities.split('\n')[0]), 0) + + def testPrompts(self): + prompts = ( + PROMPT, + 'RP/0/RP0/CPU0:xr-pe1(config)#', + 'RP/0/RP0/CPU0:xr-pe1(config-if)#', + 'RP/0/0/CPU0:ios#', # IOS XRv + ) + for prompt in prompts: + self.assertTrue( + search(self.driver.prompt_re, 'Thu Nov 10 12:00:00.000 UTC\r\n' + prompt) + ) + self.assertFalse(search(self.driver.prompt_re, '\r\nRouter#')) + + def testLogin(self): + device = Device(PROMPT, login=('Username: ', 'Password: ')) + conn = connect(self, device, 'ios_xr', login=ACCOUNT) + self.assertEqual(device.received, ['admin', 'login-secret']) + self.assertTrue(conn.is_app_authorized()) + + def testInitTerminal(self): + device = Device(PROMPT) + conn = connect(self, device, 'ios_xr') + conn.autoinit() + expected = ['terminal exec prompt no-timestamp', 'terminal len 0', 'terminal width 0'] + self.assertEqual(device.received, expected) + + +def suite(): + return unittest.TestLoader().loadTestsFromTestCase(IOSXRDriverTest) + + +if __name__ == '__main__': + unittest.TextTestRunner(verbosity=2).run(suite()) diff --git a/tests/Exscript/protocols/drivers/nxosTest.py b/tests/Exscript/protocols/drivers/nxosTest.py new file mode 100644 index 00000000..c353c51d --- /dev/null +++ b/tests/Exscript/protocols/drivers/nxosTest.py @@ -0,0 +1,57 @@ +""" +The nxos driver, with the banner and the prompts of Cisco NX-OS. +""" + +import unittest + +from Exscript.protocols.drivers import driver_map +from Exscript.protocols.exception import InvalidCommandException + +from .device import ACCOUNT, Device, banner, connect, search + + +class NXOSDriverTest(unittest.TestCase): + def setUp(self): + self.driver = driver_map['nxos'] + + def testCheckHeadForOs(self): + self.assertEqual(self.driver.check_head_for_os(banner('nxos.1')), 95) + head = 'Cisco NX-OS Software\r\nCopyright (c) 2002-2022, Cisco Systems, Inc.\r\n' + self.assertEqual(self.driver.check_head_for_os(head), 95) + self.assertEqual(self.driver.check_head_for_os('\r\nUser Access Verification\r\n'), 0) + + def testPrompts(self): + for prompt in 'switch#', 'N7K-1(config)#', 'N7K-1(config-if)# ': + self.assertTrue(search(self.driver.prompt_re, 'Last login\r\n' + prompt), prompt) + self.assertFalse(search(self.driver.prompt_re, '\r\n Description: uplink #1')) + + def testLoginWithoutEnable(self): + # NX-OS has no enable mode: the role of the user grants the privileges. + device = Device('N7K-1#', login=('Username: ', 'Password: '), enabled_prompt='N7K-1#') + conn = connect(self, device, 'nxos', login=ACCOUNT) + self.assertEqual(device.received, ['admin', 'login-secret']) + conn.auto_app_authorize(ACCOUNT) + self.assertEqual(device.received, ['admin', 'login-secret']) + + def testInitTerminal(self): + device = Device('N7K-1#') + conn = connect(self, device, 'nxos') + conn.autoinit() + self.assertEqual(device.received, ['term len 0']) + + def testErrors(self): + device = Device('N7K-1#') + device.add_command('show vlan', '\r\n% Invalid input detected at \'^\' marker.') + device.add_command('show ip', "\r\n% Incomplete command at '^' marker.") + conn = connect(self, device, 'nxos') + for command in 'show vlan foo', 'show ip': + with self.subTest(command=command): + self.assertRaises(InvalidCommandException, conn.execute, command) + + +def suite(): + return unittest.TestLoader().loadTestsFromTestCase(NXOSDriverTest) + + +if __name__ == '__main__': + unittest.TextTestRunner(verbosity=2).run(suite()) From 68873d94258058d3c418f3d95c12f3d5b357123c Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 01:52:41 +0100 Subject: [PATCH 189/250] Recognize the "Invalid command" error of NX-OS The nxos driver had the error patterns of IOS, which reports "Invalid input", so execute() returned the response of an unknown command as if it had succeeded. --- CHANGES.md | 4 ++++ src/Exscript/protocols/drivers/nxos.py | 4 ++-- tests/Exscript/protocols/drivers/nxosTest.py | 3 ++- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 0f83e9e6..40c4cb75 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -406,6 +406,10 @@ development versions; "before" refers to them. banner is now part of the test corpus. - **NX-OS Banners:** The `nxos` driver also recognizes the banner "Cisco NX-OS Software" of newer versions, e.g. nx-osv9000 (upstream issue #200). +- **NX-OS Invalid Commands:** The `nxos` driver recognizes the error message + `% Invalid command at '^' marker.` of NX-OS, so `execute()` raises an + `InvalidCommandException` for such a command. Before, it returned the response as + if the command had succeeded. - **HP ProCurve Configuration Prompts:** The `hp_pro_curve` driver recognizes the prompts of the configuration contexts, e.g. `HP-2920(config)# ` or `HP-2920(vlan-10)# `. Before, a command that entered such a context, e.g. diff --git a/src/Exscript/protocols/drivers/nxos.py b/src/Exscript/protocols/drivers/nxos.py index 30938e0d..dae827dd 100644 --- a/src/Exscript/protocols/drivers/nxos.py +++ b/src/Exscript/protocols/drivers/nxos.py @@ -39,7 +39,7 @@ _prompt_re = [re.compile(r'(?:\r\n\x00?|[\r\n\x00])[\-\w+\.:/]+(?:\([^\)]+\))?[>#] ?$')] _error_re = [ re.compile(r'%Error'), - re.compile(r'invalid input', re.I), + re.compile(r'invalid (?:input|command)', re.I), re.compile(r'(?:incomplete|ambiguous) command', re.I), re.compile(r'connection timed out', re.I), re.compile(r'[^\r\n]+ not found', re.I), @@ -57,7 +57,7 @@ def __init__(self) -> None: """ Constructor. Sets the patterns of the user name prompt "Username:", the password prompts, the command prompt and the error messages, such - as "%Error" and "Invalid input". + as "%Error", "Invalid input" and "Invalid command". """ Driver.__init__(self, 'nxos') self.user_re = _user_re diff --git a/tests/Exscript/protocols/drivers/nxosTest.py b/tests/Exscript/protocols/drivers/nxosTest.py index c353c51d..79f8c15c 100644 --- a/tests/Exscript/protocols/drivers/nxosTest.py +++ b/tests/Exscript/protocols/drivers/nxosTest.py @@ -43,8 +43,9 @@ def testErrors(self): device = Device('N7K-1#') device.add_command('show vlan', '\r\n% Invalid input detected at \'^\' marker.') device.add_command('show ip', "\r\n% Incomplete command at '^' marker.") + device.add_command('shutdown', "\r\n ^\r\n% Invalid command at '^' marker.") conn = connect(self, device, 'nxos') - for command in 'show vlan foo', 'show ip': + for command in 'show vlan foo', 'show ip', 'shutdown force': with self.subTest(command=command): self.assertRaises(InvalidCommandException, conn.execute, command) From 88993280739d045867c768793aa4aef142860936 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 01:52:57 +0100 Subject: [PATCH 190/250] Test SSH2 logins without a method left and closes with bad output --- tests/Exscript/protocols/SSH2Test.py | 40 ++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/tests/Exscript/protocols/SSH2Test.py b/tests/Exscript/protocols/SSH2Test.py index 0d4cf9b6..8a511b24 100644 --- a/tests/Exscript/protocols/SSH2Test.py +++ b/tests/Exscript/protocols/SSH2Test.py @@ -409,6 +409,33 @@ def testKeyOfTheAccount(self): with self.assertRaisesRegex(LoginFailure, 'requires one of: password'): conn.protocol_authenticate(account) + def testNoMethodLeftAfterAPartialSuccess(self): + # A method that succeeded partially is not tried again. + interface = type('PasswordTwice', (_PasswordTwice,), {'passwords': []}) + server = KeyServer(self, paramiko.RSAKey(filename=keyfile), interface=interface) + conn = SSH2(timeout=1, connect_rate_limit=0, verify_fingerprint=False) + self.addCleanup(conn.close, force=True) + conn.connect(HOST, server.port) + with self.assertRaisesRegex( + LoginFailure, + r"No method to try after a partial authentication; allowed: \['password'\]", + ): + conn._paramiko_auth('admin', 'S3cret') + self.assertEqual(interface.passwords, ['S3cret']) + + +class _PasswordTwice(paramiko.ServerInterface): + """Accepts each password partially, and then allows only the password method.""" + + passwords: list[str] + + def get_allowed_auths(self, username): + return 'password' + + def check_auth_password(self, username, password): + self.passwords.append(password) + return paramiko.AUTH_PARTIALLY_SUCCESSFUL + class ConnectTest(unittest.TestCase): def testTriesAllAddresses(self): @@ -831,6 +858,19 @@ def testGracefulCloseReceivesThePendingOutput(self): conn.close() self.assertIn('file1 file2', stdout.getvalue()) + def testGracefulCloseWithPendingOutputThatIsNotValid(self): + # The output cannot be decoded, but the connection is closed nevertheless. + self.device.add_command('cat', 'Grüße') + conn = self.login(encoding='ascii', debug=1) + channel, transport = conn._channel, conn._transport + conn.send('cat\r') + select.select([channel], [], [], 5) # the output arrived, but was not read + with self.assertLogs('Exscript.protocols.ssh2', logging.DEBUG) as logs: + conn.close() + self.assertNotIn('Error while closing the connection', '\n'.join(logs.output)) + self.assertTrue(channel.closed) + self.assertFalse(transport.is_active()) + def suite(): loader = unittest.TestLoader() From c5d17da4f5f3954bb4b9946b7799445d3ccc6959 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 01:57:00 +0100 Subject: [PATCH 191/250] Raise the error of opening the file in get_accounts_from_file() configparser's read() skips a file that it cannot open, so a missing file raised NoSectionError. The function now opens the file itself and raises the OSError, like the other readers of util.file. --- CHANGES.md | 5 +++++ src/Exscript/util/file.py | 6 ++++-- tests/Exscript/util/fileTest.py | 11 +++++++++++ 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 40c4cb75..47429ddc 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -638,6 +638,11 @@ The names below refer to functions callable from templates, not protocol methods and the copies keep the default factory and the order of the keys. Before, `copy.deepcopy()` failed with `TypeError: cannot pickle 'odict_items' object`, and `pickle.dumps()` with a `PicklingError`. +- **`Exscript.util.file.get_accounts_from_file()` With a Missing File:** Raises the + `OSError` of opening the file, e.g. `FileNotFoundError` for a missing file, as the + other readers of `Exscript.util.file` raise an `OSError`. Before, it raised + `configparser.NoSectionError`, because `configparser` skips the files that it cannot + read, so a missing file looked like a file without an `[account-pool]` section. - **`Exscript.util.ipv4.is_ip()` and `Exscript.util.ip.is_ip()`:** Return True only for a complete address. Before, they also returned True for text that starts with an IPv4 address, so a `Host` with a name such as `10.0.0.1.nip.io` failed with a diff --git a/src/Exscript/util/file.py b/src/Exscript/util/file.py index 50bb2253..d87983bd 100644 --- a/src/Exscript/util/file.py +++ b/src/Exscript/util/file.py @@ -58,7 +58,8 @@ def get_accounts_from_file( python3 -c 'import base64; print(base64.b64encode("thepassword".encode()).decode())' - Raises an Exception if a password can not be decoded. + Raises OSError if the file cannot be opened, e.g. FileNotFoundError if + it does not exist, and an Exception if a password can not be decoded. :type filename: str|os.PathLike :param filename: The name of the file containing the list of accounts. @@ -74,7 +75,8 @@ def get_accounts_from_file( # str keeps the case of the user names. The stub declares optionxform as # a method, which may not be replaced; the cast is for the type checker. cast(Any, parser).optionxform = str - parser.read(filename, encoding) + with open(filename, encoding=encoding) as fp: # read() skips files it cannot open + parser.read_file(fp) password: str | bytes # read as a string, then decoded to bytes for user, password in parser.items('account-pool'): try: diff --git a/tests/Exscript/util/fileTest.py b/tests/Exscript/util/fileTest.py index 33191059..e0d3e16c 100644 --- a/tests/Exscript/util/fileTest.py +++ b/tests/Exscript/util/fileTest.py @@ -81,6 +81,17 @@ def testMissingFiles(self): with self.assertRaisesRegex(OSError, 'No such file: ' + re.escape(filename)): func(filename) + def testGetAccountsFromAFileThatCannotBeRead(self): + from Exscript.util.file import get_accounts_from_file + + tmpdir = self.enterContext(tempfile.TemporaryDirectory()) + filename = os.path.join(tmpdir, 'missing.cfg') + with self.assertRaisesRegex(FileNotFoundError, 'No such file') as cm: + get_accounts_from_file(filename) + self.assertEqual(cm.exception.filename, filename) + # Opening a directory fails, too: IsADirectoryError, PermissionError on Windows. + self.assertRaises(OSError, get_accounts_from_file, tmpdir) + def testGetAccountsFromFileWithOtherSections(self): from Exscript.util.file import get_accounts_from_file From a82998c28c36ab5900e5811aa0f8516d69926f29 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 01:58:11 +0100 Subject: [PATCH 192/250] Test the Juniper, Huawei and ZTE drivers with the output of the devices --- tests/Exscript/protocols/drivers/device.py | 8 ++- tests/Exscript/protocols/drivers/junosTest.py | 64 +++++++++++++++++++ .../protocols/drivers/junos_erxTest.py | 52 +++++++++++++++ tests/Exscript/protocols/drivers/vrpTest.py | 64 +++++++++++++++++++ .../Exscript/protocols/drivers/vxworksTest.py | 64 +++++++++++++++++++ tests/Exscript/protocols/drivers/zteTest.py | 53 +++++++++++++++ 6 files changed, 303 insertions(+), 2 deletions(-) create mode 100644 tests/Exscript/protocols/drivers/junosTest.py create mode 100644 tests/Exscript/protocols/drivers/junos_erxTest.py create mode 100644 tests/Exscript/protocols/drivers/vrpTest.py create mode 100644 tests/Exscript/protocols/drivers/vxworksTest.py create mode 100644 tests/Exscript/protocols/drivers/zteTest.py diff --git a/tests/Exscript/protocols/drivers/device.py b/tests/Exscript/protocols/drivers/device.py index 68e25fcd..e9b2807e 100644 --- a/tests/Exscript/protocols/drivers/device.py +++ b/tests/Exscript/protocols/drivers/device.py @@ -30,8 +30,9 @@ class Device(VirtualDevice): Shows the banner, the login prompts if given, and the prompt; answers the commands that were added with add_command(), and others with the prompt only; and records the commands without their line breaks. With an - enabled prompt, the enable command asks for a password, and the device - shows the enabled prompt afterwards. + enabled prompt, the enable command asks for a password, unless the + password prompt is None, and the device shows the enabled prompt + afterwards. :param login: The user name prompt and the password prompt of the login. """ @@ -64,6 +65,9 @@ def do(self, command): self.prompt = self.enabled_prompt return '\r\n' + self.prompt if command == self.enable_command and self.enabled_prompt is not None: + if self.enable_password_prompt is None: + self.prompt = self.enabled_prompt + return command + '\r\n' + self.prompt self._enabling = True return command + '\r\n' + self.enable_password_prompt return super().do(command) diff --git a/tests/Exscript/protocols/drivers/junosTest.py b/tests/Exscript/protocols/drivers/junosTest.py new file mode 100644 index 00000000..e8b825e8 --- /dev/null +++ b/tests/Exscript/protocols/drivers/junosTest.py @@ -0,0 +1,64 @@ +""" +The junos driver, with the banner, the prompts and the messages of Junos. +""" + +import unittest + +from Exscript.protocols.drivers import driver_map +from Exscript.protocols.exception import InvalidCommandException + +from .device import ACCOUNT, Device, banner, connect + +PROMPT = 'admin@mx1> ' + + +class JunOSDriverTest(unittest.TestCase): + def setUp(self): + self.driver = driver_map['junos'] + + def testCheckHeadForOs(self): + self.assertEqual(self.driver.check_head_for_os(banner('junos.1')), 80) + # The BSD terminal name before "login:", e.g. of a Junos without banner. + self.assertEqual(self.driver.check_head_for_os('\r\nmx1 (ttyp0)\r\n\r\nlogin: '), 35) + self.assertEqual(self.driver.check_head_for_os('\r\nrouter1 login: '), 0) # Linux + + def testCheckNetconfCapabilities(self): + capability = 'http://xml.juniper.net/netconf/junos/1.0' + self.assertEqual(self.driver.check_netconf_capabilities(capability), 100) + self.assertEqual( + self.driver.check_netconf_capabilities('urn:ietf:params:netconf:base:1.0'), 0 + ) + + def testLogin(self): + device = Device(PROMPT, banner='\r\nmx1 (ttyp0)\r\n\r\n', login=('login: ', 'Password:')) + conn = connect(self, device, 'junos', login=ACCOUNT) + self.assertEqual(device.received, ['admin', 'login-secret']) + self.assertTrue(conn.is_app_authorized()) + + def testInitTerminal(self): + device = Device(PROMPT) + conn = connect(self, device, 'junos') + conn.autoinit() + expected = ['set cli screen-length 0', 'set cli screen-width 0', 'set cli terminal ansi'] + self.assertEqual(device.received, expected) + + def testErrors(self): + device = Device('[edit]\r\n' + PROMPT.replace('>', '#')) + device.add_command('show foo', "\r\n ^\r\nunknown command.") + device.add_command('set interfaces', "\r\n ^\r\nsyntax error.") + device.add_command('commit', '\r\nerror: configuration check-out failed') + device.add_command('show version', '\r\nJunos: 23.4R1.10\r\n') + conn = connect(self, device, 'junos') + for command in 'show foo', 'set interfaces ge-0/0/0 unit', 'commit': + with self.subTest(command=command): + self.assertRaises(InvalidCommandException, conn.execute, command) + conn.execute('show version') + self.assertIn('Junos: 23.4R1.10', conn.response) + + +def suite(): + return unittest.TestLoader().loadTestsFromTestCase(JunOSDriverTest) + + +if __name__ == '__main__': + unittest.TextTestRunner(verbosity=2).run(suite()) diff --git a/tests/Exscript/protocols/drivers/junos_erxTest.py b/tests/Exscript/protocols/drivers/junos_erxTest.py new file mode 100644 index 00000000..f4db887a --- /dev/null +++ b/tests/Exscript/protocols/drivers/junos_erxTest.py @@ -0,0 +1,52 @@ +""" +The junos_erx driver, with the banner and the prompts of the Juniper E-series +(JUNOSe, formerly ERX). +""" + +import unittest + +from Exscript.protocols.drivers import driver_map + +from .device import ACCOUNT, Device, banner, connect, search + + +class JunOSERXDriverTest(unittest.TestCase): + def setUp(self): + self.driver = driver_map['junos_erx'] + + def testCheckHeadForOs(self): + self.assertEqual(self.driver.check_head_for_os(banner('junos_erx')), 75) + self.assertEqual(self.driver.check_head_for_os('\r\nUser Access Verification\r\n'), 0) + + def testPrompts(self): + for prompt in 'erx1>', 'erx1#', 'erx1(config)#', 'erx1(config-if)#', 'erx1:vr1#': + self.assertTrue(search(self.driver.prompt_re, 'logged in\r\n' + prompt), prompt) + + def testLoginAndEnable(self): + device = Device( + 'erx1>', + login=('User: ', 'Password: '), + enabled_prompt='erx1#', + enable_command='enable 15', + ) + conn = connect(self, device, 'junos_erx', login=ACCOUNT) + expected = ['admin', 'login-secret', 'enable 15', 'enable-secret'] + self.assertEqual(device.received, expected) + self.assertTrue(conn.is_app_authorized()) + + def testTelnetPassword(self): + self.assertTrue(search(self.driver.password_re, '\r\nTelnet password: ')) + + def testInitTerminal(self): + device = Device('erx1#') + conn = connect(self, device, 'junos_erx') + conn.autoinit() + self.assertEqual(device.received, ['terminal length 0', 'terminal width 512']) + + +def suite(): + return unittest.TestLoader().loadTestsFromTestCase(JunOSERXDriverTest) + + +if __name__ == '__main__': + unittest.TextTestRunner(verbosity=2).run(suite()) diff --git a/tests/Exscript/protocols/drivers/vrpTest.py b/tests/Exscript/protocols/drivers/vrpTest.py new file mode 100644 index 00000000..2725f07b --- /dev/null +++ b/tests/Exscript/protocols/drivers/vrpTest.py @@ -0,0 +1,64 @@ +""" +The vrp driver, with the banners and the prompts of Huawei VRP. +""" + +import unittest + +from Exscript.protocols.drivers import driver_map +from Exscript.protocols.exception import InvalidCommandException + +from .device import Device, banner, connect, search + + +class VRPDriverTest(unittest.TestCase): + def setUp(self): + self.driver = driver_map['vrp'] + + def testCheckHeadForOs(self): + head = 'Huawei Versatile Routing Platform Software\r\nVRP (R) software, Version 8.180\r\n' + self.assertEqual(self.driver.check_head_for_os(head), 80) + self.assertEqual(self.driver.check_head_for_os(banner('vrp.1')), 70) + self.assertEqual(self.driver.check_head_for_os('Info: Welcome\r\n'), 60) + self.assertEqual(self.driver.check_head_for_os('\r\nUser Access Verification\r\n'), 0) + + def testPrompts(self): + # The user view, the system view, the view of an interface and of + # OSPF, and the prompts with uncommitted changes (VRP 8). + prompts = ( + '', + '[HUAWEI]', + '[HUAWEI-GigabitEthernet0/0/1]', + '[~HUAWEI]', + '[*HUAWEI-ospf-1]', + ) + for prompt in prompts: + self.assertTrue(search(self.driver.prompt_re, 'Info: done\r\n' + prompt), prompt) + self.assertFalse(search(self.driver.prompt_re, '\r\n display version')) + + def testInitTerminal(self): + device = Device('') + conn = connect(self, device, 'vrp') + conn.autoinit() + self.assertEqual(device.received, ['screen-length 0 temporary']) + + def testError(self): + device = Device('') + device.add_command( + 'disp', "\r\n ^\r\nError: Unrecognized command found at '^' position." + ) + conn = connect(self, device, 'vrp') + self.assertRaises(InvalidCommandException, conn.execute, 'disp ip rout') + + def testLogout(self): + device = Device('') + conn = connect(self, device, 'vrp') + self.driver.logout(conn) + self.assertEqual(device.received, ['quit']) + + +def suite(): + return unittest.TestLoader().loadTestsFromTestCase(VRPDriverTest) + + +if __name__ == '__main__': + unittest.TextTestRunner(verbosity=2).run(suite()) diff --git a/tests/Exscript/protocols/drivers/vxworksTest.py b/tests/Exscript/protocols/drivers/vxworksTest.py new file mode 100644 index 00000000..0d58c217 --- /dev/null +++ b/tests/Exscript/protocols/drivers/vxworksTest.py @@ -0,0 +1,64 @@ +""" +The vxworks driver, with the banner, the prompts and the messages of a Huawei +MA5600T access node. +""" + +import unittest + +from Exscript.protocols.drivers import driver_map +from Exscript.protocols.exception import InvalidCommandException + +from .device import ACCOUNT, Device, banner, connect, search + + +class VxworksDriverTest(unittest.TestCase): + def setUp(self): + self.driver = driver_map['vxworks'] + + def testCheckHeadForOs(self): + self.assertEqual(self.driver.check_head_for_os(banner('vxworks')), 90) + self.assertEqual(self.driver.check_head_for_os('\r\nUser Access Verification\r\n'), 0) + + def testCheckResponseForOs(self): + self.assertEqual(self.driver.check_response_for_os(banner('vxworks')), 90) + self.assertEqual(self.driver.check_response_for_os('\r\nLast login: Mon Nov 9\r\n'), 0) + + def testPrompts(self): + for prompt in 'MA5600T>', 'MA5600T#', 'MA5600T(config)#', 'MA5600T(config-if-gpon-0/2)#': + self.assertTrue(search(self.driver.prompt_re, ' ---\r\n' + prompt), prompt) + question = '\r\n Are you sure to reboot the system? (y/n)[n]:' + self.assertTrue(search(self.driver.prompt_re, question)) + + def testLoginAndEnable(self): + # The level of the user grants the privileges: enable asks for no password. + device = Device( + 'MA5600T>', + banner=banner('vxworks').split('>>')[0], + login=('>>User name:', '>>User password:'), + enabled_prompt='MA5600T#', + enable_password_prompt=None, + ) + conn = connect(self, device, 'vxworks', login=ACCOUNT) + self.assertEqual(device.received, ['admin', 'login-secret', 'enable']) + self.assertTrue(conn.is_app_authorized()) + + def testErrors(self): + device = Device('MA5600T#') + device.add_command( + 'display boad', + "\r\n ^\r\n % Unknown command, the error locates at '^'", + ) + device.add_command('display board 9', '\r\n Failure: The board does not exist') + device.add_command('display port', "\r\n % Parameter error, the error locates at '^'") + conn = connect(self, device, 'vxworks') + for command in 'display boad 0', 'display board 9', 'display port state x': + with self.subTest(command=command): + self.assertRaises(InvalidCommandException, conn.execute, command) + + +def suite(): + return unittest.TestLoader().loadTestsFromTestCase(VxworksDriverTest) + + +if __name__ == '__main__': + unittest.TextTestRunner(verbosity=2).run(suite()) diff --git a/tests/Exscript/protocols/drivers/zteTest.py b/tests/Exscript/protocols/drivers/zteTest.py new file mode 100644 index 00000000..7fe2ae2b --- /dev/null +++ b/tests/Exscript/protocols/drivers/zteTest.py @@ -0,0 +1,53 @@ +""" +The zte driver, with the banner, the prompts and the messages of a ZTE ZXA10 +access node. +""" + +import unittest + +from Exscript.protocols.drivers import driver_map +from Exscript.protocols.exception import InvalidCommandException + +from .device import ACCOUNT, Device, banner, connect, search + + +class ZteDriverTest(unittest.TestCase): + def setUp(self): + self.driver = driver_map['zte'] + + def testCheckHeadForOs(self): + self.assertEqual(self.driver.check_head_for_os(banner('zte')), 90) + self.assertEqual(self.driver.check_head_for_os('\r\nUser Access Verification\r\n'), 0) + + def testPrompts(self): + for prompt in 'ZXAN#', 'ZXAN>', 'ZXAN(config)#', 'ZXAN(config-if-gpon-olt_1/2/1)#': + self.assertTrue(search(self.driver.prompt_re, 'Done\r\n' + prompt), prompt) + + def testLoginAndEnable(self): + device = Device( + 'ZXAN>', + banner=banner('zte').split('Username')[0], + login=('Username:', 'Password:'), + enabled_prompt='ZXAN#', + enable_password_prompt='Password:', + ) + conn = connect(self, device, 'zte', login=ACCOUNT) + self.assertEqual(device.received, ['admin', 'login-secret', 'enable', 'enable-secret']) + self.assertTrue(conn.is_app_authorized()) + + def testErrors(self): + device = Device('ZXAN#') + device.add_command('show car', "\r\n%Error 20203: Incomplete command.") + device.add_command('shw', "\r\n%Error 20200: Invalid input detected at '^' marker.") + conn = connect(self, device, 'zte') + for command in 'show card', 'shw card': + with self.subTest(command=command): + self.assertRaises(InvalidCommandException, conn.execute, command) + + +def suite(): + return unittest.TestLoader().loadTestsFromTestCase(ZteDriverTest) + + +if __name__ == '__main__': + unittest.TextTestRunner(verbosity=2).run(suite()) From 5de468578d7c0ac64b706658cddf1b40f548b688 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 02:01:30 +0100 Subject: [PATCH 193/250] Test the drivers of switches and controllers with their prompts --- .../Exscript/protocols/drivers/adtranTest.py | 47 +++++++++++++ tests/Exscript/protocols/drivers/arubaTest.py | 53 ++++++++++++++ .../Exscript/protocols/drivers/brocadeTest.py | 66 +++++++++++++++++ .../protocols/drivers/cienasaosTest.py | 45 ++++++++++++ .../protocols/drivers/enterasys_wcTest.py | 43 ++++++++++++ tests/Exscript/protocols/drivers/eosTest.py | 70 +++++++++++++++++++ .../Exscript/protocols/drivers/one_osTest.py | 58 +++++++++++++++ 7 files changed, 382 insertions(+) create mode 100644 tests/Exscript/protocols/drivers/adtranTest.py create mode 100644 tests/Exscript/protocols/drivers/arubaTest.py create mode 100644 tests/Exscript/protocols/drivers/brocadeTest.py create mode 100644 tests/Exscript/protocols/drivers/cienasaosTest.py create mode 100644 tests/Exscript/protocols/drivers/enterasys_wcTest.py create mode 100644 tests/Exscript/protocols/drivers/eosTest.py create mode 100644 tests/Exscript/protocols/drivers/one_osTest.py diff --git a/tests/Exscript/protocols/drivers/adtranTest.py b/tests/Exscript/protocols/drivers/adtranTest.py new file mode 100644 index 00000000..5af457b9 --- /dev/null +++ b/tests/Exscript/protocols/drivers/adtranTest.py @@ -0,0 +1,47 @@ +""" +The adtran driver, with the banner and the prompts of an ADTRAN Total Access +DSLAM. +""" + +import unittest + +from .device import ACCOUNT, Device, connect + + +class AdtranDriverTest(unittest.TestCase): + def device(self): + return Device( + 'dslam123> ', + login=('dslam123 login: ', 'Password:'), + enabled_prompt='dslam123# ', + enable_password_prompt='Password:', + ) + + def testLoginAndEnable(self): + # The authorization prepares the terminal before it sends enable. + device = self.device() + conn = connect(self, device, 'adtran', login=ACCOUNT) + expected = [ + 'admin', + 'login-secret', + 'terminal length 0', + 'terminal column 0', + 'enable', + 'enable-secret', + ] + self.assertEqual(device.received, expected) + self.assertTrue(conn.is_app_authorized()) + + def testInitTerminal(self): + device = Device('dslam123# ') + conn = connect(self, device, 'adtran') + conn.autoinit() + self.assertEqual(device.received, ['terminal length 0', 'terminal column 0']) + + +def suite(): + return unittest.TestLoader().loadTestsFromTestCase(AdtranDriverTest) + + +if __name__ == '__main__': + unittest.TextTestRunner(verbosity=2).run(suite()) diff --git a/tests/Exscript/protocols/drivers/arubaTest.py b/tests/Exscript/protocols/drivers/arubaTest.py new file mode 100644 index 00000000..9d4db4d1 --- /dev/null +++ b/tests/Exscript/protocols/drivers/arubaTest.py @@ -0,0 +1,53 @@ +""" +The aruba driver, with the banner and the prompts of an Aruba mobility +controller (ArubaOS). +""" + +import unittest + +from Exscript.protocols.drivers import driver_map +from Exscript.protocols.exception import InvalidCommandException + +from .device import ACCOUNT, Device, banner, connect, search + + +class ArubaDriverTest(unittest.TestCase): + def setUp(self): + self.driver = driver_map['aruba'] + + def testCheckHeadForOs(self): + self.assertEqual(self.driver.check_head_for_os(banner('aruba.1')), 88) + self.assertEqual(self.driver.check_head_for_os('\r\n(ArubaS3500-24P) #'), 88) + self.assertEqual(self.driver.check_head_for_os('\r\nUser Access Verification\r\n'), 0) + + def testPrompts(self): + for prompt in '(Aruba650) >', '(Aruba650) #', '(Aruba7030) # ': + self.assertTrue(search(self.driver.prompt_re, 'Last login\r\n' + prompt), prompt) + self.assertFalse(search(self.driver.prompt_re, '\r\nAruba650 >')) + + def testEnable(self): + device = Device('(Aruba650) >', enabled_prompt='(Aruba650) #') + conn = connect(self, device, 'aruba') + conn.auto_app_authorize(ACCOUNT) + self.assertEqual(device.received, ['enable', 'enable-secret']) + self.assertTrue(conn.is_app_authorized()) + + def testInitTerminal(self): + device = Device('(Aruba650) #') + conn = connect(self, device, 'aruba') + conn.autoinit() + self.assertEqual(device.received, ['no paging']) + + def testError(self): + device = Device('(Aruba650) #') + device.add_command('show ap databse', "\r\n% Invalid input detected at '^' marker.") + conn = connect(self, device, 'aruba') + self.assertRaises(InvalidCommandException, conn.execute, 'show ap databse') + + +def suite(): + return unittest.TestLoader().loadTestsFromTestCase(ArubaDriverTest) + + +if __name__ == '__main__': + unittest.TextTestRunner(verbosity=2).run(suite()) diff --git a/tests/Exscript/protocols/drivers/brocadeTest.py b/tests/Exscript/protocols/drivers/brocadeTest.py new file mode 100644 index 00000000..12d286ea --- /dev/null +++ b/tests/Exscript/protocols/drivers/brocadeTest.py @@ -0,0 +1,66 @@ +""" +The brocade driver, with the banners and the prompts of a Brocade NetIron +MLX router. +""" + +import unittest + +from Exscript.protocols.drivers import driver_map +from Exscript.protocols.exception import InvalidCommandException + +from .device import ACCOUNT, Device, banner, connect, search + + +class BrocadeDriverTest(unittest.TestCase): + def setUp(self): + self.driver = driver_map['brocade'] + + def testCheckHeadForOs(self): + self.assertEqual(self.driver.check_head_for_os(banner('brocade.2')), 95) + # The prompt shows the protocol. + self.assertEqual(self.driver.check_head_for_os(banner('brocade.1')), 90) + self.assertEqual(self.driver.check_head_for_os('\r\nRouter#'), 0) + + def testPrompts(self): + prompts = ( + 'SSH@mpls1.ams2.nl>', + 'telnet@mlx1#', + 'SSH@mlx1(config)#', + 'SSH@mlx1(config-if-e10000-1/1)#', + 'Warning: 1 user(s) already in config mode.\r\nSSH@mlx1(config)#', + ) + for prompt in prompts: + self.assertTrue(search(self.driver.prompt_re, 'Done.\r\n' + prompt), prompt) + self.assertFalse(search(self.driver.prompt_re, '\r\nmlx1#')) + + def testLoginAndEnable(self): + device = Device( + 'telnet@mlx1>', + banner='\r\nUser Access Verification\r\n\r\n', + login=('Please Enter Login Name: ', 'Please Enter Password: '), + enabled_prompt='telnet@mlx1#', + enable_password_prompt='Password:', + ) + conn = connect(self, device, 'brocade', login=ACCOUNT) + self.assertEqual(device.received, ['admin', 'login-secret', 'enable', 'enable-secret']) + self.assertTrue(conn.is_app_authorized()) + + def testInitTerminal(self): + device = Device('SSH@mlx1#') + conn = connect(self, device, 'brocade') + conn.autoinit() + self.assertEqual(device.received, ['terminal length 0']) + + def testError(self): + device = Device('SSH@mlx1#') + device.add_command('shw', '\r\nInvalid input -> shw ver\r\nType ? for a list') + conn = connect(self, device, 'brocade') + self.assertRaises(InvalidCommandException, conn.execute, 'shw ver') + + +def suite(): + return unittest.TestLoader().loadTestsFromTestCase(BrocadeDriverTest) + + +if __name__ == '__main__': + unittest.TextTestRunner(verbosity=2).run(suite()) diff --git a/tests/Exscript/protocols/drivers/cienasaosTest.py b/tests/Exscript/protocols/drivers/cienasaosTest.py new file mode 100644 index 00000000..fa4bdfab --- /dev/null +++ b/tests/Exscript/protocols/drivers/cienasaosTest.py @@ -0,0 +1,45 @@ +""" +The cienasaos driver, with the banner and the prompt of a Ciena SAOS switch. +""" + +import unittest + +from Exscript.protocols.drivers import driver_map +from Exscript.protocols.exception import InvalidCommandException + +from .device import ACCOUNT, Device, banner, connect + + +class CienaSAOSDriverTest(unittest.TestCase): + def setUp(self): + self.driver = driver_map['cienasaos'] + + def testCheckHeadForOs(self): + self.assertEqual(self.driver.check_head_for_os(banner('cienasaos.1')), 90) + self.assertEqual(self.driver.check_head_for_os('\r\nsaos-5160 login: '), 0) + + def testLogin(self): + device = Device('saos-5160> ', login=('saos-5160 login: ', 'Password:')) + conn = connect(self, device, 'cienasaos', login=ACCOUNT) + self.assertEqual(device.received, ['admin', 'login-secret']) + self.assertTrue(conn.is_app_authorized()) + + def testInitTerminal(self): + device = Device('saos-5160> ') + conn = connect(self, device, 'cienasaos') + conn.autoinit() + self.assertEqual(device.received, ['system shell session set more off']) + + def testError(self): + device = Device('saos-5160> ') + device.add_command('port shw', "\r\nSHELL PARSER FAILURE: 'shw' - no valid completion") + conn = connect(self, device, 'cienasaos') + self.assertRaises(InvalidCommandException, conn.execute, 'port shw') + + +def suite(): + return unittest.TestLoader().loadTestsFromTestCase(CienaSAOSDriverTest) + + +if __name__ == '__main__': + unittest.TextTestRunner(verbosity=2).run(suite()) diff --git a/tests/Exscript/protocols/drivers/enterasys_wcTest.py b/tests/Exscript/protocols/drivers/enterasys_wcTest.py new file mode 100644 index 00000000..0f1095a9 --- /dev/null +++ b/tests/Exscript/protocols/drivers/enterasys_wcTest.py @@ -0,0 +1,43 @@ +""" +The enterasys_wc driver, with the banner and the prompts of an Enterasys +(HiPath) wireless controller. +""" + +import unittest + +from Exscript.protocols.drivers import driver_map + +from .device import ACCOUNT, Device, banner, connect, search + + +class EnterasysWCDriverTest(unittest.TestCase): + def setUp(self): + self.driver = driver_map['enterasys_wc'] + + def testCheckHeadForOs(self): + self.assertEqual(self.driver.check_head_for_os(banner('enterasys_wc')), 85) + self.assertEqual(self.driver.check_head_for_os('\r\nUser Access Verification\r\n'), 0) + + def testPrompts(self): + for prompt in 'HWC-Name# ', 'HWC-Name:ap# ', 'HWC-Name:ap:defaults# ': + self.assertTrue(search(self.driver.prompt_re, 'Done\r\n' + prompt), prompt) + + def testLogin(self): + device = Device('HWC-Name# ', login=('Username: ', 'Password: ')) + conn = connect(self, device, 'enterasys_wc', login=ACCOUNT) + self.assertEqual(device.received, ['admin', 'login-secret']) + self.assertTrue(conn.is_app_authorized()) + + def testInitTerminalSendsNothing(self): + device = Device('HWC-Name# ') + conn = connect(self, device, 'enterasys_wc') + conn.autoinit() + self.assertEqual(device.received, []) + + +def suite(): + return unittest.TestLoader().loadTestsFromTestCase(EnterasysWCDriverTest) + + +if __name__ == '__main__': + unittest.TextTestRunner(verbosity=2).run(suite()) diff --git a/tests/Exscript/protocols/drivers/eosTest.py b/tests/Exscript/protocols/drivers/eosTest.py new file mode 100644 index 00000000..0b421f69 --- /dev/null +++ b/tests/Exscript/protocols/drivers/eosTest.py @@ -0,0 +1,70 @@ +""" +The eos driver, with the banners, the prompts and the messages of Arista EOS. +""" + +import unittest + +from Exscript.protocols.drivers import driver_map +from Exscript.protocols.exception import InvalidCommandException + +from .device import ACCOUNT, Device, banner, connect, search + + +class EOSDriverTest(unittest.TestCase): + def setUp(self): + self.driver = driver_map['eos'] + + def testPrompts(self): + prompts = ( + 'switch>', + 'switch#', + 'switch(config)#', + 'switch(config-if-Et1)#', + 'hostname(s1)#', + 'switch(vrf:blah)(config)#', + ) + for prompt in prompts: + self.assertTrue(search(self.driver.prompt_re, 'Last login\r\n' + prompt), prompt) + # EOS sends no line break before the first prompt. + self.assertTrue(search(self.driver.prompt_re, 'switch>')) + + def testLoginPatterns(self): + self.assertTrue(search(self.driver.user_re, banner('eos.1').split('testuser')[0])) + self.assertTrue(search(self.driver.password_re, 'testuser\r\npassword:')) + for error in '% Bad secrets', 'Login incorrect': + self.assertTrue(search(self.driver.login_error_re, 'Password:\r\n' + error), error) + + def testLoginAndEnable(self): + device = Device( + 'switch>', + banner='---\r\nThis is a login banner\r\n---\r\n', + login=('switch login: ', 'Password: '), + enabled_prompt='switch#', + ) + conn = connect(self, device, 'eos', login=ACCOUNT) + self.assertEqual(device.received, ['admin', 'login-secret', 'enable', 'enable-secret']) + self.assertTrue(conn.is_app_authorized()) + + def testInitTerminal(self): + device = Device('switch#') + conn = connect(self, device, 'eos') + conn.autoinit() + self.assertEqual(device.received, ['terminal dont-ask', 'terminal length 0']) + + def testErrors(self): + device = Device('switch#') + device.add_command('show vlna', "\r\n% Invalid input") + device.add_command('show ip', "\r\n% Incomplete command") + device.add_command('sh', "\r\n% Ambiguous command") + conn = connect(self, device, 'eos') + for command in 'show vlna', 'show ip', 'sh': + with self.subTest(command=command): + self.assertRaises(InvalidCommandException, conn.execute, command) + + +def suite(): + return unittest.TestLoader().loadTestsFromTestCase(EOSDriverTest) + + +if __name__ == '__main__': + unittest.TextTestRunner(verbosity=2).run(suite()) diff --git a/tests/Exscript/protocols/drivers/one_osTest.py b/tests/Exscript/protocols/drivers/one_osTest.py new file mode 100644 index 00000000..00fd43d1 --- /dev/null +++ b/tests/Exscript/protocols/drivers/one_osTest.py @@ -0,0 +1,58 @@ +""" +The one_os driver, with the banners and the prompts of OneAccess OneOS. +""" + +import unittest + +from Exscript.protocols.drivers import driver_map + +from .device import ACCOUNT, Device, banner, connect, search + + +class OneOSDriverTest(unittest.TestCase): + def setUp(self): + self.driver = driver_map['one_os'] + + def testCheckHeadForOs(self): + # The first prompt follows an empty line. + self.assertEqual(self.driver.check_head_for_os(banner('one_os.1').rstrip('\n')), 40) + self.assertEqual(self.driver.check_head_for_os('Last login\r\ntesthost>'), 0) + + def testPrompts(self): + for prompt in 'ONEOS5-ROUTER>', 'ONEOS5-ROUTER#', 'router1(config)#', 'router1(cfg-if)#': + self.assertTrue(search(self.driver.prompt_re, 'Done\r\n' + prompt), prompt) + + def testLoginAndEnable(self): + device = Device( + 'testhost>', + banner=banner('one_os.1').split('Username')[0], + login=('Username:', 'Password:'), + enabled_prompt='testhost#', + enable_password_prompt='Password:', + ) + conn = connect(self, device, 'one_os', login=ACCOUNT) + self.assertEqual(device.received, ['admin', 'login-secret', 'enable', 'enable-secret']) + self.assertTrue(conn.is_app_authorized()) + + def testInitTerminal(self): + # OneOS 5 sets the width with stty. + device = Device('router1#') + conn = connect(self, device, 'one_os') + conn.autoinit() + self.assertEqual(device.received, ['term len 0', 'stty columns 255']) + + def testInitTerminalOfOneOS4(self): + # OneOS 4 has no stty; the error is ignored. + device = Device('router1#') + device.add_command('stty', '\r\n% Unrecognized command') + conn = connect(self, device, 'one_os') + conn.autoinit() + self.assertEqual(device.received, ['term len 0', 'stty columns 255']) + + +def suite(): + return unittest.TestLoader().loadTestsFromTestCase(OneOSDriverTest) + + +if __name__ == '__main__': + unittest.TextTestRunner(verbosity=2).run(suite()) From 7be9998dd4300e5b5199b147b5a681e5f87a9323 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 02:01:41 +0100 Subject: [PATCH 194/250] Test the drivers of routers, firewalls and WAN appliances --- tests/Exscript/protocols/drivers/bigipTest.py | 55 ++++++++++++++ .../Exscript/protocols/drivers/fortiosTest.py | 54 ++++++++++++++ tests/Exscript/protocols/drivers/mrvTest.py | 39 ++++++++++ tests/Exscript/protocols/drivers/riosTest.py | 52 +++++++++++++ .../protocols/drivers/smart_edge_osTest.py | 56 ++++++++++++++ .../Exscript/protocols/drivers/sp_vxoaTest.py | 45 ++++++++++++ tests/Exscript/protocols/drivers/srosTest.py | 73 +++++++++++++++++++ 7 files changed, 374 insertions(+) create mode 100644 tests/Exscript/protocols/drivers/bigipTest.py create mode 100644 tests/Exscript/protocols/drivers/fortiosTest.py create mode 100644 tests/Exscript/protocols/drivers/mrvTest.py create mode 100644 tests/Exscript/protocols/drivers/riosTest.py create mode 100644 tests/Exscript/protocols/drivers/smart_edge_osTest.py create mode 100644 tests/Exscript/protocols/drivers/sp_vxoaTest.py create mode 100644 tests/Exscript/protocols/drivers/srosTest.py diff --git a/tests/Exscript/protocols/drivers/bigipTest.py b/tests/Exscript/protocols/drivers/bigipTest.py new file mode 100644 index 00000000..27f6bc5a --- /dev/null +++ b/tests/Exscript/protocols/drivers/bigipTest.py @@ -0,0 +1,55 @@ +""" +The bigip driver, with the banner and the prompt of the TMSH shell of an F5 +BIG-IP. +""" + +import unittest + +from Exscript.protocols.drivers import driver_map +from Exscript.protocols.exception import InvalidCommandException + +from .device import ACCOUNT, Device, banner, connect, search + +PROMPT = 'admin@(bigip1)(cfg-sync Standalone)(Active)(/Common)(tmos)# ' + + +class BigIPDriverTest(unittest.TestCase): + def setUp(self): + self.driver = driver_map['bigip'] + + def testCheckHeadForOs(self): + self.assertEqual(self.driver.check_head_for_os(banner('bigip.1')), 90) + self.assertEqual(self.driver.check_head_for_os('\r\n[admin@bigip1:Active] ~ # '), 0) + + def testPrompts(self): + prompts = PROMPT, 'admin@(bigip1)(cfg-sync In Sync)(Active)(/Common)(tmos.ltm)# ' + for prompt in prompts: + self.assertTrue(search(self.driver.prompt_re, 'Last login\r\n' + prompt), prompt) + self.assertFalse(search(self.driver.prompt_re, '\r\n[admin@bigip1:Active] ~ # ')) # bash + + def testNoAuthorization(self): + # The role of the user grants the privileges. + device = Device(PROMPT) + conn = connect(self, device, 'bigip') + conn.auto_app_authorize(ACCOUNT) + self.assertEqual(device.received, []) + + def testInitTerminal(self): + device = Device(PROMPT) + conn = connect(self, device, 'bigip') + conn.autoinit() + self.assertEqual(device.received, ['modify cli preference pager disabled']) + + def testError(self): + device = Device(PROMPT) + device.add_command('list ltm pol', '\r\nSyntax Error: "pol" unknown property') + conn = connect(self, device, 'bigip') + self.assertRaises(InvalidCommandException, conn.execute, 'list ltm pol') + + +def suite(): + return unittest.TestLoader().loadTestsFromTestCase(BigIPDriverTest) + + +if __name__ == '__main__': + unittest.TextTestRunner(verbosity=2).run(suite()) diff --git a/tests/Exscript/protocols/drivers/fortiosTest.py b/tests/Exscript/protocols/drivers/fortiosTest.py new file mode 100644 index 00000000..10bbf0b6 --- /dev/null +++ b/tests/Exscript/protocols/drivers/fortiosTest.py @@ -0,0 +1,54 @@ +""" +The fortios driver, with the prompts and the messages of a FortiGate. +""" + +import unittest + +from Exscript.protocols.drivers import driver_map +from Exscript.protocols.exception import InvalidCommandException + +from .device import Device, banner, connect, search + + +class FortiOSDriverTest(unittest.TestCase): + def setUp(self): + self.driver = driver_map['fortios'] + + def testCheckHeadForOs(self): + # By default, a FortiGate shows no banner, only the prompt. + self.assertEqual(self.driver.check_head_for_os(banner('fortios.1')), 50) + self.assertEqual(self.driver.check_head_for_os("admin@fgt1's password: "), 50) + self.assertEqual(self.driver.check_head_for_os('Last login: Wed May 8 09:11:32 2019'), 0) + self.assertEqual(self.driver.check_head_for_os('Welcome\r\nFGT60E # '), 0) + + def testPrompts(self): + for prompt in 'FGT60E # ', 'FGT60E (global) # ', 'FGT60E (console) # ', 'FGT60E $ ': + self.assertTrue(search(self.driver.prompt_re, 'end\r\n' + prompt), prompt) + self.assertFalse(search(self.driver.prompt_re, '\r\n set output standard')) + + def testInitTerminal(self): + # The pager is a setting of the console, in the global configuration. + device = Device('FGT60E # ') + conn = connect(self, device, 'fortios') + conn.autoinit() + expected = ['config global', 'config system console', 'set output standard', 'end', 'end'] + self.assertEqual(device.received, expected) + + def testErrors(self): + device = Device('FGT60E # ') + device.add_command( + 'set ip', '\r\ninvalid netmask.\r\nobject check operator error, -9, discard the setting' + ) + device.add_command('edit 99', '\r\nCommand fail. Return code -3') + conn = connect(self, device, 'fortios') + for command in 'set ip 10.0.0.1 255.0.0.1', 'edit 99': + with self.subTest(command=command): + self.assertRaises(InvalidCommandException, conn.execute, command) + + +def suite(): + return unittest.TestLoader().loadTestsFromTestCase(FortiOSDriverTest) + + +if __name__ == '__main__': + unittest.TextTestRunner(verbosity=2).run(suite()) diff --git a/tests/Exscript/protocols/drivers/mrvTest.py b/tests/Exscript/protocols/drivers/mrvTest.py new file mode 100644 index 00000000..871637fa --- /dev/null +++ b/tests/Exscript/protocols/drivers/mrvTest.py @@ -0,0 +1,39 @@ +""" +The mrv driver, with the banners and the prompt of MRV Linux, e.g. of an MRV +EM316 network module. +""" + +import unittest + +from Exscript.protocols.drivers import driver_map + +from .device import Device, banner, connect, search + + +class MRVDriverTest(unittest.TestCase): + def setUp(self): + self.driver = driver_map['mrv'] + + def testCheckHeadForOs(self): + self.assertEqual(self.driver.check_head_for_os(banner('mrv.1')), 75) + self.assertEqual(self.driver.check_head_for_os('\r\nAuthorized Login: '), 20) + self.assertEqual(self.driver.check_head_for_os('\r\nlogin: '), 0) + + def testPrompts(self): + self.assertTrue(search(self.driver.prompt_re, banner('mrv.2').rstrip('\n'))) + self.assertTrue(search(self.driver.prompt_re, '\r\nOS-9024:root$ ')) + self.assertFalse(search(self.driver.prompt_re, '\r\nYou are a VIEW user!')) + + def testInitTerminal(self): + device = Device('mcc$ ') + conn = connect(self, device, 'mrv') + conn.autoinit() + self.assertEqual(device.received, ['terminal length 0', 'terminal width 512']) + + +def suite(): + return unittest.TestLoader().loadTestsFromTestCase(MRVDriverTest) + + +if __name__ == '__main__': + unittest.TextTestRunner(verbosity=2).run(suite()) diff --git a/tests/Exscript/protocols/drivers/riosTest.py b/tests/Exscript/protocols/drivers/riosTest.py new file mode 100644 index 00000000..193ad775 --- /dev/null +++ b/tests/Exscript/protocols/drivers/riosTest.py @@ -0,0 +1,52 @@ +""" +The rios driver, with the prompts and the messages of Riverbed RiOS +(SteelHead). +""" + +import unittest + +from Exscript.protocols.drivers import driver_map +from Exscript.protocols.exception import InvalidCommandException + +from .device import ACCOUNT, Device, connect, search + + +class RIOSDriverTest(unittest.TestCase): + def setUp(self): + self.driver = driver_map['rios'] + + def testPrompts(self): + for prompt in 'amnesiac > ', 'amnesiac # ', 'amnesiac (config) # ', 'sh-1 (config) # ': + self.assertTrue(search(self.driver.prompt_re, 'Done\r\n' + prompt), prompt) + self.assertFalse(search(self.driver.prompt_re, '\r\namnesiac>')) + + def testInitTerminalEnables(self): + device = Device('amnesiac > ', enabled_prompt='amnesiac # ', enable_password_prompt=None) + conn = connect(self, device, 'rios') + conn.autoinit() + self.assertEqual(device.received, ['term len 0', 'enable']) + + def testNoAuthorization(self): + device = Device('amnesiac > ') + conn = connect(self, device, 'rios') + conn.auto_app_authorize(ACCOUNT) + self.assertEqual(device.received, []) + + def testErrors(self): + device = Device('amnesiac # ') + device.add_command( + 'shw', '\r\n% Unrecognized command "shw".\r\nType "help" to learn how to use' + ) + device.add_command('show in', '\r\n% Inactive command "show in".') + conn = connect(self, device, 'rios') + for command in 'shw version', 'show in': + with self.subTest(command=command): + self.assertRaises(InvalidCommandException, conn.execute, command) + + +def suite(): + return unittest.TestLoader().loadTestsFromTestCase(RIOSDriverTest) + + +if __name__ == '__main__': + unittest.TextTestRunner(verbosity=2).run(suite()) diff --git a/tests/Exscript/protocols/drivers/smart_edge_osTest.py b/tests/Exscript/protocols/drivers/smart_edge_osTest.py new file mode 100644 index 00000000..adaba673 --- /dev/null +++ b/tests/Exscript/protocols/drivers/smart_edge_osTest.py @@ -0,0 +1,56 @@ +""" +The smart_edge_os driver, with the banners and the prompts of a Redback +SmartEdge router. +""" + +import unittest + +from Exscript.protocols.drivers import driver_map + +from .device import ACCOUNT, Device, banner, connect, search + + +class SmartEdgeOSDriverTest(unittest.TestCase): + def setUp(self): + self.driver = driver_map['smart_edge_os'] + + def testCheckHeadForOs(self): + self.assertEqual(self.driver.check_head_for_os(banner('smart_edge_os')), 60) + self.assertEqual(self.driver.check_head_for_os('\r\nWelcome\r\nlogin: '), 20) + self.assertEqual(self.driver.check_head_for_os('\r\nUsername: '), 0) + + def testPrompts(self): + prompts = ( + '[local]Redback>', + '[local]Redback#', + '[local]Redback(config)#', + '[isp1]SE800-1(config-ctx)#', + ) + for prompt in prompts: + self.assertTrue(search(self.driver.prompt_re, 'Done\r\n' + prompt), prompt) + self.assertFalse(search(self.driver.prompt_re, '\r\nRedback#')) + + def testLoginAndEnable(self): + device = Device( + '[local]SE800-1>', + banner='\r\nSE800-1-se800\r\n\r\n', + login=('login: ', 'Password: '), + enabled_prompt='[local]SE800-1#', + ) + conn = connect(self, device, 'smart_edge_os', login=ACCOUNT) + self.assertEqual(device.received, ['admin', 'login-secret', 'enable', 'enable-secret']) + self.assertTrue(conn.is_app_authorized()) + + def testInitTerminal(self): + device = Device('[local]SE800-1#') + conn = connect(self, device, 'smart_edge_os') + conn.autoinit() + self.assertEqual(device.received, ['terminal length 0', 'terminal width 65536']) + + +def suite(): + return unittest.TestLoader().loadTestsFromTestCase(SmartEdgeOSDriverTest) + + +if __name__ == '__main__': + unittest.TextTestRunner(verbosity=2).run(suite()) diff --git a/tests/Exscript/protocols/drivers/sp_vxoaTest.py b/tests/Exscript/protocols/drivers/sp_vxoaTest.py new file mode 100644 index 00000000..a451f869 --- /dev/null +++ b/tests/Exscript/protocols/drivers/sp_vxoaTest.py @@ -0,0 +1,45 @@ +""" +The sp_vxoa driver, with the banner and the prompts of a Silver Peak +appliance (VXOA). +""" + +import unittest + +from Exscript.protocols.drivers import driver_map + +from .device import ACCOUNT, Device, banner, connect, search + + +class SilverPeakVXOADriverTest(unittest.TestCase): + def setUp(self): + self.driver = driver_map['sp_vxoa'] + + def testCheckHeadForOs(self): + # The banner breaks the line within the name of the company. + self.assertEqual(self.driver.check_head_for_os(banner('sp_vxoa')), 90) + self.assertEqual(self.driver.check_head_for_os('\r\nLast login: Wed May 8\r\n'), 0) + + def testPrompts(self): + for prompt in 'silverpeak_vxoa >', 'silverpeak_vxoa # ', 'silverpeak_vxoa (config) # ': + self.assertTrue(search(self.driver.prompt_re, 'Done\r\n' + prompt), prompt) + + def testEnable(self): + device = Device('silverpeak_vxoa >', enabled_prompt='silverpeak_vxoa # ') + conn = connect(self, device, 'sp_vxoa') + conn.auto_app_authorize(ACCOUNT) + self.assertEqual(device.received, ['enable', 'enable-secret']) + self.assertTrue(conn.is_app_authorized()) + + def testInitTerminal(self): + device = Device('silverpeak_vxoa # ') + conn = connect(self, device, 'sp_vxoa') + conn.autoinit() + self.assertEqual(device.received, ['terminal length 65536']) + + +def suite(): + return unittest.TestLoader().loadTestsFromTestCase(SilverPeakVXOADriverTest) + + +if __name__ == '__main__': + unittest.TextTestRunner(verbosity=2).run(suite()) diff --git a/tests/Exscript/protocols/drivers/srosTest.py b/tests/Exscript/protocols/drivers/srosTest.py new file mode 100644 index 00000000..e02bc874 --- /dev/null +++ b/tests/Exscript/protocols/drivers/srosTest.py @@ -0,0 +1,73 @@ +""" +The sros driver, with the banner, the prompts and the messages of Nokia +(Alcatel-Lucent) SR OS. +""" + +import unittest + +from Exscript.protocols.drivers import driver_map +from Exscript.protocols.exception import InvalidCommandException + +from .device import ACCOUNT, Device, banner, connect, search + + +class SROSDriverTest(unittest.TestCase): + def setUp(self): + self.driver = driver_map['sros'] + + def testCheckHeadForOs(self): + self.assertEqual(self.driver.check_head_for_os(banner('sros').rstrip('\n')), 95) + self.assertEqual(self.driver.check_head_for_os('\r\nRouter#'), 0) + + def testPrompts(self): + # The prompt shows the active CPM (A or B), the context, and an + # asterisk for unsaved changes. + prompts = ( + 'A:sros-pe1# ', + '*A:sros-pe1# ', + 'B:sros-pe1>config# ', + 'A:sros-pe1>config>router# ', + '*A:sros-pe1>config>service>vprn$ ', + ) + for prompt in prompts: + self.assertTrue(search(self.driver.prompt_re, 'Done\r\n' + prompt), prompt) + self.assertFalse(search(self.driver.prompt_re, '\r\nsros-pe1# ')) + + def testLogin(self): + device = Device( + 'A:sros-pe1# ', banner=banner('sros').split('Login')[0], login=('Login: ', 'Password: ') + ) + conn = connect(self, device, 'sros', login=ACCOUNT) + self.assertEqual(device.received, ['admin', 'login-secret']) + self.assertTrue(conn.is_app_authorized()) + + def testInitTerminal(self): + device = Device('A:sros-pe1# ') + conn = connect(self, device, 'sros') + conn.autoinit() + expected = [ + 'environment no more', + 'environment reduced-prompt 2', + 'environment no saved-ind-prompt', + ] + self.assertEqual(device.received, expected) + + def testError(self): + device = Device('A:sros-pe1# ') + device.add_command('shw', '\r\nError: Bad command.') + conn = connect(self, device, 'sros') + self.assertRaises(InvalidCommandException, conn.execute, 'shw router interface') + + def testLogout(self): + device = Device('A:sros-pe1# ') + conn = connect(self, device, 'sros') + self.driver.logout(conn) + self.assertEqual(device.received, ['logout']) + + +def suite(): + return unittest.TestLoader().loadTestsFromTestCase(SROSDriverTest) + + +if __name__ == '__main__': + unittest.TextTestRunner(verbosity=2).run(suite()) From 15aa744d5d67e49359cd780f047fe196b5a574a8 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 02:05:04 +0100 Subject: [PATCH 195/250] Leave the Windows fallback for termios out of the coverage The coverage is measured on Linux, where termios can always be imported. --- src/Exscript/protocols/protocol.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Exscript/protocols/protocol.py b/src/Exscript/protocols/protocol.py index 1b8a42d4..b60f1391 100644 --- a/src/Exscript/protocols/protocol.py +++ b/src/Exscript/protocols/protocol.py @@ -71,7 +71,7 @@ import tty _have_termios = True -except ImportError: +except ImportError: # pragma: no cover - Windows has no termios _have_termios = False _logger = logging.getLogger(__name__) From 227567a8ac48d47472a1ea6e569367523515c051 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 02:05:59 +0100 Subject: [PATCH 196/250] Test the command files and the copies of the emulator's command sets --- tests/Exscript/emulators/CommandSetTest.py | 44 +++++++++++++++++++++- 1 file changed, 43 insertions(+), 1 deletion(-) diff --git a/tests/Exscript/emulators/CommandSetTest.py b/tests/Exscript/emulators/CommandSetTest.py index d55595dd..d03e8976 100644 --- a/tests/Exscript/emulators/CommandSetTest.py +++ b/tests/Exscript/emulators/CommandSetTest.py @@ -1,3 +1,6 @@ +import copy +import os +import tempfile import unittest from Exscript.emulators import CommandSet @@ -29,12 +32,51 @@ def sayhello(cmd): cs.add('hi', sayhello) self.assertEqual(cs.eval('hi'), 'hello') + def writeFile(self, source): + tmpdir = self.enterContext(tempfile.TemporaryDirectory()) + filename = os.path.join(tmpdir, 'commands.py') + with open(filename, 'w', encoding='utf-8') as fp: + fp.write(source) + return filename + def testAddFromFile(self): - pass # FIXME + # A response of None marks a command without a response, also in strict mode. + filename = self.writeFile( + 'def uptime(command):\n' + ' return "up 2 days"\n' + 'commands = [("show uptime", uptime), (r"show ver.*", "IOS 15.2"), ("clear", None)]\n' + ) + cs = CommandSet(strict=True) + cs.add_from_file(filename) + self.assertEqual(cs.eval('show uptime'), 'up 2 days') + self.assertEqual(cs.eval('show version'), 'IOS 15.2') + self.assertIsNone(cs.eval('clear')) + self.assertRaises(ValueError, cs.eval, 'reload') + + def testAddFromFileWithoutCommands(self): + cs = CommandSet() + for source, error in ( + ('responses = []\n', 'no variable named "commands"'), + ('commands = 42\n', 'variable "commands" is not iterable'), + ): + with self.subTest(source=source): + filename = self.writeFile(source) + with self.assertRaisesRegex(ValueError, error): + cs.add_from_file(filename) def testEval(self): pass # See testAdd() + def testDeepcopy(self): + cs = CommandSet(strict=False) + cs.add('show version', 'IOS 15.2') + clone = copy.deepcopy(cs) + self.assertFalse(clone.strict) + self.assertEqual(clone.eval('show version'), 'IOS 15.2') + clone.add('show uptime', 'up 2 days') + self.assertEqual(clone.eval('show uptime'), 'up 2 days') + self.assertIsNone(cs.eval('show uptime')) # the original is unchanged + def suite(): return unittest.TestLoader().loadTestsFromTestCase(CommandSetTest) From fd5bd17f4fc5b7297b9703c37b5d4c7a71ec2385 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 02:07:30 +0100 Subject: [PATCH 197/250] Test copies of hosts, invalid URIs, and hosts without variables --- tests/Exscript/HostTest.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/tests/Exscript/HostTest.py b/tests/Exscript/HostTest.py index 6fc00493..e2fecfb0 100644 --- a/tests/Exscript/HostTest.py +++ b/tests/Exscript/HostTest.py @@ -1,3 +1,5 @@ +import copy +import re import unittest from util.urlTest import urls @@ -65,6 +67,25 @@ def testNameThatStartsWithAnIp(self): self.assertEqual(host.get_name(), name) self.assertEqual(host.get_address(), name) + def testInvalidUri(self): + for uri in 'ssh://router:abc', 'ssh://[::1': + with self.assertRaisesRegex( + ValueError, re.escape('Hostname parse error: ' + repr(uri)) + ): + Host(uri) + + def testCopy(self): + # The copy gets the URI, i.e. all but the options and the variables + # that are no strings, and the name. + host = Host('telnet://admin:secret@10.0.0.1:2323?vlan=10') + host.set_name('router1') + clone = copy.copy(host) + self.assertEqual(clone.get_name(), 'router1') + self.assertEqual(clone.get_uri(), host.get_uri()) + self.assertEqual((clone.get_protocol(), clone.get_tcp_port()), ('telnet', 2323)) + self.assertEqual(clone.get_account().get_password(), 'secret') + self.assertEqual(clone.get('vlan'), ['10']) + def testGetAddress(self): self.assertEqual(self.host.get_address(), 'localhost') # Additional tests are in testSetAddress(). @@ -98,6 +119,8 @@ def testSetOption(self): self.host.set_option('verify_fingerprint', True) self.assertEqual(self.host.get_option('verify_fingerprint'), True) self.assertEqual(self.host.get_options(), {'verify_fingerprint': True}) + self.host.set_option('timeout', 60) + self.assertEqual(self.host.get_options(), {'verify_fingerprint': True, 'timeout': 60}) def testGetOption(self): pass # Tested in testSetOption(). @@ -172,6 +195,13 @@ def testGet(self): self.assertEqual(self.host.get('test', 1), 3) self.assertEqual(self.host.get('test2', 1), 1) + def testWithoutVariables(self): + host = Host('router1') + self.assertFalse(host.has_key('vlan')) + self.assertEqual(host.get('vlan', 10), 10) + host.set_default('vlan', 20) + self.assertEqual(host.get('vlan', 10), 20) + def suite(): return unittest.TestLoader().loadTestsFromTestCase(HostTest) From f501c7145010b03c277a8d0b54340730ab4aea1f Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 02:08:37 +0100 Subject: [PATCH 198/250] Test percent signs, empty variables and partial URLs --- tests/Exscript/util/urlTest.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/Exscript/util/urlTest.py b/tests/Exscript/util/urlTest.py index 3153fd0b..cc6cf429 100644 --- a/tests/Exscript/util/urlTest.py +++ b/tests/Exscript/util/urlTest.py @@ -88,6 +88,24 @@ def testFromString(self): self.assertIsInstance(result, Url) self.assertTrue(str(result) == expected, error) + def testPercentSignsThatEscapeNothing(self): + # They are kept, in the passwords and in the variables. + url = Url.from_string('ssh://admin:100%sure@testhost?note=50%25+of+100%') + self.assertEqual(url.password1, '100%sure') + self.assertEqual(url.vars, {'note': ['50% of 100%']}) + + def testVariablesWithoutValue(self): + url = Url.from_string('telnet://testhost?vlan=&interface=ge-0/0/1&shutdown') + self.assertEqual(url.vars, {'interface': ['ge-0/0/1']}) + + def testToStringOfSomeParts(self): + # Without protocol and user name, but with a password and a path. + url = Url() + url.hostname = 'testhost' + url.password1 = 'secret' + url.path = 'configs/r1.cfg' + self.assertEqual(str(url), ':secret@testhost/configs/r1.cfg') + def suite(): return unittest.TestLoader().loadTestsFromTestCase(urlTest) From b2d7ec78db5b4ff208b65cc486a5a4a61700e9ad Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 02:09:06 +0100 Subject: [PATCH 199/250] Show the intended message when Url.from_string() gets no string Building the message of the TypeError added a type to a string, which raised a TypeError of its own. --- CHANGES.md | 3 +++ src/Exscript/util/url.py | 2 +- tests/Exscript/util/urlTest.py | 4 ++++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 47429ddc..1ca6f977 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -630,6 +630,9 @@ The names below refer to functions callable from templates, not protocol methods a `KeyError` before Python 3.13. - **Pseudo Protocol on Windows:** `pseudo://` URLs accept absolute Windows paths such as `pseudo://C:\devices\router.py`; the drive letter was taken for a port number. +- **`Exscript.util.url.Url.from_string(None)`:** Raises a `TypeError` with the message + `Expected string but got NoneType`. Before, building the message failed, so the + `TypeError` said `can only concatenate str (not "type") to str`. - **`Exscript.util.tty.get_terminal_size()`:** Uses `shutil.get_terminal_size()`: the `LINES` and `COLUMNS` environment variables, then the terminal of the standard output, then the defaults. It no longer asks standard input, standard error and the diff --git a/src/Exscript/util/url.py b/src/Exscript/util/url.py index 7a66594f..ae408c62 100644 --- a/src/Exscript/util/url.py +++ b/src/Exscript/util/url.py @@ -191,7 +191,7 @@ def from_string(url: str, default_protocol: str = 'ssh') -> 'Url': :return: The Url object constructed from the given URL. """ if url is None: - raise TypeError('Expected string but got' + type(url)) + raise TypeError('Expected string but got ' + type(url).__name__) # Extract the protocol name from the URL. result = Url() diff --git a/tests/Exscript/util/urlTest.py b/tests/Exscript/util/urlTest.py index cc6cf429..c55e6da0 100644 --- a/tests/Exscript/util/urlTest.py +++ b/tests/Exscript/util/urlTest.py @@ -106,6 +106,10 @@ def testToStringOfSomeParts(self): url.path = 'configs/r1.cfg' self.assertEqual(str(url), ':secret@testhost/configs/r1.cfg') + def testFromStringWithoutString(self): + with self.assertRaisesRegex(TypeError, '^Expected string but got NoneType$'): + Url.from_string(None) + def suite(): return unittest.TestLoader().loadTestsFromTestCase(urlTest) From f2372d86ae7f33aab04ed6f0d7ea5bcd604ef9d0 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 02:11:28 +0100 Subject: [PATCH 200/250] Test the SSH server with a client that disappears during a command --- tests/Exscript/servers/SSHdTest.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tests/Exscript/servers/SSHdTest.py b/tests/Exscript/servers/SSHdTest.py index acb5e111..3a502650 100644 --- a/tests/Exscript/servers/SSHdTest.py +++ b/tests/Exscript/servers/SSHdTest.py @@ -133,6 +133,34 @@ def fail(command): self.assertRaises(ConnectionClosedException, conn.execute, 'fail') self.assertIn('RuntimeError in connection handler: handler failed', '\n'.join(logs.output)) + def testClientDisappearsDuringACommand(self): + # Sending the response fails, and the server ends the session. + channels = [] + + class Server(sshd._ParamikoServer): + def check_channel_shell_request(self, channel): + channels.append(channel) + return super().check_channel_shell_request(channel) + + entered = threading.Event() + + def slow(command): + entered.set() + channels[0].status_event.wait(5) # set when the client has closed the channel + return 'done' + + self.device.add_command('slow', slow) + self.start() + with mock.patch.object(sshd, '_ParamikoServer', Server): + conn = self.connect() + with self.assertLogs('Exscript.servers.sshd', logging.DEBUG) as logs: + conn.send('slow\r') + self.assertTrue(entered.wait(5)) + conn.close(force=True) + self.daemon.exit() + self.daemon.join() + self.assertIn('Client disappeared: Socket is closed', '\n'.join(logs.output)) + def testClientWithoutShell(self): server_sock, client_sock = socket.socketpair() self.daemon.running = True From 68061d298c9025e49e6e954fb13d49cb72be7c65 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 02:12:31 +0100 Subject: [PATCH 201/250] Test the argument checks of the S/Key one-time password --- tests/Exscript/util/cryptTest.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/Exscript/util/cryptTest.py b/tests/Exscript/util/cryptTest.py index e1823064..5ca0e113 100644 --- a/tests/Exscript/util/cryptTest.py +++ b/tests/Exscript/util/cryptTest.py @@ -18,6 +18,23 @@ def testOtp(self): hash = otp('pass', 'abc888', 9998) self.assertEqual('DEBT BOON ASKS ORAL MEN WEE', hash) + def testOtpWithInvalidArguments(self): + # RFC 2289: a passphrase of 4 to 63 characters, an alphanumeric seed + # of 1 to 16 characters, and a sequence number that is not negative. + from Exscript.util.crypt import otp + + for args, error in ( + (('abc', 'abc123', 99), 'passphrase length'), + (('x' * 64, 'abc123', 99), 'passphrase length'), + (('password', '', 99), 'seed length'), + (('password', 'a' * 17, 99), 'seed length'), + (('password', 'abc 123', 99), 'seed composition'), + (('password', 'abc123', -1), 'sequence'), + ): + with self.subTest(args=args): + self.assertRaisesRegex(ValueError, '^%s$' % error, otp, *args) + self.assertEqual(len(otp('x' * 63, 'A' * 16, 0).split()), 6) + def suite(): return unittest.TestLoader().loadTestsFromTestCase(cryptTest) From af94ddf3eaa8a8ecbe0df4c9faa919256fbc2168 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 02:13:58 +0100 Subject: [PATCH 202/250] Test the hooks, the dependency check and unfitting calls of the tracing --- tests/Exscript/contrib/OtelTest.py | 61 ++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/tests/Exscript/contrib/OtelTest.py b/tests/Exscript/contrib/OtelTest.py index 30c91791..8bbb9ce4 100644 --- a/tests/Exscript/contrib/OtelTest.py +++ b/tests/Exscript/contrib/OtelTest.py @@ -1,3 +1,4 @@ +import importlib.metadata import unittest from unittest import mock @@ -135,3 +136,63 @@ def testUninstrument(self): self.assertIsNot(Dummy.connect, original) ExscriptInstrumentor().uninstrument() self.assertIs(Dummy.connect, original) + + def testHooks(self): + calls = [] + + def request_hook(span, conn): + calls.append(('request', span.name)) + span.set_attribute('test.hook', 'request') + + def response_hook(span, conn, result): + calls.append(('response', span.name, result)) + + self.instrument(request_hook=request_hook, response_hook=response_hook) + conn = Dummy(device=VirtualDevice('r1', echo=False, strict=False), connect_rate_limit=0) + conn.connect('r1') + conn.close(force=True) + expected = [ + ('request', 'Dummy connect'), + ('response', 'Dummy connect', True), + ('request', 'Dummy close'), + ('response', 'Dummy close', None), + ] + self.assertEqual(calls, expected) + self.assertEqual({span.attributes['test.hook'] for span in self.spans()}, {'request'}) + + def testFailedCallGetsNoResponseHook(self): + responses = [] + self.instrument(response_hook=lambda span, conn, result: responses.append(result)) + conn = Dummy(connect_rate_limit=0) + with mock.patch.object(Dummy, '_connect', side_effect=OSError('unreachable')): + self.assertRaises(OSError, conn.connect, 'r1') + self.assertEqual(responses, []) + + def testArgumentsThatDoNotFit(self): + # The method raises the TypeError, and no span is created. + self.instrument() + conn = Dummy(connect_rate_limit=0) + self.assertRaises(TypeError, conn.execute) + self.assertRaises(TypeError, conn.connect, 'r1', 23, 'extra') + self.assertEqual(self.spans(), []) + + def testWithoutAddress(self): + # The address and the port are set only if they are known. + self.instrument() + Dummy(connect_rate_limit=0).close() + (span,) = self.spans('Dummy close') + self.assertNotIn('server.address', span.attributes) + self.assertNotIn('server.port', span.attributes) + + def testDependencyCheck(self): + # Exscript is installed, so instrument() works without skip_dep_check. + try: + importlib.metadata.version('Exscript') + except importlib.metadata.PackageNotFoundError: + self.skipTest('Exscript is not installed') + instrumentor = ExscriptInstrumentor() + self.assertEqual(instrumentor.instrumentation_dependencies(), ('exscript',)) + original = Dummy.connect + instrumentor.instrument() + self.addCleanup(instrumentor.uninstrument) + self.assertIsNot(Dummy.connect, original) From 7e1eb27a1372c46dfcafbf5778012ced22a91e39 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 02:14:50 +0100 Subject: [PATCH 203/250] Test the reports of failed actions and the options of format() --- tests/Exscript/util/reportTest.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/Exscript/util/reportTest.py b/tests/Exscript/util/reportTest.py index 06c4ee25..ee45c650 100644 --- a/tests/Exscript/util/reportTest.py +++ b/tests/Exscript/util/reportTest.py @@ -70,6 +70,15 @@ def testStatus(self): expect = '3 actions total (1 failed, 2 succeeded)' self.assertEqual(status(self.logger), expect) + def testStatusOfFailedActions(self): + from Exscript.util.report import status + + self.assertEqual(status(self.logger), 'No actions done') + self.createAbortedLog() + self.assertEqual(status(self.logger), 'One action done (failed)') + self.createAbortedLog() + self.assertEqual(status(self.logger), '2 actions total (all failed)') + def testSummarize(self): from Exscript.util.report import summarize @@ -105,6 +114,17 @@ def testFormat(self): expected = expected.replace('FakeError\n', 'util.reportTest.FakeError\n') self.assertEqual(format(self.logger), expected) + def testFormatOptions(self): + from Exscript.util.report import format + + self.createSucceededLog() + self.createAbortedLog() + errors = 'Failed actions:\n---------------\nfake2: FakeError' + successes = 'Successful actions:\n-------------------\nfake1' + self.assertEqual(format(self.logger, show_traceback=False), errors + '\n\n' + successes) + self.assertEqual(format(self.logger, show_successful=False, show_traceback=False), errors) + self.assertEqual(format(self.logger, show_errors=False), successes) + def suite(): return unittest.TestLoader().loadTestsFromTestCase(reportTest) From d2c20d98d2eee5cc488262424557217a263d2b89 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 02:15:46 +0100 Subject: [PATCH 204/250] Test the detection of the type of a key file --- tests/Exscript/PrivateKeyTest.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/tests/Exscript/PrivateKeyTest.py b/tests/Exscript/PrivateKeyTest.py index bbad3339..e3ac205f 100644 --- a/tests/Exscript/PrivateKeyTest.py +++ b/tests/Exscript/PrivateKeyTest.py @@ -1,5 +1,6 @@ import io import os +import tempfile import unittest import paramiko @@ -39,6 +40,36 @@ def testFromFile(self): self.assertEqual(self.filename, self.key.get_filename()) self.assertEqual(self.password, self.key.get_password()) + def testFromFileDetectsTheType(self): + tmpdir = self.enterContext(tempfile.TemporaryDirectory()) + ed25519_text = ( + ed25519.Ed25519PrivateKey.generate() + .private_bytes( + serialization.Encoding.PEM, + serialization.PrivateFormat.OpenSSH, + serialization.BestAvailableEncryption(self.password.encode()), + ) + .decode() + ) + for keytype, text in ( + ('rsa', key_text(paramiko.RSAKey(filename=KEYFILE), self.password)), + ('ecdsa', key_text(paramiko.ECDSAKey.generate(), self.password)), + ('ed25519', ed25519_text), + ): + with self.subTest(keytype=keytype): + filename = os.path.join(tmpdir, 'id_' + keytype) + with open(filename, 'w', encoding='utf-8') as fp: + fp.write(text) + key = PrivateKey.from_file(filename, self.password) + self.assertEqual(key.get_type(), keytype) + self.assertEqual(key.get_filename(), filename) + + filename = os.path.join(tmpdir, 'id_rsa.pub') + with open(filename, 'w', encoding='utf-8') as fp: + fp.write('ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIA admin@host\n') + with self.assertRaisesRegex(ValueError, 'not a recognized private key'): + PrivateKey.from_file(filename) + def testFromString(self): ed25519_text = ( ed25519.Ed25519PrivateKey.generate() From 02defa6e17491740f3e67c83c16a6e3c1dfd51ec Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 02:19:17 +0100 Subject: [PATCH 205/250] Test the decorators that log the sessions of a queue --- tests/Exscript/util/logTest.py | 54 ++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 tests/Exscript/util/logTest.py diff --git a/tests/Exscript/util/logTest.py b/tests/Exscript/util/logTest.py new file mode 100644 index 00000000..200e0b7a --- /dev/null +++ b/tests/Exscript/util/logTest.py @@ -0,0 +1,54 @@ +import os +import tempfile +import unittest + +import Exscript.util.log +from Exscript import Account, Host, Logger, Queue +from Exscript.util.decorator import autologin +from Exscript.util.log import log_to, log_to_file + + +class logTest(unittest.TestCase): + CORRELATE = Exscript.util.log + + def runQueue(self, function, hosts): + queue = Queue(verbose=0, max_threads=1) + self.addCleanup(queue.destroy) + queue.add_account(Account('admin', 'secret')) + queue.run(hosts, autologin()(function)) + queue.shutdown() + + def testLogTo(self): + # The decorator labels the function, and the queue logs each host. + logger = Logger() + + def show_version(job, host, conn): + conn.execute('show version') + + self.assertIs(log_to(logger)(show_version), show_version) + self.runQueue(show_version, [Host('dummy://r1'), Host('dummy://r2')]) + logs = sorted(logger.get_succeeded_logs(), key=lambda log: log.get_name()) + self.assertEqual([log.get_name() for log in logs], ['r1', 'r2']) + self.assertIn('show version', str(logs[0])) + + def testLogToFile(self): + # The directory is created, and each host gets its log file. + logdir = os.path.join(self.enterContext(tempfile.TemporaryDirectory()), 'logs') + + def show_version(job, host, conn): + conn.execute('show version') + + log_to_file(logdir)(show_version) + self.assertEqual(os.listdir(logdir), []) + self.runQueue(show_version, [Host('dummy://r1')]) + self.assertEqual(os.listdir(logdir), ['r1.log']) + with open(os.path.join(logdir, 'r1.log'), encoding='utf-8') as fp: + self.assertIn('show version', fp.read()) + + +def suite(): + return unittest.TestLoader().loadTestsFromTestCase(logTest) + + +if __name__ == '__main__': + unittest.TextTestRunner(verbosity=2).run(suite()) From 4ca010313c22910eec232e8ca23b7d8f7d9a7b9d Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 02:24:27 +0100 Subject: [PATCH 206/250] Test the matching of the response of a connection --- tests/Exscript/util/matchTest.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/tests/Exscript/util/matchTest.py b/tests/Exscript/util/matchTest.py index 231c2982..169bbc0d 100644 --- a/tests/Exscript/util/matchTest.py +++ b/tests/Exscript/util/matchTest.py @@ -1,6 +1,15 @@ import unittest import Exscript.util.match +from Exscript.emulators import VirtualDevice +from Exscript.protocols import Dummy + +IP_INTERFACES = ''' +Interface IP-Address OK? Method Status Protocol +Ethernet0 10.0.0.1 YES NVRAM up up +Loopback0 10.0.255.1 YES NVRAM up up +Serial0 unassigned YES unset down down +'''.replace('\n', '\r\n') class matchTest(unittest.TestCase): @@ -34,6 +43,20 @@ def testAnyMatch(self): expected = [('one', 'uno'), ('two', 'due')] self.assertEqual(any_match(string, r'(\S+) (\S+)'), expected) + def testMatchTheResponseOfAConnection(self): + from Exscript.util.match import any_match, first_match + + device = VirtualDevice('r1', login_type=VirtualDevice.LOGIN_TYPE_NONE) + device.add_command('show ip int brief', IP_INTERFACES) + conn = Dummy(device=device) + conn.connect('r1') + self.addCleanup(conn.close, force=True) + conn.expect_prompt() + conn.execute('show ip int brief') + self.assertEqual(first_match(conn, r'^(\S+)\s+10\.0\.0\.1\b'), 'Ethernet0') + expected = [('Ethernet0', 'up'), ('Loopback0', 'up'), ('Serial0', 'down')] + self.assertEqual(any_match(conn, r'^(\S+)\s+\S+\s+\S+\s+\S+\s+(up|down)'), expected) + def suite(): return unittest.TestLoader().loadTestsFromTestCase(matchTest) From f648e00b7a94bc31e90a963d08b568081a06a07d Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 02:24:27 +0100 Subject: [PATCH 207/250] Test disconnecting a function that is not connected to an event --- tests/Exscript/util/eventTest.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/Exscript/util/eventTest.py b/tests/Exscript/util/eventTest.py index 337a1a8f..2a2817ce 100644 --- a/tests/Exscript/util/eventTest.py +++ b/tests/Exscript/util/eventTest.py @@ -105,6 +105,12 @@ def testDisconnect(self): self.event.disconnect(self.callback2) self.assertEqual(self.event.n_subscribers(), 0) + def testDisconnectAFunctionThatIsNotConnected(self): + self.event.listen(self.callback) + self.event.disconnect(self.callback2) + self.assertTrue(self.event.is_connected(self.callback)) + self.assertEqual(self.event.n_subscribers(), 1) + def testDisconnectAll(self): self.assertEqual(self.event.n_subscribers(), 0) self.event.connect(self.callback) From 366067836c77c64c01aee0fb4813f2790f44e53d Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 02:24:27 +0100 Subject: [PATCH 208/250] Test the version of a source tree without an installation --- tests/Exscript/VersionTest.py | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 tests/Exscript/VersionTest.py diff --git a/tests/Exscript/VersionTest.py b/tests/Exscript/VersionTest.py new file mode 100644 index 00000000..5c471a09 --- /dev/null +++ b/tests/Exscript/VersionTest.py @@ -0,0 +1,34 @@ +import importlib +import importlib.metadata +import unittest +from unittest import mock + +import Exscript +import Exscript.version + + +class VersionTest(unittest.TestCase): + CORRELATE = Exscript.version + + def testInstalled(self): + try: + installed = importlib.metadata.version('Exscript') + except importlib.metadata.PackageNotFoundError: + self.skipTest('Exscript is not installed') + self.assertEqual(Exscript.version.__version__, installed) + self.assertEqual(Exscript.__version__, installed) + + def testSourceTreeWithoutInstallation(self): + self.addCleanup(importlib.reload, Exscript.version) + error = importlib.metadata.PackageNotFoundError('Exscript') + with mock.patch('importlib.metadata.version', side_effect=error): + importlib.reload(Exscript.version) + self.assertEqual(Exscript.version.__version__, 'DEVELOPMENT') + + +def suite(): + return unittest.TestLoader().loadTestsFromTestCase(VersionTest) + + +if __name__ == '__main__': + unittest.TextTestRunner(verbosity=2).run(suite()) From 14806c2420c815b90ea0fea4a83f9e4d0ae18b7c Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 02:38:56 +0100 Subject: [PATCH 209/250] Ignore the coverage data files of subprocesses pytest-cov combines them, but multiprocessing's resource tracker ends only after pytest and leaves an empty one behind. --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 1a48fb71..7d9a1c44 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ .coverage +.coverage.* .mypy_cache .tox .venv From 7709c226ae0df3b1df20b2dddf8956ddd066b978 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 02:39:15 +0100 Subject: [PATCH 210/250] Raise the minimum test coverage in CI to 97 % The tests cover 98.6 % now; the margin allows for the small differences between the Python versions. --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 74d9341d..2448dfc9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -56,9 +56,9 @@ jobs: env: PYTHON: ${{ matrix.python }} PYTHON_GIL: ${{ matrix.gil }} # empty, i.e. ignored, except for free-threading - # Line and branch coverage: 94.2 % on 2026-11-10. Raise the floor, never lower it. + # Line and branch coverage: 98.6 % on 2026-11-11. Raise the floor, never lower it. # Windows and macOS skip the tests of the POSIX terminal, so it applies on Ubuntu only. - COV_FAIL_UNDER: ${{ matrix.os == 'ubuntu-latest' && '93' || '0' }} + COV_FAIL_UNDER: ${{ matrix.os == 'ubuntu-latest' && '97' || '0' }} run: >- uv run --frozen --python "$PYTHON" pytest --cov=Exscript --cov-report=term-missing:skip-covered --cov-fail-under="$COV_FAIL_UNDER" From 445fdd3ea0974b7a66126a665064708de87de2ad Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 02:47:07 +0100 Subject: [PATCH 211/250] Recognize the NX-OS login prompt of Telnet and the console At the console and over Telnet, NX-OS asks for the user name with " login: ", so a login with the nxos driver waited until the timeout. The driver now accepts it, as upstream PR #178 proposed. --- CHANGES.md | 4 ++ src/Exscript/protocols/drivers/nxos.py | 13 +++-- tests/Exscript/protocols/NXOSDriverTest.py | 64 ++++++++++++++++++++++ 3 files changed, 77 insertions(+), 4 deletions(-) create mode 100644 tests/Exscript/protocols/NXOSDriverTest.py diff --git a/CHANGES.md b/CHANGES.md index 1ca6f977..61522083 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -406,6 +406,10 @@ development versions; "before" refers to them. banner is now part of the test corpus. - **NX-OS Banners:** The `nxos` driver also recognizes the banner "Cisco NX-OS Software" of newer versions, e.g. nx-osv9000 (upstream issue #200). +- **NX-OS Login Prompt:** The `nxos` driver recognizes the user name prompt that NX-OS + shows at the console and over Telnet, e.g. `switch login: `, or `login: ` without the + host name, besides `Username:` (upstream PR #178). Before, a login with + `driver='nxos'` waited for `Username:` until the timeout. - **NX-OS Invalid Commands:** The `nxos` driver recognizes the error message `% Invalid command at '^' marker.` of NX-OS, so `execute()` raises an `InvalidCommandException` for such a command. Before, it returned the response as diff --git a/src/Exscript/protocols/drivers/nxos.py b/src/Exscript/protocols/drivers/nxos.py index dae827dd..a1396c6a 100644 --- a/src/Exscript/protocols/drivers/nxos.py +++ b/src/Exscript/protocols/drivers/nxos.py @@ -34,7 +34,11 @@ from ...util.impl import _Context from ..protocol import Protocol -_user_re = [re.compile(r'user ?name: ?$', re.I)] +# The user name prompt of Telnet and the console is " login: ", or +# "login: " without the hostname. A read may end after "Last login: ", which +# is no prompt. +_login_re = re.compile(r'(?:^|[\r\n])(?![Ll]ast )[^\s:]* ?login: ?$') +_user_re = [re.compile(r'user ?name: ?$', re.I), _login_re] _password_re = [re.compile(r'(?:(?:\r\n|[\r\n])Password: ?|last resort password:)$')] _prompt_re = [re.compile(r'(?:\r\n\x00?|[\r\n\x00])[\-\w+\.:/]+(?:\([^\)]+\))?[>#] ?$')] _error_re = [ @@ -55,9 +59,10 @@ class NXOSDriver(Driver): def __init__(self) -> None: """ - Constructor. Sets the patterns of the user name prompt "Username:", - the password prompts, the command prompt and the error messages, such - as "%Error", "Invalid input" and "Invalid command". + Constructor. Sets the patterns of the user name prompts + " login:" (also without the hostname) and "Username:", the + password prompts, the command prompt and the error messages, such as + "%Error" and "Invalid input". """ Driver.__init__(self, 'nxos') self.user_re = _user_re diff --git a/tests/Exscript/protocols/NXOSDriverTest.py b/tests/Exscript/protocols/NXOSDriverTest.py new file mode 100644 index 00000000..0fd2cee8 --- /dev/null +++ b/tests/Exscript/protocols/NXOSDriverTest.py @@ -0,0 +1,64 @@ +""" +Tests of the nxos driver with an emulated NX-OS switch. +""" + +import unittest + +from Exscript import Account +from Exscript.emulators import VirtualDevice +from Exscript.protocols import Dummy +from Exscript.protocols.drivers import driver_map + + +def nxos_device(user_prompt='switch login: '): + """ + Returns a switch that shows the default banner and the login prompt of + NX-OS 7.x to 10.x at the console and over Telnet. + """ + device = VirtualDevice('switch', banner='User Access Verification\r\n', strict=False) + device.user_prompt = user_prompt + device.prompt = 'switch# ' + device.add_command('show clock', '12:00:00.000 UTC Wed Nov 11 2026') + return device + + +class LoginTest(unittest.TestCase): + """Logins at the console and over Telnet.""" + + def login(self, device, driver=None): + conn = Dummy(device=device, driver=driver, timeout=1) + self.addCleanup(conn.close, force=True) + conn.connect('switch') + conn.login(Account('admin', 'secret', needs_lock=False)) + return conn + + def testLoginPrompt(self): + # Before, the driver waited for "Username:" until the timeout (upstream PR #178). + for prompt in 'switch login: ', 'login: ', ' login: ', 'Username: ': + with self.subTest(prompt=prompt): + conn = self.login(nxos_device(prompt), driver='nxos') + conn.execute('show clock') + self.assertEqual(conn.get_response(), '12:00:00.000 UTC Wed Nov 11 2026') + + def testUserPromptPatterns(self): + def matches(text): + return any(regex.search(text) for regex in driver_map['nxos'].user_re) + + for text in ( + '\r\nRackX_XXX login: ', + '\r\nIP_switch_A_1 login: ', + 'login: ', + '\r\nUsername:', + ): + self.assertTrue(matches(text), repr(text)) + # A read may end after "Last login: ". + for text in '\r\nLast login: ', '\r\nswitch login: admin\r\nPassword: ': + self.assertFalse(matches(text), repr(text)) + + +def suite(): + return unittest.TestLoader().loadTestsFromTestCase(LoginTest) + + +if __name__ == '__main__': + unittest.TextTestRunner(verbosity=2).run(suite()) From 0faedf2b4e5a89f2938808727bfaffdc1d219d44 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 02:51:45 +0100 Subject: [PATCH 212/250] Guess NX-OS at the login prompt of Telnet and the console At the console and over Telnet, NX-OS shows the banner of IOS, so the guess was ios, and the login timed out at the prompt " login: ". The guess is now nxos if the banner ends with this prompt. The transcript of upstream PR #178 is added to the test corpus. --- CHANGES.md | 6 ++ src/Exscript/protocols/drivers/nxos.py | 7 ++- tests/Exscript/protocols/NXOSDriverTest.py | 19 ++++++ tests/Exscript/protocols/OsGuesserTest.py | 15 +++++ tests/Exscript/protocols/banners/nxos.2 | 25 ++++++++ .../Exscript/protocols/osguess-baseline.json | 60 +++++++++++++++++++ tools/osguess_report.py | 2 +- 7 files changed, 132 insertions(+), 2 deletions(-) create mode 100644 tests/Exscript/protocols/banners/nxos.2 diff --git a/CHANGES.md b/CHANGES.md index 61522083..ae9592aa 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -410,6 +410,12 @@ development versions; "before" refers to them. shows at the console and over Telnet, e.g. `switch login: `, or `login: ` without the host name, besides `Username:` (upstream PR #178). Before, a login with `driver='nxos'` waited for `Username:` until the timeout. +- **NX-OS Guessed at the Login Prompt:** A login to NX-OS at the console or over Telnet + is guessed as `nxos` at the user name prompt: NX-OS shows the banner "User Access + Verification" of IOS, too, but then asks e.g. `switch login: ` instead of + `Username:`. Before, the guess was `ios`, whose patterns do not match this prompt, so + the login waited until the timeout. The transcript of upstream PR #178 is now part of + the test corpus. - **NX-OS Invalid Commands:** The `nxos` driver recognizes the error message `% Invalid command at '^' marker.` of NX-OS, so `execute()` raises an `InvalidCommandException` for such a command. Before, it returned the response as diff --git a/src/Exscript/protocols/drivers/nxos.py b/src/Exscript/protocols/drivers/nxos.py index a1396c6a..b45c3e43 100644 --- a/src/Exscript/protocols/drivers/nxos.py +++ b/src/Exscript/protocols/drivers/nxos.py @@ -74,12 +74,17 @@ def check_head_for_os(self, string: str) -> int: """ Returns 95 if the data contains "Cisco Nexus Operating System (NX-OS) Software" or, as newer - versions print, "Cisco NX-OS Software", else 0. + versions print, "Cisco NX-OS Software", else 70 if it contains + "User Access Verification" and ends with the user name prompt of + Telnet and the console, e.g. "switch login: ", where IOS asks for + "Username:", else 0. """ if 'Cisco Nexus Operating System (NX-OS) Software' in string: return 95 if 'Cisco NX-OS Software' in string: return 95 + if 'User Access Verification' in string and _login_re.search(string): + return 70 return 0 def init_terminal(self, conn: 'Protocol') -> None: diff --git a/tests/Exscript/protocols/NXOSDriverTest.py b/tests/Exscript/protocols/NXOSDriverTest.py index 0fd2cee8..53bb1aad 100644 --- a/tests/Exscript/protocols/NXOSDriverTest.py +++ b/tests/Exscript/protocols/NXOSDriverTest.py @@ -9,6 +9,8 @@ from Exscript.protocols import Dummy from Exscript.protocols.drivers import driver_map +from .ExpectTest import connect + def nxos_device(user_prompt='switch login: '): """ @@ -55,6 +57,23 @@ def matches(text): for text in '\r\nLast login: ', '\r\nswitch login: admin\r\nPassword: ': self.assertFalse(matches(text), repr(text)) + def testGuessAtTheLoginPrompt(self): + # IOS shows the banner, too, but asks for "Username:". Before, the guess was + # ios, whose patterns do not match the prompt, and the login timed out. + conn = self.login(nxos_device()) + self.assertEqual(conn.guess_os(), 'nxos') + conn.execute('show clock') + self.assertEqual(conn.get_response(), '12:00:00.000 UTC Wed Nov 11 2026') + + def testGuessInSmallReads(self): + # Over Telnet, e.g. one character per read: the guess is ios until the prompt. + head = '\r\nUser Access Verification\r\nswitch login: ' + conn = connect([*head, 'admin\r\nPassword: ', '\r\nswitch# ']) + self.addCleanup(conn.close, force=True) + conn.login(Account('admin', 'secret', needs_lock=False)) + self.assertEqual(conn.sent, ['admin\r', 'secret\r']) # no "enable" as for ios + self.assertEqual(conn.guess_os(), 'nxos') + def suite(): return unittest.TestLoader().loadTestsFromTestCase(LoginTest) diff --git a/tests/Exscript/protocols/OsGuesserTest.py b/tests/Exscript/protocols/OsGuesserTest.py index 3be9ed03..4af4d6ad 100644 --- a/tests/Exscript/protocols/OsGuesserTest.py +++ b/tests/Exscript/protocols/OsGuesserTest.py @@ -155,6 +155,21 @@ def testNxosBanner(self): banner = 'Cisco Nexus Operating System (NX-OS) Software\r\n' self.assertEqual(guess(head=banner), 'nxos') + def testNxosLoginPrompt(self): + # IOS shows this banner, too, but asks for "Username:"; NX-OS asks so at the + # console and over Telnet. The guesser gets one read or many small ones. + head = '\r\nUser Access Verification\r\nswitch login: ' + self.assertEqual(drivers.driver_map['nxos'].check_head_for_os(head), 70) + self.assertEqual(guess(head=head), 'nxos') + guesser = OsGuesser() + guesser.check_received_data(head, False) + self.assertEqual(guesser.get('os'), 'nxos') + self.assertEqual(guess(head='\r\nUser Access Verification\r\nlogin: '), 'nxos') + self.assertEqual(guess(head='\r\nUser Access Verification\r\n\r\nUsername: '), 'ios') + # A read may end after "Last login: ", e.g. of a Linux host with this banner. + last_login = 'User Access Verification\r\nLast login: Mon Oct 26 10:00:00 2026\r\n' + self.assertNotEqual(guess(head=last_login), 'nxos') + def testIsamPrompt(self): self.assertEqual(guess(response='\r\ninfo\r\ntyp:isadmin>#'), 'isam') for prompt in 'router#', 'switch>', 'hostname >', 'S>* 10.0.0.0/8 is directly connected': diff --git a/tests/Exscript/protocols/banners/nxos.2 b/tests/Exscript/protocols/banners/nxos.2 new file mode 100644 index 00000000..848282bf --- /dev/null +++ b/tests/Exscript/protocols/banners/nxos.2 @@ -0,0 +1,25 @@ + +User Access Verification +switch login: admin +Password: +Bad terminal type: "linux". Will assume vt100. +Cisco Nexus Operating System (NX-OS) Software +TAC support: http://www.cisco.com/tac +Copyright (C) 2002-2015, Cisco and/or its affiliates. +All rights reserved. +The copyrights to certain works contained in this software are +owned by other third parties and used and distributed under their own +licenses, such as open source. This software is provided "as is," and unless +otherwise stated, there is no warranty, express or implied, including but not +limited to warranties of merchantability and fitness for a particular purpose. +Certain components of this software are licensed under +the GNU General Public License (GPL) version 2.0 or +GNU General Public License (GPL) version 3.0 or the GNU +Lesser General Public License (LGPL) Version 2.1 or +Lesser General Public License (LGPL) Version 2.0. +A copy of each such license is available at +http://www.opensource.org/licenses/gpl-2.0.php and +http://opensource.org/licenses/gpl-3.0.html and +http://www.opensource.org/licenses/lgpl-2.1.php and +http://www.gnu.org/licenses/old-licenses/library.txt. +switch# diff --git a/tests/Exscript/protocols/osguess-baseline.json b/tests/Exscript/protocols/osguess-baseline.json index e5ee38ee..7bfa0d2d 100644 --- a/tests/Exscript/protocols/osguess-baseline.json +++ b/tests/Exscript/protocols/osguess-baseline.json @@ -2879,6 +2879,66 @@ 95 ] }, + "nxos.2|-|64": { + "after_auth": [ + "nxos", + 95 + ], + "final": [ + "nxos", + 95 + ] + }, + "nxos.2|-|65536": { + "after_auth": [ + "nxos", + 95 + ], + "final": [ + "nxos", + 95 + ] + }, + "nxos.2|SSH-2.0-Cisco-1.25|64": { + "after_auth": [ + "nxos", + 95 + ], + "final": [ + "nxos", + 95 + ] + }, + "nxos.2|SSH-2.0-Cisco-1.25|65536": { + "after_auth": [ + "nxos", + 95 + ], + "final": [ + "nxos", + 95 + ] + }, + "nxos.2|SSH-2.0-OpenSSH_9.6|64": { + "after_auth": [ + "nxos", + 95 + ], + "final": [ + "nxos", + 95 + ] + }, + "nxos.2|SSH-2.0-OpenSSH_9.6|65536": { + "after_auth": [ + "nxos", + 95 + ], + "final": [ + "nxos", + 95 + ] + }, "one_os.1|-|64": { "after_auth": [ "one_os", diff --git a/tools/osguess_report.py b/tools/osguess_report.py index 74bbd5a5..7e2a3b39 100644 --- a/tools/osguess_report.py +++ b/tools/osguess_report.py @@ -28,7 +28,7 @@ from tests.Exscript.protocols import ReplayTest as replays # noqa: E402 from tests.Exscript.protocols.replay import Recording # noqa: E402 -MIN_CORRECT = 217 # 2026-11-10; raise it when a change guesses more sessions right +MIN_CORRECT = 221 # raise it when a change guesses more sessions right def banner_sessions(): From d81798065d582695ffde4aa96dfd62561b3471a1 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 02:56:13 +0100 Subject: [PATCH 213/250] Recognize the error messages of NX-OS execute() raised no error for many commands that NX-OS rejected. The nxos driver now also recognizes the parser errors, which end with "at '^' marker.", "% Permission denied for the role", and lines that start with "ERROR: " or "Error: ". --- CHANGES.md | 9 ++++ docs/migration.rst | 8 +++ src/Exscript/protocols/drivers/nxos.py | 11 +++- tests/Exscript/protocols/NXOSDriverTest.py | 59 +++++++++++++++++++++- 4 files changed, 85 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index ae9592aa..ce8c6632 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -416,6 +416,15 @@ development versions; "before" refers to them. `Username:`. Before, the guess was `ios`, whose patterns do not match this prompt, so the login waited until the timeout. The transcript of upstream PR #178 is now part of the test corpus. +- **BREAKING - NX-OS Error Messages:** The `nxos` driver recognizes more error messages + of NX-OS, so `execute()` raises an `InvalidCommandException` for the commands that + NX-OS rejects with them: all parser errors, which end with `at '^' marker.`, e.g. + `% Invalid command at '^' marker.` or `Invalid range at '^' marker.`, then + `% Permission denied for the role`, and lines that start with `ERROR: ` or `Error: `, + e.g. `ERROR: port already in a port-channel, no config allowed.`. Before, it knew + only the messages that IOS shows, too, such as `% Incomplete command`, and returned + the response of the other commands as if they had succeeded. To migrate, catch the + exception where a command may fail. - **NX-OS Invalid Commands:** The `nxos` driver recognizes the error message `% Invalid command at '^' marker.` of NX-OS, so `execute()` raises an `InvalidCommandException` for such a command. Before, it returned the response as diff --git a/docs/migration.rst b/docs/migration.rst index 55bb55f6..d0141153 100644 --- a/docs/migration.rst +++ b/docs/migration.rst @@ -95,6 +95,14 @@ Driver names option of a ``Host`` and ``--use-driver``. 2.x ignored it and guessed the driver. +NX-OS errors + The ``nxos`` driver recognizes more error messages of NX-OS, e.g. + ``% Invalid command at '^' marker.`` and lines that start with + ``ERROR:``, so ``execute()`` raises + :class:`~Exscript.protocols.exception.InvalidCommandException` for the + commands that NX-OS rejects with them. 2.x returned their response. + Catch the exception where a command may fail. + Templates --------- diff --git a/src/Exscript/protocols/drivers/nxos.py b/src/Exscript/protocols/drivers/nxos.py index b45c3e43..9788acfc 100644 --- a/src/Exscript/protocols/drivers/nxos.py +++ b/src/Exscript/protocols/drivers/nxos.py @@ -47,6 +47,13 @@ re.compile(r'(?:incomplete|ambiguous) command', re.I), re.compile(r'connection timed out', re.I), re.compile(r'[^\r\n]+ not found', re.I), + # The parser errors, e.g. "% Invalid command at '^' marker." or "Invalid + # range at '^' marker.", after a line with a caret under the wrong word. + re.compile(r"at '\^' marker\.\s*$"), + re.compile(r'^% Permission denied for the role'), + # Configuration errors, e.g. "ERROR: port already in a port-channel, no + # config allowed." or "Error: AAA authorization failed ...". + re.compile(r'^(?:ERROR|Error): '), ] @@ -62,7 +69,9 @@ def __init__(self) -> None: Constructor. Sets the patterns of the user name prompts " login:" (also without the hostname) and "Username:", the password prompts, the command prompt and the error messages, such as - "%Error" and "Invalid input". + "%Error", "Invalid input", the parser errors that end with + "at '^' marker.", "% Permission denied for the role", and lines that + start with "ERROR: " or "Error: ". """ Driver.__init__(self, 'nxos') self.user_re = _user_re diff --git a/tests/Exscript/protocols/NXOSDriverTest.py b/tests/Exscript/protocols/NXOSDriverTest.py index 53bb1aad..7b6da335 100644 --- a/tests/Exscript/protocols/NXOSDriverTest.py +++ b/tests/Exscript/protocols/NXOSDriverTest.py @@ -8,6 +8,7 @@ from Exscript.emulators import VirtualDevice from Exscript.protocols import Dummy from Exscript.protocols.drivers import driver_map +from Exscript.protocols.exception import InvalidCommandException from .ExpectTest import connect @@ -75,8 +76,64 @@ def testGuessInSmallReads(self): self.assertEqual(conn.guess_os(), 'nxos') +class ErrorTest(unittest.TestCase): + """The error messages of NX-OS in the response of a command.""" + + def setUp(self): + self.output = '' + device = nxos_device() + device.add_command('command', lambda command: self.output) + self.conn = Dummy(device=device, driver='nxos', timeout=1) + self.addCleanup(self.conn.close, force=True) + self.conn.connect('switch') + self.conn.login(Account('admin', 'secret', needs_lock=False)) + + def testErrors(self): + # The parser errors follow a line with a caret under the wrong word. + outputs = [ + " ^\r\n% Invalid command at '^' marker.", + " ^\r\n% Incomplete command at '^' marker.", + " ^\r\n% Ambiguous command at '^' marker.", + " ^\r\n% Invalid number, range is (4096:16777215) at '^' marker.", + " ^\r\n% Invalid parameter detected at '^' marker.", + " ^\r\nInvalid range at '^' marker.", + " ^\r\nInvalid interface format at '^' marker.", + '% Permission denied for the role', + 'Error: AAA authorization failed AAA_AUTHOR_STATUS_METHOD=16(0x10)', + 'ERROR: port already in a port-channel, no config allowed.', + 'Error: default interface is not supported as speed-group is configured', + ] + for output in outputs: + with self.subTest(output=output): + self.output = output + with self.assertRaises(InvalidCommandException) as cm: + self.conn.execute('command') + line = output.split('\r\n')[-1] + self.assertEqual( + str(cm.exception), 'Error in the response from the device: ' + line + ) + + def testNoErrors(self): + outputs = [ + ' 0 input error 0 short frame', + 'Errors: 0', + 'Ethernet1/1 is down (error-disabled)', + '2026 Nov 11 10:00:00 switch %ETHPORT-5-IF_DOWN_ERROR_DISABLED: Interface' + ' Ethernet1/1 is down (Error disabled. Reason:BPDUGuard)', + ' description ERROR: spare port, do not use', + ] + for output in outputs: + with self.subTest(output=output): + self.output = output + self.conn.execute('command') + self.assertEqual(self.conn.get_response(), output) + + def suite(): - return unittest.TestLoader().loadTestsFromTestCase(LoginTest) + loader = unittest.TestLoader() + return unittest.TestSuite( + [loader.loadTestsFromTestCase(LoginTest), loader.loadTestsFromTestCase(ErrorTest)] + ) if __name__ == '__main__': From 63eb557d243ef0ca97f96c85bc2c5f1e4cc5ba52 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 03:01:53 +0100 Subject: [PATCH 214/250] Recognize NX-OS prompts with more than one mode In maintenance mode and in the boot image, NX-OS shows prompts such as switch(maint-mode)(config)#, so execute() waited for them until the timeout. The modes after the first contain no line break or NUL, so that the prompts that matched before keep their position. --- CHANGES.md | 4 ++ src/Exscript/protocols/drivers/nxos.py | 11 +++-- tests/Exscript/protocols/PromptRegexTest.py | 53 +++++++++++++++++++++ 3 files changed, 65 insertions(+), 3 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index ce8c6632..9d1e15ff 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -425,6 +425,10 @@ development versions; "before" refers to them. only the messages that IOS shows, too, such as `% Incomplete command`, and returned the response of the other commands as if they had succeeded. To migrate, catch the exception where a command may fail. +- **NX-OS Prompts With Two Modes:** The `nxos` driver recognizes prompts with more than + one mode in parentheses, e.g. `switch(maint-mode)(config)# ` in maintenance mode or + `switch(boot)(config)# ` in the boot image. Before, a command that entered such a + mode waited for the prompt until the timeout. - **NX-OS Invalid Commands:** The `nxos` driver recognizes the error message `% Invalid command at '^' marker.` of NX-OS, so `execute()` raises an `InvalidCommandException` for such a command. Before, it returned the response as diff --git a/src/Exscript/protocols/drivers/nxos.py b/src/Exscript/protocols/drivers/nxos.py index 9788acfc..a4bc4347 100644 --- a/src/Exscript/protocols/drivers/nxos.py +++ b/src/Exscript/protocols/drivers/nxos.py @@ -40,7 +40,12 @@ _login_re = re.compile(r'(?:^|[\r\n])(?![Ll]ast )[^\s:]* ?login: ?$') _user_re = [re.compile(r'user ?name: ?$', re.I), _login_re] _password_re = [re.compile(r'(?:(?:\r\n|[\r\n])Password: ?|last resort password:)$')] -_prompt_re = [re.compile(r'(?:\r\n\x00?|[\r\n\x00])[\-\w+\.:/]+(?:\([^\)]+\))?[>#] ?$')] +# A name and modes in parentheses, e.g. "switch(maint-mode)(config)#". The +# modes after the first one contain none of the characters before a prompt, so +# each prompt that the pattern matched with one mode keeps its position. +_prompt_re = [ + re.compile(r'(?:\r\n\x00?|[\r\n\x00])[\-\w+\.:/]+(?:\([^\)]+\))?(?:\([^\)\r\n\x00]+\))*[>#] ?$') +] _error_re = [ re.compile(r'%Error'), re.compile(r'invalid (?:input|command)', re.I), @@ -60,8 +65,8 @@ class NXOSDriver(Driver): """ The ``nxos`` driver for Cisco Nexus OS (NXOS). Its prompt is a name with - an optional mode in parentheses, followed by ">" or "#", e.g. - "hostname#". + optional modes in parentheses, followed by ">" or "#", e.g. "switch#", + "switch(config-if)#" or "switch(maint-mode)(config)#". """ def __init__(self) -> None: diff --git a/tests/Exscript/protocols/PromptRegexTest.py b/tests/Exscript/protocols/PromptRegexTest.py index f1a5f6d1..35e21d61 100644 --- a/tests/Exscript/protocols/PromptRegexTest.py +++ b/tests/Exscript/protocols/PromptRegexTest.py @@ -38,6 +38,8 @@ def span(regex, text): _JUNOS_PREFIX + r'(?:\r\n|[\r\n])+[\w\-\.]+@[\-\w+\.:]++(?:[~/][\w\-\./]*+ ?)?[%>#] $' ) MRV_BEFORE = re.compile(r'(?:\r\n|[\r\n])+[^$\r\n]+\$ ?$') +# A reference pattern for the NX-OS prompt, with at most one mode in parentheses. +NXOS_BEFORE = re.compile(r'(?:\r\n\x00?|[\r\n\x00])[\-\w+\.:/]+(?:\([^\)]+\))?[>#] ?$') def random_texts(alphabet, count, seed): @@ -109,6 +111,57 @@ def testJunosPrompts(self): if JUNOS_2X.search(text): self.assertEqual(span(JUNOS_2X, text), span(regex, text), repr(text)) + def testNxosPrompts(self): + regex = driver_map['nxos'].prompt_re[0] + prompts = [ + # Exec mode, also of a network-operator, and after a CR. + 'switch# ', + 'nx1#', + '\rTEST-12345# ', + # Configuration mode and its sub-modes. + 'switch(config)# ', + 'switch(config-if)#', + 'switch(config-subif)#', + 'switch(config-vrf)#', + 'switch(config-if-range)#', + 'switch(config-vlan)#', + 'switch(config-router)#', + 'switch(config-router-af)#', + 'switch(config-router-neighbor)#', + 'switch(config-router-neighbor-af)#', + 'switch(config-route-map)#', + 'N7K-1(config-vpc-domain)# ', + 'switch(config-console)#', + 'switch(config-line)#', + 'switch(config-applet)#', + 'switch(config-tacacs+)# ', + 'switch(config-s)# ', + 'switch(config-s-acl)# ', + 'switch(config-dual-stage)#', + # Maintenance mode, standby supervisor, boot image, routing context, Tcl. + 'switch(maint-mode)# ', + 'switch(maint-mode)(config)# ', + 'switch(standby)# ', + 'switch(boot)# ', + 'switch(boot)(config)# ', + 'switch-red#', + 'switch-tcl# ', + 'switch(config-tcl)#', + 'switch(config-if-tcl)#', + ] + for prompt in prompts: + self.assertTrue(regex.search('output\r\n' + prompt), repr(prompt)) + for text in 'switch(config)', 'switch (config)# ', 'line without prompt': + self.assertIsNone(regex.search('output\r\n' + text), repr(text)) + + # The pattern matches everything that NXOS_BEFORE matches, at the same position. + tokens = ['x', 'x(y)', '(', ')', '#', '\r\n', '\n', '\x00'] + for length in range(1, 6): + for chars in itertools.product(tokens, repeat=length): + text = '\r\n' + ''.join(chars) + if NXOS_BEFORE.search(text): + self.assertEqual(span(NXOS_BEFORE, text), span(regex, text), repr(text)) + def testRunsOfLineBreaksAsBefore(self): # The patterns match each run of line breaks as a whole, and the same texts at the # same positions as JUNOS_BEFORE and MRV_BEFORE, which backtrack on the runs. From 2146d4b1fc2ae365b4e62287a78b3e6b8db5340d Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 03:06:10 +0100 Subject: [PATCH 215/250] Guess NX-OS by its NETCONF capabilities Such sessions were guessed as unknown. The nxos driver now recognizes the capability of the NX-OS device model in the hello of the server. --- CHANGES.md | 6 +++ docs/netconf.rst | 8 +-- src/Exscript/protocols/drivers/nxos.py | 17 +++++-- tests/Exscript/protocols/NXOSDriverTest.py | 58 +++++++++++++++++++++- tests/Exscript/protocols/OsGuesserTest.py | 24 +++++++++ 5 files changed, 105 insertions(+), 8 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 9d1e15ff..62007c3a 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -429,6 +429,12 @@ development versions; "before" refers to them. one mode in parentheses, e.g. `switch(maint-mode)(config)# ` in maintenance mode or `switch(boot)(config)# ` in the boot image. Before, a command that entered such a mode waited for the prompt until the timeout. +- **NX-OS Over NETCONF:** A NETCONF session with NX-OS is guessed as `nxos` by the + capability of its device model, `http://cisco.com/ns/yang/cisco-nx-os-device`, as + sessions with IOS XR and Junos are guessed by theirs. Before, the guess was + `unknown`. The driver keeps ncclient's default device handler: the `nexus` handler + replaces the base capability of RFC 6241 in the hello of the client with the one of + the old XML agent, which a server may reject. - **NX-OS Invalid Commands:** The `nxos` driver recognizes the error message `% Invalid command at '^' marker.` of NX-OS, so `execute()` raises an `InvalidCommandException` for such a command. Before, it returned the response as diff --git a/docs/netconf.rst b/docs/netconf.rst index 9d7cc586..c76c900c 100644 --- a/docs/netconf.rst +++ b/docs/netconf.rst @@ -116,12 +116,14 @@ for devices that you trust. Drivers ------- -Exscript recognizes IOS XR and Junos devices by the capabilities in their -```` (:doc:`drivers`). The driver provides the patterns that detect +Exscript recognizes IOS XR, Junos and NX-OS devices by the capabilities in +their ```` (:doc:`drivers`). The driver provides the patterns that detect the end of a reply and errors in it, and the ncclient device handler: its ``netconf_device_params`` attribute is passed to ncclient's ``make_device_handler()``. The built-in drivers use ncclient's default -handler. Because the device handler is chosen when the session starts, +handler; for NX-OS, ncclient's ``nexus`` handler would replace the base +capability of RFC 6241 in the ```` of the client with the one of the +old XML agent. Because the device handler is chosen when the session starts, before the capabilities are known, pass a driver that sets ``netconf_device_params`` with ``driver=``. diff --git a/src/Exscript/protocols/drivers/nxos.py b/src/Exscript/protocols/drivers/nxos.py index a4bc4347..cd9422e1 100644 --- a/src/Exscript/protocols/drivers/nxos.py +++ b/src/Exscript/protocols/drivers/nxos.py @@ -60,6 +60,9 @@ # config allowed." or "Error: AAA authorization failed ...". re.compile(r'^(?:ERROR|Error): '), ] +# The capability of the device model, with or without parameters, e.g. +# http://cisco.com/ns/yang/cisco-nx-os-device?revision=2023-12-13&module=... +_netconf_caps_re = [re.compile(r'http://cisco\.com/ns/yang/cisco-nx-os-device(?![\w-])')] class NXOSDriver(Driver): @@ -73,16 +76,22 @@ def __init__(self) -> None: """ Constructor. Sets the patterns of the user name prompts " login:" (also without the hostname) and "Username:", the - password prompts, the command prompt and the error messages, such as - "%Error", "Invalid input", the parser errors that end with - "at '^' marker.", "% Permission denied for the role", and lines that - start with "ERROR: " or "Error: ". + password prompts, the command prompt, the NETCONF capability of the + device model, and the error messages, such as "%Error", + "Invalid input", the parser errors that end with "at '^' marker.", + "% Permission denied for the role", and lines that start with + "ERROR: " or "Error: ". """ Driver.__init__(self, 'nxos') self.user_re = _user_re self.password_re = _password_re self.prompt_re = _prompt_re self.error_re = _error_re + self.netconf_capabilities_re = _netconf_caps_re + # No netconf_device_params={'name': 'nexus'}: ncclient's nexus handler + # replaces the base capability "urn:ietf:params:netconf:base:1.0" in + # the hello of the client with "urn:ietf:params:xml:ns:netconf:base:1.0" + # of the old XML agent, which a server of RFC 6241 may reject. def check_head_for_os(self, string: str) -> int: """ diff --git a/tests/Exscript/protocols/NXOSDriverTest.py b/tests/Exscript/protocols/NXOSDriverTest.py index 7b6da335..3e9d97ed 100644 --- a/tests/Exscript/protocols/NXOSDriverTest.py +++ b/tests/Exscript/protocols/NXOSDriverTest.py @@ -4,13 +4,32 @@ import unittest +from lxml import etree + from Exscript import Account from Exscript.emulators import VirtualDevice from Exscript.protocols import Dummy from Exscript.protocols.drivers import driver_map from Exscript.protocols.exception import InvalidCommandException +from Exscript.protocols.netconf import Netconf from .ExpectTest import connect +from .NetconfTest import NS, NetconfServer + +# The capabilities in the hello of NX-OS 10.4(2)F, escaped for XML. +NXOS_CAPABILITIES = ( + 'urn:ietf:params:netconf:base:1.0', + 'urn:ietf:params:netconf:base:1.1', + 'urn:ietf:params:netconf:capability:writable-running:1.0', + 'urn:ietf:params:netconf:capability:rollback-on-error:1.0', + 'urn:ietf:params:netconf:capability:candidate:1.0', + 'urn:ietf:params:netconf:capability:validate:1.1', + 'urn:ietf:params:netconf:capability:confirmed-commit:1.1', + 'urn:ietf:params:netconf:capability:notification:1.0', + 'urn:ietf:params:netconf:capability:interleave:1.0', + 'urn:ietf:params:netconf:capability:with-defaults:1.0?basic-mode=report-all', + 'http://cisco.com/ns/yang/cisco-nx-os-device?revision=2023-12-13&module=Cisco-NX-OS-device', +) def nxos_device(user_prompt='switch login: '): @@ -129,10 +148,47 @@ def testNoErrors(self): self.assertEqual(self.conn.get_response(), output) +class NetconfSessionTest(unittest.TestCase): + """NETCONF sessions with a server that sends the hello of NX-OS.""" + + def login(self, driver=None): + self.server = NetconfServer(self, capabilities=NXOS_CAPABILITIES) + conn = Netconf( + timeout=10, + connect_rate_limit=0, + verify_fingerprint=False, + preferred_authentications=['password'], + driver=driver, + ) + self.addCleanup(conn.close, force=True) + conn.connect('127.0.0.1', self.server.port) + conn.login(Account('admin', 'secret', needs_lock=False)) + return conn + + def testGuess(self): + conn = self.login() + self.assertEqual(conn.guess_os(), 'nxos') + self.assertEqual(conn.get_driver().name, 'nxos') + + def testHelloOfTheClient(self): + # The driver keeps ncclient's default handler; the nexus handler would replace + # the base capability of RFC 6241 with the one of the old XML agent. + self.login(driver='nxos') + self.assertTrue(self.server.hello_received.wait(5)) + hello = etree.fromstring(self.server.client_hello) + capabilities = [element.text for element in hello.iter(f'{{{NS}}}capability')] + self.assertIn('urn:ietf:params:netconf:base:1.0', capabilities) + self.assertNotIn('urn:ietf:params:xml:ns:netconf:base:1.0', capabilities) + + def suite(): loader = unittest.TestLoader() return unittest.TestSuite( - [loader.loadTestsFromTestCase(LoginTest), loader.loadTestsFromTestCase(ErrorTest)] + [ + loader.loadTestsFromTestCase(LoginTest), + loader.loadTestsFromTestCase(ErrorTest), + loader.loadTestsFromTestCase(NetconfSessionTest), + ] ) diff --git a/tests/Exscript/protocols/OsGuesserTest.py b/tests/Exscript/protocols/OsGuesserTest.py index 4af4d6ad..9fa510df 100644 --- a/tests/Exscript/protocols/OsGuesserTest.py +++ b/tests/Exscript/protocols/OsGuesserTest.py @@ -113,6 +113,30 @@ def testCheckNetconfCapabilitiesAsText(self): ) self.assertEqual(osg.get('os'), 'junos') + def testCheckNetconfCapabilitiesOfNxos(self): + # The device model of NX-OS 10.4(2)F, with the deviations of 9.x, and alone. + base = ['urn:ietf:params:netconf:base:1.0', 'urn:ietf:params:netconf:base:1.1'] + device = 'http://cisco.com/ns/yang/cisco-nx-os-device' + for capability in ( + device + '?revision=2023-12-13&module=Cisco-NX-OS-device', + device + '?revision=2018-07-17&module=Cisco-NX-OS-device' + '&deviations=Cisco-NX-OS-device-deviations', + device, + ): + osg = OsGuesser() + osg.check_netconf_capabilities([*base, capability]) + self.assertEqual(osg.get('os'), 'nxos', capability) + # The models of IOS XE are no sign. + osg = OsGuesser() + osg.check_netconf_capabilities( + [ + *base, + 'http://cisco.com/ns/yang/Cisco-IOS-XE-native' + '?module=Cisco-IOS-XE-native&revision=2019-11-01', + ] + ) + self.assertEqual(osg.get('os'), 'unknown') + def testSetWithoutValue(self): self.sa.set('os', 'ios', 50) self.sa.set('os', None, 100) # ignored From f3b7309a153114df1ac3b48095b22a03fcf26993 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 03:19:16 +0100 Subject: [PATCH 216/250] Move the NX-OS tests to the tests of the drivers --- tests/Exscript/protocols/NXOSDriverTest.py | 196 ------------------- tests/Exscript/protocols/drivers/nxosTest.py | 166 +++++++++++++++- 2 files changed, 164 insertions(+), 198 deletions(-) delete mode 100644 tests/Exscript/protocols/NXOSDriverTest.py diff --git a/tests/Exscript/protocols/NXOSDriverTest.py b/tests/Exscript/protocols/NXOSDriverTest.py deleted file mode 100644 index 3e9d97ed..00000000 --- a/tests/Exscript/protocols/NXOSDriverTest.py +++ /dev/null @@ -1,196 +0,0 @@ -""" -Tests of the nxos driver with an emulated NX-OS switch. -""" - -import unittest - -from lxml import etree - -from Exscript import Account -from Exscript.emulators import VirtualDevice -from Exscript.protocols import Dummy -from Exscript.protocols.drivers import driver_map -from Exscript.protocols.exception import InvalidCommandException -from Exscript.protocols.netconf import Netconf - -from .ExpectTest import connect -from .NetconfTest import NS, NetconfServer - -# The capabilities in the hello of NX-OS 10.4(2)F, escaped for XML. -NXOS_CAPABILITIES = ( - 'urn:ietf:params:netconf:base:1.0', - 'urn:ietf:params:netconf:base:1.1', - 'urn:ietf:params:netconf:capability:writable-running:1.0', - 'urn:ietf:params:netconf:capability:rollback-on-error:1.0', - 'urn:ietf:params:netconf:capability:candidate:1.0', - 'urn:ietf:params:netconf:capability:validate:1.1', - 'urn:ietf:params:netconf:capability:confirmed-commit:1.1', - 'urn:ietf:params:netconf:capability:notification:1.0', - 'urn:ietf:params:netconf:capability:interleave:1.0', - 'urn:ietf:params:netconf:capability:with-defaults:1.0?basic-mode=report-all', - 'http://cisco.com/ns/yang/cisco-nx-os-device?revision=2023-12-13&module=Cisco-NX-OS-device', -) - - -def nxos_device(user_prompt='switch login: '): - """ - Returns a switch that shows the default banner and the login prompt of - NX-OS 7.x to 10.x at the console and over Telnet. - """ - device = VirtualDevice('switch', banner='User Access Verification\r\n', strict=False) - device.user_prompt = user_prompt - device.prompt = 'switch# ' - device.add_command('show clock', '12:00:00.000 UTC Wed Nov 11 2026') - return device - - -class LoginTest(unittest.TestCase): - """Logins at the console and over Telnet.""" - - def login(self, device, driver=None): - conn = Dummy(device=device, driver=driver, timeout=1) - self.addCleanup(conn.close, force=True) - conn.connect('switch') - conn.login(Account('admin', 'secret', needs_lock=False)) - return conn - - def testLoginPrompt(self): - # Before, the driver waited for "Username:" until the timeout (upstream PR #178). - for prompt in 'switch login: ', 'login: ', ' login: ', 'Username: ': - with self.subTest(prompt=prompt): - conn = self.login(nxos_device(prompt), driver='nxos') - conn.execute('show clock') - self.assertEqual(conn.get_response(), '12:00:00.000 UTC Wed Nov 11 2026') - - def testUserPromptPatterns(self): - def matches(text): - return any(regex.search(text) for regex in driver_map['nxos'].user_re) - - for text in ( - '\r\nRackX_XXX login: ', - '\r\nIP_switch_A_1 login: ', - 'login: ', - '\r\nUsername:', - ): - self.assertTrue(matches(text), repr(text)) - # A read may end after "Last login: ". - for text in '\r\nLast login: ', '\r\nswitch login: admin\r\nPassword: ': - self.assertFalse(matches(text), repr(text)) - - def testGuessAtTheLoginPrompt(self): - # IOS shows the banner, too, but asks for "Username:". Before, the guess was - # ios, whose patterns do not match the prompt, and the login timed out. - conn = self.login(nxos_device()) - self.assertEqual(conn.guess_os(), 'nxos') - conn.execute('show clock') - self.assertEqual(conn.get_response(), '12:00:00.000 UTC Wed Nov 11 2026') - - def testGuessInSmallReads(self): - # Over Telnet, e.g. one character per read: the guess is ios until the prompt. - head = '\r\nUser Access Verification\r\nswitch login: ' - conn = connect([*head, 'admin\r\nPassword: ', '\r\nswitch# ']) - self.addCleanup(conn.close, force=True) - conn.login(Account('admin', 'secret', needs_lock=False)) - self.assertEqual(conn.sent, ['admin\r', 'secret\r']) # no "enable" as for ios - self.assertEqual(conn.guess_os(), 'nxos') - - -class ErrorTest(unittest.TestCase): - """The error messages of NX-OS in the response of a command.""" - - def setUp(self): - self.output = '' - device = nxos_device() - device.add_command('command', lambda command: self.output) - self.conn = Dummy(device=device, driver='nxos', timeout=1) - self.addCleanup(self.conn.close, force=True) - self.conn.connect('switch') - self.conn.login(Account('admin', 'secret', needs_lock=False)) - - def testErrors(self): - # The parser errors follow a line with a caret under the wrong word. - outputs = [ - " ^\r\n% Invalid command at '^' marker.", - " ^\r\n% Incomplete command at '^' marker.", - " ^\r\n% Ambiguous command at '^' marker.", - " ^\r\n% Invalid number, range is (4096:16777215) at '^' marker.", - " ^\r\n% Invalid parameter detected at '^' marker.", - " ^\r\nInvalid range at '^' marker.", - " ^\r\nInvalid interface format at '^' marker.", - '% Permission denied for the role', - 'Error: AAA authorization failed AAA_AUTHOR_STATUS_METHOD=16(0x10)', - 'ERROR: port already in a port-channel, no config allowed.', - 'Error: default interface is not supported as speed-group is configured', - ] - for output in outputs: - with self.subTest(output=output): - self.output = output - with self.assertRaises(InvalidCommandException) as cm: - self.conn.execute('command') - line = output.split('\r\n')[-1] - self.assertEqual( - str(cm.exception), 'Error in the response from the device: ' + line - ) - - def testNoErrors(self): - outputs = [ - ' 0 input error 0 short frame', - 'Errors: 0', - 'Ethernet1/1 is down (error-disabled)', - '2026 Nov 11 10:00:00 switch %ETHPORT-5-IF_DOWN_ERROR_DISABLED: Interface' - ' Ethernet1/1 is down (Error disabled. Reason:BPDUGuard)', - ' description ERROR: spare port, do not use', - ] - for output in outputs: - with self.subTest(output=output): - self.output = output - self.conn.execute('command') - self.assertEqual(self.conn.get_response(), output) - - -class NetconfSessionTest(unittest.TestCase): - """NETCONF sessions with a server that sends the hello of NX-OS.""" - - def login(self, driver=None): - self.server = NetconfServer(self, capabilities=NXOS_CAPABILITIES) - conn = Netconf( - timeout=10, - connect_rate_limit=0, - verify_fingerprint=False, - preferred_authentications=['password'], - driver=driver, - ) - self.addCleanup(conn.close, force=True) - conn.connect('127.0.0.1', self.server.port) - conn.login(Account('admin', 'secret', needs_lock=False)) - return conn - - def testGuess(self): - conn = self.login() - self.assertEqual(conn.guess_os(), 'nxos') - self.assertEqual(conn.get_driver().name, 'nxos') - - def testHelloOfTheClient(self): - # The driver keeps ncclient's default handler; the nexus handler would replace - # the base capability of RFC 6241 with the one of the old XML agent. - self.login(driver='nxos') - self.assertTrue(self.server.hello_received.wait(5)) - hello = etree.fromstring(self.server.client_hello) - capabilities = [element.text for element in hello.iter(f'{{{NS}}}capability')] - self.assertIn('urn:ietf:params:netconf:base:1.0', capabilities) - self.assertNotIn('urn:ietf:params:xml:ns:netconf:base:1.0', capabilities) - - -def suite(): - loader = unittest.TestLoader() - return unittest.TestSuite( - [ - loader.loadTestsFromTestCase(LoginTest), - loader.loadTestsFromTestCase(ErrorTest), - loader.loadTestsFromTestCase(NetconfSessionTest), - ] - ) - - -if __name__ == '__main__': - unittest.TextTestRunner(verbosity=2).run(suite()) diff --git a/tests/Exscript/protocols/drivers/nxosTest.py b/tests/Exscript/protocols/drivers/nxosTest.py index 79f8c15c..e4dfe7d3 100644 --- a/tests/Exscript/protocols/drivers/nxosTest.py +++ b/tests/Exscript/protocols/drivers/nxosTest.py @@ -1,14 +1,47 @@ """ -The nxos driver, with the banner and the prompts of Cisco NX-OS. +The nxos driver, with the banners, the prompts, the messages and the NETCONF +capabilities of Cisco NX-OS. """ import unittest +from lxml import etree + from Exscript.protocols.drivers import driver_map from Exscript.protocols.exception import InvalidCommandException +from Exscript.protocols.netconf import Netconf +from ..ExpectTest import connect as scripted +from ..NetconfTest import NS, NetconfServer from .device import ACCOUNT, Device, banner, connect, search +# The capabilities in the hello of NX-OS 10.4(2)F, escaped for XML. +NXOS_CAPABILITIES = ( + 'urn:ietf:params:netconf:base:1.0', + 'urn:ietf:params:netconf:base:1.1', + 'urn:ietf:params:netconf:capability:writable-running:1.0', + 'urn:ietf:params:netconf:capability:rollback-on-error:1.0', + 'urn:ietf:params:netconf:capability:candidate:1.0', + 'urn:ietf:params:netconf:capability:validate:1.1', + 'urn:ietf:params:netconf:capability:confirmed-commit:1.1', + 'urn:ietf:params:netconf:capability:notification:1.0', + 'urn:ietf:params:netconf:capability:interleave:1.0', + 'urn:ietf:params:netconf:capability:with-defaults:1.0?basic-mode=report-all', + 'http://cisco.com/ns/yang/cisco-nx-os-device?revision=2023-12-13&module=Cisco-NX-OS-device', +) + + +def login_device(user_prompt='switch login: '): + """ + Returns a switch with the default banner and the given user name prompt, + as NX-OS 7.x to 10.x shows them at the console and over Telnet. + """ + device = Device( + 'switch# ', banner='User Access Verification\r\n', login=(user_prompt, 'Password: ') + ) + device.add_command('show clock', '\r\n12:00:00.000 UTC Wed Nov 11 2026') + return device + class NXOSDriverTest(unittest.TestCase): def setUp(self): @@ -25,6 +58,18 @@ def testPrompts(self): self.assertTrue(search(self.driver.prompt_re, 'Last login\r\n' + prompt), prompt) self.assertFalse(search(self.driver.prompt_re, '\r\n Description: uplink #1')) + def testUserPromptPatterns(self): + for text in ( + '\r\nRackX_XXX login: ', + '\r\nIP_switch_A_1 login: ', + 'login: ', + '\r\nUsername:', + ): + self.assertTrue(search(self.driver.user_re, text), repr(text)) + # A read may end after "Last login: ". + for text in '\r\nLast login: ', '\r\nswitch login: admin\r\nPassword: ': + self.assertFalse(search(self.driver.user_re, text), repr(text)) + def testLoginWithoutEnable(self): # NX-OS has no enable mode: the role of the user grants the privileges. device = Device('N7K-1#', login=('Username: ', 'Password: '), enabled_prompt='N7K-1#') @@ -33,6 +78,33 @@ def testLoginWithoutEnable(self): conn.auto_app_authorize(ACCOUNT) self.assertEqual(device.received, ['admin', 'login-secret']) + def testLoginPrompt(self): + # The login works at each user name prompt of NX-OS, also without the host name. + for prompt in 'switch login: ', 'login: ', ' login: ', 'Username: ': + with self.subTest(prompt=prompt): + conn = connect(self, login_device(prompt), 'nxos', login=ACCOUNT) + conn.execute('show clock') + self.assertEqual(conn.get_response(), '12:00:00.000 UTC Wed Nov 11 2026') + + def testGuessAtTheLoginPrompt(self): + # IOS shows the banner, too, but asks for "Username:"; the "login:" prompt makes + # the guess nxos. + device = login_device() + conn = connect(self, device, None, login=ACCOUNT) + self.assertEqual(conn.guess_os(), 'nxos') + conn.execute('show clock') + self.assertEqual(conn.get_response(), '12:00:00.000 UTC Wed Nov 11 2026') + self.assertEqual(device.received, ['admin', 'login-secret', 'show clock']) + + def testGuessInSmallReads(self): + # Over Telnet, e.g. one character per read: the guess is ios until the prompt. + head = '\r\nUser Access Verification\r\nswitch login: ' + conn = scripted([*head, 'admin\r\nPassword: ', '\r\nswitch# ']) + self.addCleanup(conn.close, force=True) + conn.login(ACCOUNT) + self.assertEqual(conn.sent, ['admin\r', 'login-secret\r']) # no "enable" as for ios + self.assertEqual(conn.guess_os(), 'nxos') + def testInitTerminal(self): device = Device('N7K-1#') conn = connect(self, device, 'nxos') @@ -50,8 +122,98 @@ def testErrors(self): self.assertRaises(InvalidCommandException, conn.execute, command) +class ErrorTest(unittest.TestCase): + """The error messages of NX-OS in the response of a command.""" + + def setUp(self): + self.output = '' + device = Device('switch# ') + device.add_command('command', lambda command: '\r\n' + self.output) + self.conn = connect(self, device, 'nxos') + + def testErrors(self): + # The parser errors follow a line with a caret under the wrong word. + outputs = [ + " ^\r\n% Invalid command at '^' marker.", + " ^\r\n% Incomplete command at '^' marker.", + " ^\r\n% Ambiguous command at '^' marker.", + " ^\r\n% Invalid number, range is (4096:16777215) at '^' marker.", + " ^\r\n% Invalid parameter detected at '^' marker.", + " ^\r\nInvalid range at '^' marker.", + " ^\r\nInvalid interface format at '^' marker.", + '% Permission denied for the role', + 'Error: AAA authorization failed AAA_AUTHOR_STATUS_METHOD=16(0x10)', + 'ERROR: port already in a port-channel, no config allowed.', + 'Error: default interface is not supported as speed-group is configured', + ] + for output in outputs: + with self.subTest(output=output): + self.output = output + with self.assertRaises(InvalidCommandException) as cm: + self.conn.execute('command') + line = output.split('\r\n')[-1] + self.assertEqual( + str(cm.exception), 'Error in the response from the device: ' + line + ) + + def testNoErrors(self): + outputs = [ + ' 0 input error 0 short frame', + 'Errors: 0', + 'Ethernet1/1 is down (error-disabled)', + '2026 Nov 11 10:00:00 switch %ETHPORT-5-IF_DOWN_ERROR_DISABLED: Interface' + ' Ethernet1/1 is down (Error disabled. Reason:BPDUGuard)', + ' description ERROR: spare port, do not use', + ] + for output in outputs: + with self.subTest(output=output): + self.output = output + self.conn.execute('command') + self.assertEqual(self.conn.get_response(), output) + + +class NetconfSessionTest(unittest.TestCase): + """NETCONF sessions with a server that sends the hello of NX-OS.""" + + def login(self, driver=None): + self.server = NetconfServer(self, capabilities=NXOS_CAPABILITIES) + conn = Netconf( + timeout=10, + connect_rate_limit=0, + verify_fingerprint=False, + preferred_authentications=['password'], + driver=driver, + ) + self.addCleanup(conn.close, force=True) + conn.connect('127.0.0.1', self.server.port) + conn.login(ACCOUNT) + return conn + + def testGuess(self): + conn = self.login() + self.assertEqual(conn.guess_os(), 'nxos') + self.assertEqual(conn.get_driver().name, 'nxos') + + def testHelloOfTheClient(self): + # The driver keeps ncclient's default handler; the nexus handler would replace + # the base capability of RFC 6241 with the one of the old XML agent. + self.login(driver='nxos') + self.assertTrue(self.server.hello_received.wait(5)) + hello = etree.fromstring(self.server.client_hello) + capabilities = [element.text for element in hello.iter(f'{{{NS}}}capability')] + self.assertIn('urn:ietf:params:netconf:base:1.0', capabilities) + self.assertNotIn('urn:ietf:params:xml:ns:netconf:base:1.0', capabilities) + + def suite(): - return unittest.TestLoader().loadTestsFromTestCase(NXOSDriverTest) + loader = unittest.TestLoader() + return unittest.TestSuite( + [ + loader.loadTestsFromTestCase(NXOSDriverTest), + loader.loadTestsFromTestCase(ErrorTest), + loader.loadTestsFromTestCase(NetconfSessionTest), + ] + ) if __name__ == '__main__': From 4501b7d7695a47fe2fdad19928c38a900a31a1cf Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 03:20:48 +0100 Subject: [PATCH 217/250] Merge the changelog entries of the NX-OS error messages One release had two entries for them. The migration guide now lists the same messages as the merged entry. --- CHANGES.md | 19 ++++++++----------- docs/migration.rst | 10 ++++++---- 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 62007c3a..42f73ffb 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -416,14 +416,15 @@ development versions; "before" refers to them. `Username:`. Before, the guess was `ios`, whose patterns do not match this prompt, so the login waited until the timeout. The transcript of upstream PR #178 is now part of the test corpus. -- **BREAKING - NX-OS Error Messages:** The `nxos` driver recognizes more error messages - of NX-OS, so `execute()` raises an `InvalidCommandException` for the commands that - NX-OS rejects with them: all parser errors, which end with `at '^' marker.`, e.g. - `% Invalid command at '^' marker.` or `Invalid range at '^' marker.`, then - `% Permission denied for the role`, and lines that start with `ERROR: ` or `Error: `, - e.g. `ERROR: port already in a port-channel, no config allowed.`. Before, it knew +- **BREAKING - NX-OS Error Messages:** The `nxos` driver recognizes the error messages + of NX-OS, so `execute()` raises an `InvalidCommandException` for a command that NX-OS + rejects: for the parser errors, which end with `at '^' marker.`, such as + `% Invalid command at '^' marker.` for an unknown command or keyword, the most common + one, or `Invalid range at '^' marker.`; for `% Permission denied for the role`; and + for lines that start with `ERROR: ` or `Error: `, e.g. + `ERROR: port already in a port-channel, no config allowed.`. Before, the driver knew only the messages that IOS shows, too, such as `% Incomplete command`, and returned - the response of the other commands as if they had succeeded. To migrate, catch the + the response of such a command as if it had succeeded. To migrate, catch the exception where a command may fail. - **NX-OS Prompts With Two Modes:** The `nxos` driver recognizes prompts with more than one mode in parentheses, e.g. `switch(maint-mode)(config)# ` in maintenance mode or @@ -435,10 +436,6 @@ development versions; "before" refers to them. `unknown`. The driver keeps ncclient's default device handler: the `nexus` handler replaces the base capability of RFC 6241 in the hello of the client with the one of the old XML agent, which a server may reject. -- **NX-OS Invalid Commands:** The `nxos` driver recognizes the error message - `% Invalid command at '^' marker.` of NX-OS, so `execute()` raises an - `InvalidCommandException` for such a command. Before, it returned the response as - if the command had succeeded. - **HP ProCurve Configuration Prompts:** The `hp_pro_curve` driver recognizes the prompts of the configuration contexts, e.g. `HP-2920(config)# ` or `HP-2920(vlan-10)# `. Before, a command that entered such a context, e.g. diff --git a/docs/migration.rst b/docs/migration.rst index d0141153..92d51f95 100644 --- a/docs/migration.rst +++ b/docs/migration.rst @@ -96,11 +96,13 @@ Driver names the driver. NX-OS errors - The ``nxos`` driver recognizes more error messages of NX-OS, e.g. - ``% Invalid command at '^' marker.`` and lines that start with - ``ERROR:``, so ``execute()`` raises + The ``nxos`` driver recognizes the error messages of NX-OS: the parser + errors, which end with ``at '^' marker.``, such as + ``% Invalid command at '^' marker.``, then + ``% Permission denied for the role``, and lines that start with + ``ERROR:`` or ``Error:``. So ``execute()`` raises :class:`~Exscript.protocols.exception.InvalidCommandException` for the - commands that NX-OS rejects with them. 2.x returned their response. + commands that NX-OS rejects with them; 2.x returned their response. Catch the exception where a command may fail. Templates From 5993b836eafb4f2bcad10a8a050c36d2a0f4f3bc Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 03:20:04 +0100 Subject: [PATCH 218/250] End ncclient's thread also if a callback blocks it during close() If a monitor callback blocked the thread longer than close() waits, closing the channel hid the end of the session from it, and it polled until the process ended. The channel now stays open while the thread is alive; paramiko closes it when it is garbage-collected. --- CHANGES.md | 6 +++-- src/Exscript/protocols/netconf.py | 13 +++++++++- tests/Exscript/protocols/NetconfTest.py | 32 +++++++++++++++++++++++++ 3 files changed, 48 insertions(+), 3 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 42f73ffb..8da0dcfe 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -312,8 +312,10 @@ development versions; "before" refers to them. - **`close()` Ends ncclient's Thread:** `close()` ends the thread in which ncclient reads the session, also with `force=True` or if the server does not answer - ``. Before, the thread then kept running until the process ended, and - `close()` waited 0.3 seconds for it in vain. + ``. If a callback, e.g. of a monitor, keeps the thread busy during + `close()` or calls `close()` itself, the thread ends when the callback returns. + Before, the thread then kept running until the process ended, and `close()` waited + 0.3 seconds for it in vain. - **RPC Body of One Element:** `execute()` and `send()` raise `XMLValidationError` before sending if the RPC body is not one XML element, e.g. `` or text. Before, they raised lxml's `XMLSyntaxError`. diff --git a/src/Exscript/protocols/netconf.py b/src/Exscript/protocols/netconf.py index 920ea559..69ba4a9d 100644 --- a/src/Exscript/protocols/netconf.py +++ b/src/Exscript/protocols/netconf.py @@ -801,7 +801,8 @@ def _close(self, force: bool = False) -> None: """ Tells ncclient's IO thread that the session is closed on purpose, closes the transport, and waits up to a second for the IO thread to - end. Then closes the channel and the socket like SSH2._close(). + end. Then closes the socket like SSH2._close(), and the channel if the + IO thread ended; else the thread ends when its listener returns. """ # Clear the ncclient manager reference. if self._ncc_manager: @@ -836,5 +837,15 @@ def _close(self, force: bool = False) -> None: if ncc_session.is_alive(): self._dbg(1, "ncclient's IO thread did not end within a second.") + # If the thread still runs a listener, e.g. a slow monitor callback, or + # this is that thread, leave the channel open: closing it would close the + # pipe that the thread selects on, so the thread would never see the end + # of the session and would wake up every 0.1 seconds until the process + # ends. The pipe stays readable, so the thread ends by itself once the + # listener returns; paramiko closes the channel when it is + # garbage-collected. + if ncc_session and ncc_session.is_alive(): + self._channel = None + # SSH2 closes the paramiko channel, the transport, and the socket. super()._close(force=force) diff --git a/tests/Exscript/protocols/NetconfTest.py b/tests/Exscript/protocols/NetconfTest.py index 6dac51da..3da95c6d 100644 --- a/tests/Exscript/protocols/NetconfTest.py +++ b/tests/Exscript/protocols/NetconfTest.py @@ -801,6 +801,38 @@ def testCloseEndsTheIOThread(self): conn.close(force=force) self.assertFalse(thread.is_alive()) + def testCloseWhileAListenerBlocksTheIOThread(self): + # E.g. a slow monitor callback: the thread ends when the listener returns. + conn = self.login() + thread = conn._ncc_session + entered, release = threading.Event(), threading.Event() + self.addCleanup(release.set) + + def block(root, raw): + entered.set() + release.wait(5) + + listener = ncclient.transport.session.SessionListener() + listener.callback = block + listener.errback = lambda error: None + thread.add_listener(listener) + self.server.send(NOTIFICATION) + self.assertTrue(entered.wait(5)) + conn.close(force=True) # waits a second for the thread in vain + release.set() + thread.join(5) + self.assertFalse(thread.is_alive()) + + def testCloseInAMonitorCallback(self): + # The callback runs in the IO thread, which ends when the callback returns. + conn = self.login() + thread = conn._ncc_session + conn.add_monitor('', lambda conn, index, match: conn.close(force=True)) + self.server.send(NOTIFICATION) + thread.join(5) + self.assertFalse(thread.is_alive()) + self.assertFalse(conn.connected) + def testSubsystemRefused(self): for options in {'subsystem': False}, {'channels': False}: with self.assertRaisesRegex(LoginFailure, 'any subsystem: netconf'): From 8c7723b600f2108e97f9ffef2e66300bc5600bcc Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 03:22:10 +0100 Subject: [PATCH 219/250] Return from interact() also if the terminal is gone Restoring the settings of a closed terminal raised termios.error, which replaced the error that ended the session and left the SIGWINCH handler installed. Ignore it, and restore the handler first. --- CHANGES.md | 4 ++ src/Exscript/protocols/protocol.py | 6 ++- tests/Exscript/protocols/ExpectTest.py | 55 ++++++++++++++++++++++++++ 3 files changed, 63 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 8da0dcfe..57b7b8f6 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -218,6 +218,10 @@ marked with **BREAKING**; a migration guide will accompany the release. instead of shown, and typed text is sent like with `send()`, so Telnet escapes the byte 255. The previous `SIGWINCH` handler is restored when the session ends. On Windows, the thread that receives no longer keeps the process alive. +- **`interact()` on a Closed Terminal:** If the terminal is gone when the session ends, + e.g. because its window was closed, `interact()` returns, or raises the error that + ended the session, and restores the previous `SIGWINCH` handler. Before, it raised + `termios.error` instead and kept its own `SIGWINCH` handler. ### Waiting for Responses diff --git a/src/Exscript/protocols/protocol.py b/src/Exscript/protocols/protocol.py index b60f1391..ac1ec83c 100644 --- a/src/Exscript/protocols/protocol.py +++ b/src/Exscript/protocols/protocol.py @@ -1972,7 +1972,7 @@ def _open_posix_shell( terminal to raw mode, passes the typed characters to the key handlers or sends them, and receives data from the channel, until the host closes the connection or the input ends. Restores the terminal - settings afterwards. + settings afterwards, unless the terminal is gone, e.g. closed. """ # Typed characters may take several bytes. decoder = codecs.getincrementaldecoder(sys.stdin.encoding or 'utf-8')(errors='replace') @@ -2024,9 +2024,11 @@ def handle_sigwinch(signum: int | None, frame: FrameType | None) -> None: if self.send_data is not None: self.send_data.write(data) finally: - termios.tcsetattr(stdin, termios.TCSADRAIN, oldtty) if handle_window_size: signal.signal(signal.SIGWINCH, old_handler or signal.SIG_DFL) + # The terminal may be gone, e.g. closed: its settings do not matter then. + with contextlib.suppress(termios.error): + termios.tcsetattr(stdin, termios.TCSADRAIN, oldtty) def _open_windows_shell( self, diff --git a/tests/Exscript/protocols/ExpectTest.py b/tests/Exscript/protocols/ExpectTest.py index 77e0faca..d11650f5 100644 --- a/tests/Exscript/protocols/ExpectTest.py +++ b/tests/Exscript/protocols/ExpectTest.py @@ -659,3 +659,58 @@ def testSSH2(self): preferred_authentications=['password'], ) self.interact(SSHd, conn, key=PrivateKey.from_file(KEYFILE)) + + +@unittest.skipUnless(hasattr(os, 'openpty'), 'interact() reads from a POSIX terminal') +class ClosedTerminalTest(unittest.TestCase): + """ + interact() on a terminal that is closed while it runs, e.g. its window, + so that the settings of the terminal cannot be restored. + """ + + def setUp(self): + self.old_handler = signal.getsignal(signal.SIGWINCH) + self.addCleanup(signal.signal, signal.SIGWINCH, self.old_handler) + + def interact(self, typed=b'', key_handlers=None): + """ + Runs interact() on a new terminal, which gets the typed bytes when it + is in raw mode, or else is closed then. + """ + import tty # POSIX only + + device = VirtualDevice(HOST, login_type=VirtualDevice.LOGIN_TYPE_NONE, strict=False) + conn = Telnet(timeout=1, connect_rate_limit=0) + self.addCleanup(conn.close, force=True) + conn.connect(HOST, start_daemon(self, Telnetd, device)[1]) + master, slave = os.openpty() # interact() closes the slave + self.terminal = os.fdopen(master, 'wb', 0) + self.addCleanup(self.terminal.close) + setcbreak = tty.setcbreak + + def set_raw_mode(fd, *args): + setcbreak(fd, *args) # discards the bytes typed before + if typed: + self.terminal.write(typed) + else: + self.terminal.close() + + with ( + mock.patch('sys.stdin', mock.Mock(encoding='utf-8', fileno=lambda: slave)), + mock.patch('tty.setcbreak', set_raw_mode), + ): + conn.interact(key_handlers) + + def testEndOfInput(self): + self.interact() + self.assertEqual(signal.getsignal(signal.SIGWINCH), self.old_handler) + + def testErrorOfAKeyHandler(self): + # The error that ended interact() is raised, not the error of the terminal. + def close(conn): + self.terminal.close() + raise ValueError('stop') + + with self.assertRaisesRegex(ValueError, 'stop'): + self.interact(b'\x19', {'\x19': close}) + self.assertEqual(signal.getsignal(signal.SIGWINCH), self.old_handler) From 657390f5fa737b1d7154768ac07f2aa5934c2ab1 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 03:24:03 +0100 Subject: [PATCH 220/250] Keep escaped placeholders of mail templates as text \{name} became $name, the variable syntax of Exscript templates, instead of {name}. The docstring of from_template_string() documents the escape. --- CHANGES.md | 3 +++ src/Exscript/util/mail.py | 13 +++++++------ tests/Exscript/util/mailTest.py | 8 ++++++++ 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 57b7b8f6..e1c79b6d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -660,6 +660,9 @@ The names below refer to functions callable from templates, not protocol methods line. `Mail()` without a sender, and a template without a `From:` line, raise a `ValueError`. Before, all of them failed with the `OSError` of `getuser()`, or with a `KeyError` before Python 3.13. +- **Escaped Placeholders in Mail Templates:** In the templates of `Exscript.util.mail`, + a placeholder that is escaped with a backslash, `\{name}`, gives the text `{name}`. + Before, it became `$name`. - **Pseudo Protocol on Windows:** `pseudo://` URLs accept absolute Windows paths such as `pseudo://C:\devices\router.py`; the drive letter was taken for a port number. - **`Exscript.util.url.Url.from_string(None)`:** Raises a `TypeError` with the message diff --git a/src/Exscript/util/mail.py b/src/Exscript/util/mail.py index eba3569d..466f7dc4 100644 --- a/src/Exscript/util/mail.py +++ b/src/Exscript/util/mail.py @@ -67,16 +67,14 @@ def __init__(self) -> None: # substitute the variable against its value. def _variable_sub_cb(self, match: re.Match[str]) -> str: """ - Returns the value of the variable that the given placeholder names, - as a string; the items of a list are put on separate lines. - A placeholder that is escaped with a backslash is replaced by - ``$name``. Raises an Exception if the name is not a valid variable - name, or if the variable is not defined. + Returns the replacement of the given placeholder, see + from_template_string(). Raises an Exception if the name is not a + valid variable name, or if the variable is not defined. """ escape = match.group(1) varname = match.group(2) if escape == '\\': - return '$' + varname + return '{' + varname + '}' if not _varname_re.match(varname): raise Exception('%s is not a variable name' % varname) value = self.tmpl_vars.get(varname) @@ -474,6 +472,9 @@ def from_template_string(string: str, **kwargs: Any) -> Mail: Placeholders of the form ``{name}`` are replaced by the value of the variable with that name; the items of a list are put on separate lines. + A placeholder that is escaped with a backslash, ``\\{name}``, gives the + text ``{name}``. + The variables ``date`` (the current date) and ``user`` (the name of the current user, or an empty string if it is unknown) are defined by default. A line that ends with a space is joined with the next line. The diff --git a/tests/Exscript/util/mailTest.py b/tests/Exscript/util/mailTest.py index 63f9e837..930acdb8 100644 --- a/tests/Exscript/util/mailTest.py +++ b/tests/Exscript/util/mailTest.py @@ -217,6 +217,14 @@ def testFromTemplateStringPlaceholders(self): self.assertEqual(mail.get_subject(), '2 hosts failed') self.assertEqual(mail.get_body(), 'Hosts:\nrouter1\nrouter2') + def testFromTemplateStringEscapedPlaceholders(self): + from Exscript.util.mail import from_template_string + + # A backslash keeps a placeholder as text, also one that names no variable. + tmpl = 'From: me@example.com\n\nReplace \\{hosts} and \\{1st} for {count} hosts.' + mail = from_template_string(tmpl, count=2) + self.assertEqual(mail.get_body(), 'Replace {hosts} and {1st} for 2 hosts.') + def testFromTemplateStringErrors(self): from Exscript.util.mail import from_template_string From e6a24090db2053e0b39ae0a7bc96099d7e7484d8 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 03:25:09 +0100 Subject: [PATCH 221/250] Remove three private helpers that nothing calls They are _highest_bit() of Exscript.util.ipv4, Lexer._get_line_from_char() and Logger._reset(). --- src/Exscript/logger.py | 6 ------ src/Exscript/parselib/lexer.py | 8 -------- src/Exscript/util/ipv4.py | 18 ------------------ 3 files changed, 32 deletions(-) diff --git a/src/Exscript/logger.py b/src/Exscript/logger.py index 9a90eec9..116a5371 100644 --- a/src/Exscript/logger.py +++ b/src/Exscript/logger.py @@ -361,12 +361,6 @@ def __init__(self) -> None: self.success = 0 self.failed = 0 - def _reset(self) -> None: - """ - Forgets all logs. - """ - self.logs = defaultdict(list) - def get_succeeded_actions(self) -> int: """ Returns the number of jobs that were completed successfully. diff --git a/src/Exscript/parselib/lexer.py b/src/Exscript/parselib/lexer.py index b159a34e..0e56369d 100644 --- a/src/Exscript/parselib/lexer.py +++ b/src/Exscript/parselib/lexer.py @@ -151,14 +151,6 @@ def get_current_line(self) -> str: line = self.get_current_line_number() return self._get_line(line) - def _get_line_from_char(self, char: int) -> str: - """ - Returns the line that contains the given position, without the - newline. - """ - line = self._get_line_number_from_char(char) - return self._get_line(line) - def _get_line_position_from_char(self, char: int) -> tuple[int, int]: """ Returns the position of the first character of the line that diff --git a/src/Exscript/util/ipv4.py b/src/Exscript/util/ipv4.py index d9bbb498..5cb3fe2d 100644 --- a/src/Exscript/util/ipv4.py +++ b/src/Exscript/util/ipv4.py @@ -24,7 +24,6 @@ IPv4 address calculation and conversion. """ -import math import re import socket import struct @@ -43,23 +42,6 @@ def _least_bit(number: int) -> int: return 32 -def _highest_bit(number: int) -> float: - """ - Rounds the given 32-bit number up to the next power of two and returns - the square root of the result; returns 0 for 0. - """ - if number == 0: - return 0 - number -= 1 - number |= number >> 1 - number |= number >> 2 - number |= number >> 4 - number |= number >> 8 - number |= number >> 16 - number += 1 - return math.sqrt(number) - - def is_ip(string: str) -> bool: """ Returns True if the given string is an IPv4 address, False otherwise. From 47a4f3342db775193a1d529a599a4d560bcc1144 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 03:41:29 +0100 Subject: [PATCH 222/250] Give the SSH test sessions more time on loaded machines A session had to receive each prompt within 1 s, which is tight under load. Successful sessions do not wait for the timeout, so the tests are not slower. --- tests/Exscript/protocols/SSH2Test.py | 35 +++++++++++++++++----------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/tests/Exscript/protocols/SSH2Test.py b/tests/Exscript/protocols/SSH2Test.py index 8a511b24..6ce9ffa6 100644 --- a/tests/Exscript/protocols/SSH2Test.py +++ b/tests/Exscript/protocols/SSH2Test.py @@ -33,12 +33,21 @@ def createDaemon(self): self.daemon = SSHd(self.hostname, self.port, self.device, key=key) self.daemon.dbg = 5 - def createProtocol(self): - self.protocol = SSH2(timeout=1, debug=5, verify_fingerprint=False) + # Sessions that succeed do not wait for the timeout; tests of timeouts pass a short one. + def createProtocol(self, timeout=10): + self.protocol = SSH2(timeout=timeout, debug=5, verify_fingerprint=False) def testConstructor(self): self.assertIsInstance(self.protocol, SSH2) + def testSetTimeout(self): + self.createProtocol(timeout=1) # the timeout that ProtocolTest checks + super().testSetTimeout() + + def testAppAuthorize2(self): + self.createProtocol(timeout=1) # expects a timeout + super().testAppAuthorize2() + def testGetRemoteVersion(self): self.assertEqual(self.protocol.get_remote_version(), None) self.doConnect() @@ -111,7 +120,7 @@ def writeKnownHosts(self, *entries): def connect(self, conn=None, cls=SSH2, **kwargs): if conn is None: - conn = cls(timeout=1, connect_rate_limit=0, known_hosts=self.known_hosts, **kwargs) + conn = cls(timeout=10, connect_rate_limit=0, known_hosts=self.known_hosts, **kwargs) self.addCleanup(conn.close, force=True) conn.connect(HOST, self.server.port) return conn @@ -142,7 +151,7 @@ def testDefaultKnownHostsFile(self): self.known_hosts = os.path.join(home, '.ssh', 'known_hosts') self.writeKnownHosts((self.name, self.other_key)) with mock.patch.dict(os.environ, {'HOME': home, 'USERPROFILE': home}): - conn = SSH2(timeout=1, connect_rate_limit=0) + conn = SSH2(timeout=10, connect_rate_limit=0) self.addCleanup(conn.close, force=True) with self.assertRaises(paramiko.BadHostKeyException): self.connect(conn) @@ -181,7 +190,7 @@ def redirect(address, *args, **kwargs): return connect((address[0], self.server.port), *args, **kwargs) with mock.patch('socket.create_connection', redirect): - conn = SSH2(timeout=1, connect_rate_limit=0, known_hosts=self.known_hosts) + conn = SSH2(timeout=10, connect_rate_limit=0, known_hosts=self.known_hosts) self.addCleanup(conn.close, force=True) conn.host_key_policy = 'auto-add' conn.connect(HOST, 22) @@ -237,7 +246,7 @@ class ReconnectTest(unittest.TestCase): def handshakes(self, driver): server = KeyServer(self, paramiko.RSAKey(filename=keyfile), interface=_PasswordOnly) - conn = SSH2(timeout=1, connect_rate_limit=0, verify_fingerprint=False, driver=driver) + conn = SSH2(timeout=10, connect_rate_limit=0, verify_fingerprint=False, driver=driver) self.addCleanup(conn.close, force=True) home = tempfile.TemporaryDirectory() # no key files self.addCleanup(home.cleanup) @@ -304,7 +313,7 @@ def testUnknownPrompt(self): interface = type('TwoPrompts', (_TwoPrompts,), {'answers': []}) server = KeyServer(self, paramiko.RSAKey(filename=keyfile), interface=interface) stdout = io.StringIO() - conn = SSH2(timeout=1, connect_rate_limit=0, verify_fingerprint=False, stdout=stdout) + conn = SSH2(timeout=10, connect_rate_limit=0, verify_fingerprint=False, stdout=stdout) self.addCleanup(conn.close, force=True) conn.connect(HOST, server.port) conn._paramiko_auth('admin', 'S3cret') @@ -350,7 +359,7 @@ def connect(self, next_types='password', **kwargs): attributes = {'client_key': self.client_key, 'next_types': next_types} interface = type('KeyThenPassword', (_KeyThenPassword,), attributes) self.server = KeyServer(self, paramiko.RSAKey(filename=keyfile), interface=interface) - conn = SSH2(timeout=1, connect_rate_limit=0, verify_fingerprint=False, **kwargs) + conn = SSH2(timeout=10, connect_rate_limit=0, verify_fingerprint=False, **kwargs) self.addCleanup(conn.close, force=True) conn.connect(HOST, self.server.port) return conn @@ -413,7 +422,7 @@ def testNoMethodLeftAfterAPartialSuccess(self): # A method that succeeded partially is not tried again. interface = type('PasswordTwice', (_PasswordTwice,), {'passwords': []}) server = KeyServer(self, paramiko.RSAKey(filename=keyfile), interface=interface) - conn = SSH2(timeout=1, connect_rate_limit=0, verify_fingerprint=False) + conn = SSH2(timeout=10, connect_rate_limit=0, verify_fingerprint=False) self.addCleanup(conn.close, force=True) conn.connect(HOST, server.port) with self.assertRaisesRegex( @@ -444,7 +453,7 @@ def testTriesAllAddresses(self): (socket.AF_INET6, socket.SOCK_STREAM, 6, '', ('::1', server.port, 0, 0)), # refused (socket.AF_INET, socket.SOCK_STREAM, 6, '', (HOST, server.port)), ] - conn = SSH2(timeout=1, connect_rate_limit=0, verify_fingerprint=False) + conn = SSH2(timeout=10, connect_rate_limit=0, verify_fingerprint=False) self.addCleanup(conn.close, force=True) with mock.patch('socket.getaddrinfo', return_value=addresses): conn.connect('router.example', server.port) @@ -475,7 +484,7 @@ class TerminalSizeTest(unittest.TestCase): def requestedSize(self, **kwargs): interface = type('Shell', (_Shell,), {'sizes': []}) server = KeyServer(self, paramiko.RSAKey(filename=keyfile), interface=interface) - conn = SSH2(timeout=1, connect_rate_limit=0, verify_fingerprint=False, **kwargs) + conn = SSH2(timeout=10, connect_rate_limit=0, verify_fingerprint=False, **kwargs) self.addCleanup(conn.close, force=True) conn.connect(HOST, server.port) conn.protocol_authenticate(Account('admin', 'password', needs_lock=False)) @@ -584,7 +593,7 @@ def serve(self, **attributes): return self.interface.requests def client(self, **kwargs): - conn = SSH2(timeout=1, connect_rate_limit=0, verify_fingerprint=False, **kwargs) + conn = SSH2(timeout=10, connect_rate_limit=0, verify_fingerprint=False, **kwargs) self.addCleanup(conn.close, force=True) conn.connect(HOST, self.server.port) return conn @@ -829,7 +838,7 @@ def setUp(self): def login(self, **kwargs): conn = SSH2( - timeout=1, + timeout=10, connect_rate_limit=0, verify_fingerprint=False, preferred_authentications=['password'], From 90e0dbbc9b8b1e279eb1a6d00ddbf5c066f5672f Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 04:08:11 +0100 Subject: [PATCH 223/250] Set SO_REUSEADDR only on the fixed ports of the test servers It lets a server bind a fixed port again at once after a restart. Port 0 gets a free port anyway, and on Windows, SO_REUSEADDR lets other sockets bind the same port. --- src/Exscript/servers/server.py | 5 ++++- tests/Exscript/servers/ServerTest.py | 19 +++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/src/Exscript/servers/server.py b/src/Exscript/servers/server.py index d4c7cb51..a4f0a81f 100644 --- a/src/Exscript/servers/server.py +++ b/src/Exscript/servers/server.py @@ -170,10 +170,13 @@ def start(self) -> None: Starts listening, and then the thread that accepts the connections. Clients can connect as soon as this returns. With port 0, the operating system chooses a free port; the port attribute has it then. + A fixed port is bound with SO_REUSEADDR, so that a restarted server + can take it at once. """ sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) try: - sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + if self.port: # port 0 gets a free port anyway + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) sock.bind((self.host, self.port)) sock.listen(1) except OSError: diff --git a/tests/Exscript/servers/ServerTest.py b/tests/Exscript/servers/ServerTest.py index 32d07f4e..3eda7724 100644 --- a/tests/Exscript/servers/ServerTest.py +++ b/tests/Exscript/servers/ServerTest.py @@ -121,6 +121,25 @@ def testStartOnAPortInUse(self): self.assertIsNone(server.socket) self.assertFalse(server.is_alive()) + def start(self, port): + server = Server('127.0.0.1', port, self.server.device) + self.addCleanup(server.to_parent.close) + self.addCleanup(server.to_child.close) + server.start() + self.addCleanup(server.join) + self.addCleanup(server.exit) + return server + + def testReuseOfTheAddress(self): + # Only a fixed port is reused, so that a restarted server can take it at once; port 0 + # gets a free port anyway. + server = self.start(0) + self.assertFalse(server.socket.getsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR)) + server.exit() + server.join() + restarted = self.start(server.port) + self.assertTrue(restarted.socket.getsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR)) + def suite(): loader = unittest.TestLoader() From 667fe660f8d724c647e936309fea9e7fd26a37ea Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 04:38:51 +0100 Subject: [PATCH 224/250] Try another port when a test server cannot listen on its free port WSL forwards its ports to Windows, so listen() fails now and then if Windows uses the port that Linux chose for port 0. With port 0, Server.start() now tries up to five ports. --- CHANGES.md | 4 ++++ src/Exscript/servers/server.py | 34 ++++++++++++++++++---------- tests/Exscript/servers/ServerTest.py | 19 ++++++++++++++++ 3 files changed, 45 insertions(+), 12 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index e1c79b6d..1be71bf3 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -788,3 +788,7 @@ The names below refer to functions callable from templates, not protocol methods returns, so clients can connect at once, and raises errors such as a port in use to the caller. Before, the thread bound the port a moment later. Port 0 chooses a free port, which the `port` attribute has after `start()`. +- **Port 0 on WSL:** If the port that the system chooses for port 0 is in use on the + Windows host of WSL, to which WSL forwards its ports, `start()` of `SSHd` and + `Telnetd` tries another one. Before, a start there failed now and then with + `OSError: [Errno 98] Address already in use`. diff --git a/src/Exscript/servers/server.py b/src/Exscript/servers/server.py index a4f0a81f..aeda2369 100644 --- a/src/Exscript/servers/server.py +++ b/src/Exscript/servers/server.py @@ -24,6 +24,7 @@ Base class for all servers. """ +import errno import logging import select import socket @@ -170,18 +171,27 @@ def start(self) -> None: Starts listening, and then the thread that accepts the connections. Clients can connect as soon as this returns. With port 0, the operating system chooses a free port; the port attribute has it then. - A fixed port is bound with SO_REUSEADDR, so that a restarted server - can take it at once. - """ - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - try: - if self.port: # port 0 gets a free port anyway - sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - sock.bind((self.host, self.port)) - sock.listen(1) - except OSError: - sock.close() - raise + If that port turns out to be in use, another one is tried. A fixed + port is bound with SO_REUSEADDR, so that a restarted server can take + it at once. + """ + # WSL forwards its ports to its Windows host, where the port that the + # system chose may be in use, so that listen() fails. + attempts = 1 if self.port else 5 + while True: + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + try: + if self.port: # port 0 gets a free port anyway + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + sock.bind((self.host, self.port)) + sock.listen(1) + break + except OSError as e: + sock.close() + attempts -= 1 + if e.errno != errno.EADDRINUSE or not attempts: + raise + self._dbg(1, 'The free port is in use, trying another one: %s' % e) self.socket = sock self.port = sock.getsockname()[1] self.name = '%s-%s:%s' % (self._get_server_type(), self.host, self.port) diff --git a/tests/Exscript/servers/ServerTest.py b/tests/Exscript/servers/ServerTest.py index 3eda7724..f320f9f2 100644 --- a/tests/Exscript/servers/ServerTest.py +++ b/tests/Exscript/servers/ServerTest.py @@ -1,6 +1,8 @@ +import errno import gc import re import socket +import types import unittest import unittest.mock import warnings @@ -140,6 +142,23 @@ def testReuseOfTheAddress(self): restarted = self.start(server.port) self.assertTrue(restarted.socket.getsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR)) + def testFreePortInUse(self): + # E.g. on WSL, which forwards the port to its Windows host, where it may be in use: + # start() tries another free port, but not forever. + class InUse(socket.socket): + def listen(self, *args): + raise OSError(errno.EADDRINUSE, 'Address already in use') + + module = types.SimpleNamespace(**vars(socket)) + with unittest.mock.patch('Exscript.servers.server.socket', module): + module.socket = unittest.mock.Mock(side_effect=lambda *args: InUse()) + self.assertRaises(OSError, self.start, 0) + self.assertGreater(module.socket.call_count, 1) + module.socket = unittest.mock.Mock(side_effect=[InUse(), socket.socket()]) + server = self.start(0) + self.assertEqual(module.socket.call_count, 2) + self.assertTrue(server.is_alive()) + def suite(): loader = unittest.TestLoader() From ce291fe93ffacee838aec538e28da50c9d00ceb2 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 04:59:49 +0100 Subject: [PATCH 225/250] Let the listeners of the tests take a port like the test servers They used socket.create_server(), whose listen() fails now and then on WSL like that of Server.start(). They now bind without SO_REUSEADDR and try up to five ports, as Server.start() does. --- tests/Exscript/protocols/SSH2Test.py | 3 ++- tests/Exscript/protocols/TelnetTest.py | 4 ++-- tests/Exscript/protocols/listener.py | 28 +++++++++++++++++++++++ tests/Exscript/protocols/replay.py | 4 +++- tests/Exscript/protocols/telnetlibTest.py | 4 +++- tests/Exscript/servers/ServerTest.py | 4 +++- 6 files changed, 41 insertions(+), 6 deletions(-) create mode 100644 tests/Exscript/protocols/listener.py diff --git a/tests/Exscript/protocols/SSH2Test.py b/tests/Exscript/protocols/SSH2Test.py index 6ce9ffa6..73ed55f0 100644 --- a/tests/Exscript/protocols/SSH2Test.py +++ b/tests/Exscript/protocols/SSH2Test.py @@ -19,6 +19,7 @@ from Exscript.protocols.ssh2 import _prefer_key_type from Exscript.servers import SSHd, sshd +from .listener import listen from .ProtocolTest import ProtocolTest keyfile = os.path.join(os.path.dirname(__file__), 'id_rsa') @@ -69,7 +70,7 @@ def __init__(self, test, *keys, interface=paramiko.ServerInterface): self.interface = interface self._transports = [] self._closing = threading.Event() - self._listener = socket.create_server((HOST, 0)) + self._listener = listen(HOST) self._listener.settimeout(0.1) self.port = self._listener.getsockname()[1] self._thread = threading.Thread(target=self._accept, daemon=True) diff --git a/tests/Exscript/protocols/TelnetTest.py b/tests/Exscript/protocols/TelnetTest.py index 5e106070..b13d58b2 100644 --- a/tests/Exscript/protocols/TelnetTest.py +++ b/tests/Exscript/protocols/TelnetTest.py @@ -1,6 +1,5 @@ import io import logging -import socket import threading import unittest from unittest import mock @@ -11,6 +10,7 @@ from Exscript.protocols.telnetlib import DO, IAC, NAWS, SB, SE, WILL from Exscript.servers import Telnetd +from .listener import listen from .ProtocolTest import ProtocolTest @@ -32,7 +32,7 @@ class TerminalSizeTest(unittest.TestCase): """The size sent with NAWS does not depend on the terminal that runs the script.""" def sentSize(self, **kwargs): - listener = socket.create_server(('127.0.0.1', 0)) + listener = listen() self.addCleanup(listener.close) received = bytearray() diff --git a/tests/Exscript/protocols/listener.py b/tests/Exscript/protocols/listener.py new file mode 100644 index 00000000..9b2a24a4 --- /dev/null +++ b/tests/Exscript/protocols/listener.py @@ -0,0 +1,28 @@ +""" +Listening sockets for the servers of the tests. +""" + +import errno +import socket + + +def listen(host='127.0.0.1', family=socket.AF_INET): + """ + Returns a socket that listens on a free port of the given address, like + Exscript.servers.Server.start() on port 0: without SO_REUSEADDR, unlike + socket.create_server(), and if the port that the system chose is in + use, e.g. on the Windows host of WSL, to which WSL forwards its ports, + on another one. + """ + attempts = 5 + while True: + sock = socket.socket(family, socket.SOCK_STREAM) + try: + sock.bind((host, 0)) + sock.listen() + return sock + except OSError as e: + sock.close() + attempts -= 1 + if e.errno != errno.EADDRINUSE or not attempts: + raise diff --git a/tests/Exscript/protocols/replay.py b/tests/Exscript/protocols/replay.py index 97f3b63a..1ae92f70 100644 --- a/tests/Exscript/protocols/replay.py +++ b/tests/Exscript/protocols/replay.py @@ -25,6 +25,8 @@ import paramiko +from .listener import listen + EOM = b']]>]]>' NETCONF_BASE_11 = b'urn:ietf:params:netconf:base:1.1' IAC, SB, SE = 255, 250, 240 @@ -151,7 +153,7 @@ def __init__(self, recording, chunks='recorded', seed=0): self._transports = [] self._session_threads = [] self._host_key = paramiko.ECDSAKey.generate() if recording.protocol != 'telnet' else None - self._listener = socket.create_server(('127.0.0.1', 0)) + self._listener = listen() self._listener.settimeout(0.1) self.port = self._listener.getsockname()[1] self._thread = threading.Thread(target=self._accept, daemon=True) diff --git a/tests/Exscript/protocols/telnetlibTest.py b/tests/Exscript/protocols/telnetlibTest.py index b3b19b84..6d86599a 100644 --- a/tests/Exscript/protocols/telnetlibTest.py +++ b/tests/Exscript/protocols/telnetlibTest.py @@ -9,6 +9,8 @@ from Exscript.protocols import telnetlib from Exscript.protocols.telnetlib import DO, DONT, IAC, NAWS, SB, SE, TTYPE, WILL, WONT, Telnet +from .listener import listen + ECHO, SGA, BINARY, NOP = 1, 3, 0, 241 @@ -161,7 +163,7 @@ def testSetWindowSize(self): def testOpen(self): for family, address in (socket.AF_INET, '127.0.0.1'), (socket.AF_INET6, '::1'): try: - listener = socket.create_server((address, 0), family=family) + listener = listen(address, family) except OSError: continue # no IPv6 on this host with listener: diff --git a/tests/Exscript/servers/ServerTest.py b/tests/Exscript/servers/ServerTest.py index f320f9f2..b10dd9ff 100644 --- a/tests/Exscript/servers/ServerTest.py +++ b/tests/Exscript/servers/ServerTest.py @@ -7,6 +7,8 @@ import unittest.mock import warnings +from protocols.listener import listen + from Exscript import Account from Exscript.emulators import VirtualDevice from Exscript.servers.server import Server @@ -111,7 +113,7 @@ def testHandleConnectionIsAbstract(self): self.assertRaises(NotImplementedError, self.server._handle_connection, None) def testStartOnAPortInUse(self): - with socket.create_server(('127.0.0.1', 0)) as listener: + with listen() as listener: server = Server('127.0.0.1', listener.getsockname()[1], self.server.device) self.addCleanup(server.to_parent.close) self.addCleanup(server.to_child.close) From 7d7edd53cc6055614db14414c7249116e4ff8553 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 05:01:42 +0100 Subject: [PATCH 226/250] Keep standard input open when interact() ends On a POSIX terminal, interact() read standard input through os.fdopen() in a with statement, which closed its file descriptor. It now passes closefd=False. --- CHANGES.md | 4 ++++ src/Exscript/protocols/protocol.py | 5 +++-- tests/Exscript/protocols/ExpectTest.py | 15 +++++++++++---- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 1be71bf3..e885c37e 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -222,6 +222,10 @@ marked with **BREAKING**; a migration guide will accompany the release. e.g. because its window was closed, `interact()` returns, or raises the error that ended the session, and restores the previous `SIGWINCH` handler. Before, it raised `termios.error` instead and kept its own `SIGWINCH` handler. +- **Standard Input Stays Open:** `interact()` on a POSIX terminal no longer closes the + file descriptor of standard input when it ends, so that the script can still read + from it, e.g. with `input()`. Before, `input()` then failed with + `OSError: [Errno 9] Bad file descriptor`, also in 2.6. ### Waiting for Responses diff --git a/src/Exscript/protocols/protocol.py b/src/Exscript/protocols/protocol.py index ac1ec83c..e67c8cd6 100644 --- a/src/Exscript/protocols/protocol.py +++ b/src/Exscript/protocols/protocol.py @@ -1977,8 +1977,9 @@ def _open_posix_shell( # Typed characters may take several bytes. decoder = codecs.getincrementaldecoder(sys.stdin.encoding or 'utf-8')(errors='replace') # We need to make sure to use an unbuffered stdin, else multi-byte - # chars (such as arrow keys) won't work properly. - with os.fdopen(sys.stdin.fileno(), 'rb', 0) as stdin: + # chars (such as arrow keys) won't work properly. The file descriptor + # is that of sys.stdin, so it stays open. + with os.fdopen(sys.stdin.fileno(), 'rb', 0, closefd=False) as stdin: oldtty = termios.tcgetattr(stdin) # Update the terminal size whenever the size changes. diff --git a/tests/Exscript/protocols/ExpectTest.py b/tests/Exscript/protocols/ExpectTest.py index d11650f5..cdd7ecc8 100644 --- a/tests/Exscript/protocols/ExpectTest.py +++ b/tests/Exscript/protocols/ExpectTest.py @@ -558,8 +558,9 @@ def interact(self, daemon_cls, conn, **kwargs): import tty # POSIX only - master, slave = os.openpty() # interact() closes the slave + master, slave = os.openpty() self.addCleanup(os.close, master) + self.addCleanup(os.close, slave) raw = threading.Event() setcbreak = tty.setcbreak @@ -597,7 +598,10 @@ def testSSH2(self): @unittest.skipUnless(hasattr(os, 'openpty'), 'interact() reads from a POSIX terminal') class TerminalTest(unittest.TestCase): - """interact() on a POSIX terminal: key handlers, and the size of the local terminal.""" + """ + interact() on a POSIX terminal: key handlers, the size of the local + terminal, and standard input, which stays open. + """ def interact(self, daemon_cls, conn, **kwargs): import tty # POSIX only @@ -617,8 +621,9 @@ def interact(self, daemon_cls, conn, **kwargs): received = [] conn.data_received_event.connect(received.append) - master, slave = os.openpty() # interact() closes the slave + master, slave = os.openpty() self.addCleanup(os.close, master) + self.addCleanup(os.close, slave) raw = threading.Event() setcbreak = tty.setcbreak @@ -642,6 +647,7 @@ def type_keys(): ): conn.interact({'\x19': handled.append}) typist.join() + self.assertTrue(os.isatty(slave)) # still open, e.g. for input() self.assertEqual(handled, [conn]) # ^Y was handled, not sent self.assertIn('hi!', ''.join(received)) self.assertEqual(conn.send_data.getvalue(), 'say hi\rexit\r') @@ -683,7 +689,8 @@ def interact(self, typed=b'', key_handlers=None): conn = Telnet(timeout=1, connect_rate_limit=0) self.addCleanup(conn.close, force=True) conn.connect(HOST, start_daemon(self, Telnetd, device)[1]) - master, slave = os.openpty() # interact() closes the slave + master, slave = os.openpty() + self.addCleanup(os.close, slave) self.terminal = os.fdopen(master, 'wb', 0) self.addCleanup(self.terminal.close) setcbreak = tty.setcbreak From 6f7dd295d038f47dbbe2cf62c7e94d762f535473 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 05:03:30 +0100 Subject: [PATCH 227/250] Restore the SIGWINCH handler if the first resize of interact() fails The first resize ran before the try whose finally restores the previous handler, so if it failed, e.g. on a closed channel, later resizes of the terminal raised the error wherever the script then was. --- CHANGES.md | 5 +++++ src/Exscript/protocols/protocol.py | 5 ++++- tests/Exscript/protocols/ExpectTest.py | 28 ++++++++++++++++++++++++++ 3 files changed, 37 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index e885c37e..0f44ccbd 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -226,6 +226,11 @@ marked with **BREAKING**; a migration guide will accompany the release. file descriptor of standard input when it ends, so that the script can still read from it, e.g. with `input()`. Before, `input()` then failed with `OSError: [Errno 9] Bad file descriptor`, also in 2.6. +- **`SIGWINCH` Handler After a Failed Resize:** If `interact()` fails to pass the size + of the terminal to the host at its start, e.g. because the host closed the SSH + channel, it restores the previous `SIGWINCH` handler. Before, its own handler stayed + installed, so that resizing the terminal later raised an exception wherever the + script was then. ### Waiting for Responses diff --git a/src/Exscript/protocols/protocol.py b/src/Exscript/protocols/protocol.py index e67c8cd6..7e237cec 100644 --- a/src/Exscript/protocols/protocol.py +++ b/src/Exscript/protocols/protocol.py @@ -1991,10 +1991,13 @@ def handle_sigwinch(signum: int | None, frame: FrameType | None) -> None: self._set_terminal_size(rows, cols) old_handler = signal.signal(signal.SIGWINCH, handle_sigwinch) - handle_sigwinch(None, None) # Read from stdin and write to the network, endlessly. try: + # The first resize is within the try, so that the handler is + # restored if it fails, e.g. on a channel that the host closed. + if handle_window_size: + handle_sigwinch(None, None) tty.setraw(sys.stdin.fileno()) tty.setcbreak(sys.stdin.fileno()) diff --git a/tests/Exscript/protocols/ExpectTest.py b/tests/Exscript/protocols/ExpectTest.py index cdd7ecc8..028623de 100644 --- a/tests/Exscript/protocols/ExpectTest.py +++ b/tests/Exscript/protocols/ExpectTest.py @@ -11,6 +11,8 @@ import unittest from unittest import mock +import paramiko + from Exscript import Account, PrivateKey from Exscript.emulators import VirtualDevice from Exscript.protocols import SSH2, Dummy, Telnet, drivers @@ -666,6 +668,32 @@ def testSSH2(self): ) self.interact(SSHd, conn, key=PrivateKey.from_file(KEYFILE)) + def testResizeOfAClosedChannel(self): + # The first resize fails after the host ended the session, but the SIGWINCH handler + # of interact() does not stay. + device = VirtualDevice(HOST, login_type=VirtualDevice.LOGIN_TYPE_NONE, strict=False) + daemon, port = start_daemon(self, SSHd, device, key=PrivateKey.from_file(KEYFILE)) + device.add_command('exit', daemon.exit_command) + conn = SSH2( + timeout=10, + connect_rate_limit=0, + verify_fingerprint=False, + preferred_authentications=['password'], + ) + self.addCleanup(conn.close, force=True) + conn.connect(HOST, port) + conn.protocol_authenticate(Account('user', 'password', needs_lock=False)) + conn.send('exit\r') + self.assertRaises(ConnectionClosedException, conn.expect, 'never') + master, slave = os.openpty() + self.addCleanup(os.close, master) + self.addCleanup(os.close, slave) + old_handler = signal.getsignal(signal.SIGWINCH) + self.addCleanup(signal.signal, signal.SIGWINCH, old_handler) + with mock.patch('sys.stdin', mock.Mock(encoding='utf-8', fileno=lambda: slave)): + self.assertRaisesRegex(paramiko.SSHException, 'Channel is not open', conn.interact) + self.assertEqual(signal.getsignal(signal.SIGWINCH), old_handler) + @unittest.skipUnless(hasattr(os, 'openpty'), 'interact() reads from a POSIX terminal') class ClosedTerminalTest(unittest.TestCase): From 0c37b4f85706e5cdcec870272157ce3309ee59b0 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 05:20:54 +0100 Subject: [PATCH 228/250] Stop the test servers within 0.1 seconds of exit() SSHd and Telnetd checked every 0.5 seconds whether to stop, so exit() and join() took up to a second. They now check every 0.05 seconds, which cuts the test suite from 192 s to 121 s. --- CHANGES.md | 4 ++++ src/Exscript/servers/server.py | 5 ++++- tests/Exscript/servers/ServerTest.py | 7 +++++++ 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 0f44ccbd..204d0b33 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -801,3 +801,7 @@ The names below refer to functions callable from templates, not protocol methods Windows host of WSL, to which WSL forwards its ports, `start()` of `SSHd` and `Telnetd` tries another one. Before, a start there failed now and then with `OSError: [Errno 98] Address already in use`. +- **Prompt Shutdown:** `SSHd` and `Telnetd` stop within about 0.1 seconds of `exit()`: + the server and the handlers of its clients check every 0.05 seconds whether to stop + (the `timeout` attribute). Before, they checked every 0.5 seconds, and `exit()` and + `join()` took up to a second. diff --git a/src/Exscript/servers/server.py b/src/Exscript/servers/server.py index aeda2369..31e62723 100644 --- a/src/Exscript/servers/server.py +++ b/src/Exscript/servers/server.py @@ -74,7 +74,10 @@ def __init__(self, host: str, port: int, device: VirtualDevice, encoding: str = self.name = '%s-%s:%s' % (self._get_server_type(), host, port) self.host = host self.port = int(port) - self.timeout = 0.5 + # How often, in seconds, the server and the handlers of its clients check + # whether to stop. Waking them at exit() instead would need a socket pair: + # select() cannot wait for the pipe on Windows. + self.timeout = 0.05 self.dbg = 0 self.running = False self.socket: socket.socket | None = None # set by start() diff --git a/tests/Exscript/servers/ServerTest.py b/tests/Exscript/servers/ServerTest.py index b10dd9ff..56f5e3cd 100644 --- a/tests/Exscript/servers/ServerTest.py +++ b/tests/Exscript/servers/ServerTest.py @@ -161,6 +161,13 @@ def listen(self, *args): self.assertEqual(module.socket.call_count, 2) self.assertTrue(server.is_alive()) + def testPromptExit(self): + # The server checks every 0.05 seconds whether to stop. + server = self.start(0) + server.exit() + server.join(0.4) + self.assertFalse(server.is_alive()) + def suite(): loader = unittest.TestLoader() From 0ff4468ef80483dde4e0caef9cf4d1df59050f79 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 05:46:04 +0100 Subject: [PATCH 229/250] Turn off the connect rate limit in SSH2Test and TelnetTest Their tests from ProtocolTest kept the default limit of 60 connects per minute. Without it, the test suite takes 87 s instead of 122 s. --- tests/Exscript/protocols/SSH2Test.py | 4 +++- tests/Exscript/protocols/TelnetTest.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/Exscript/protocols/SSH2Test.py b/tests/Exscript/protocols/SSH2Test.py index 73ed55f0..6b3d5fc0 100644 --- a/tests/Exscript/protocols/SSH2Test.py +++ b/tests/Exscript/protocols/SSH2Test.py @@ -36,7 +36,9 @@ def createDaemon(self): # Sessions that succeed do not wait for the timeout; tests of timeouts pass a short one. def createProtocol(self, timeout=10): - self.protocol = SSH2(timeout=timeout, debug=5, verify_fingerprint=False) + self.protocol = SSH2( + timeout=timeout, connect_rate_limit=0, debug=5, verify_fingerprint=False + ) def testConstructor(self): self.assertIsInstance(self.protocol, SSH2) diff --git a/tests/Exscript/protocols/TelnetTest.py b/tests/Exscript/protocols/TelnetTest.py index b13d58b2..30b05c3c 100644 --- a/tests/Exscript/protocols/TelnetTest.py +++ b/tests/Exscript/protocols/TelnetTest.py @@ -22,7 +22,7 @@ def createDaemon(self): self.daemon.dbg = 5 def createProtocol(self): - self.protocol = Telnet(timeout=1, debug=5) + self.protocol = Telnet(timeout=1, connect_rate_limit=0, debug=5) def testConstructor(self): self.assertIsInstance(self.protocol, Telnet) From 65b4961dbb2bb84e27672cbbee70f1eafa224ed5 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 05:58:12 +0100 Subject: [PATCH 230/250] Run the server tests that SSHdTest and TelnetdTest inherit They returned at once if the daemon was a base Server, which setUp() always creates, so they passed even for servers that answer nothing. SSHdTest now also passes the key of the tests to SSHd, which read the user's ~/.ssh/id_rsa. --- tests/Exscript/servers/SSHdTest.py | 4 ++-- tests/Exscript/servers/ServerTest.py | 16 +++++++++------- tests/Exscript/servers/TelnetdTest.py | 2 +- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/tests/Exscript/servers/SSHdTest.py b/tests/Exscript/servers/SSHdTest.py index 3a502650..5cd48a8a 100644 --- a/tests/Exscript/servers/SSHdTest.py +++ b/tests/Exscript/servers/SSHdTest.py @@ -24,10 +24,10 @@ class SSHdTest(ServerTest): CORRELATE = SSHd def _create_daemon(self): - self.daemon = SSHd(self.host, self.port, self.device) + self.daemon = SSHd(self.host, self.port, self.device, key=PrivateKey.from_file(KEYFILE)) def _create_client(self): - return SSH2(verify_fingerprint=False) + return SSH2(timeout=10, connect_rate_limit=0, verify_fingerprint=False) class _ImpatientServer(sshd._ParamikoServer): diff --git a/tests/Exscript/servers/ServerTest.py b/tests/Exscript/servers/ServerTest.py index 56f5e3cd..93586910 100644 --- a/tests/Exscript/servers/ServerTest.py +++ b/tests/Exscript/servers/ServerTest.py @@ -44,20 +44,21 @@ def _add_commands(self): def testConstructor(self): # Test can not work on the abstract base. - if self.daemon.__class__ == Server: - return + if type(self) is ServerTest: + self.skipTest('the base of SSHdTest and TelnetdTest') self._create_daemon() self.daemon.start() def testStart(self): # Test can not work on the abstract base. - if self.daemon.__class__ == Server: - return + if type(self) is ServerTest: + self.skipTest('the base of SSHdTest and TelnetdTest') self._create_daemon() self._add_commands() self.daemon.start() client = self._create_client() + self.addCleanup(client.close, force=True) client.set_prompt(re.compile(r'\w+:\d+> ?')) client.connect(self.host, self.daemon.port) client.login(Account('user', 'password')) @@ -72,13 +73,14 @@ def testExitCommand(self): def testExit(self): # Test can not work on the abstract base. - if self.daemon.__class__ == Server: - return + if type(self) is ServerTest: + self.skipTest('the base of SSHdTest and TelnetdTest') self.testStart() # Since testStart() sent an "exit" command to the server, # it should be shutting down even without us calling # self.daemon.exit(). - self.daemon.join() + self.daemon.join(5) + self.assertFalse(self.daemon.is_alive()) self.testConstructor() diff --git a/tests/Exscript/servers/TelnetdTest.py b/tests/Exscript/servers/TelnetdTest.py index 088a42a9..74efc13d 100644 --- a/tests/Exscript/servers/TelnetdTest.py +++ b/tests/Exscript/servers/TelnetdTest.py @@ -18,7 +18,7 @@ def _create_daemon(self): self.daemon = Telnetd(self.host, self.port, self.device) def _create_client(self): - return Telnet() + return Telnet(timeout=10, connect_rate_limit=0) class TelnetdSessionTest(unittest.TestCase): From b0c7617bf6d7ea32b081b4408fc8fdf5944edc65 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 06:04:30 +0100 Subject: [PATCH 231/250] Give SSHd a host key of its own instead of the user's key Without key=, SSHd read the user's private key ~/.ssh/id_rsa, which is often missing or needs a passphrase. It now creates a new ECDSA key for each server; key= still sets a fixed one. --- CHANGES.md | 4 ++++ docs/migration.rst | 3 +++ src/Exscript/servers/sshd.py | 14 +++++++------- tests/Exscript/servers/SSHdTest.py | 14 ++++++++++---- 4 files changed, 24 insertions(+), 11 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 204d0b33..592ab707 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -790,6 +790,10 @@ The names below refer to functions callable from templates, not protocol methods authentication never worked on Python 3: every login with a valid password failed with a `TypeError`. Use `http.server` from the standard library or a web framework instead. +- **BREAKING - Host Key of `SSHd`:** Without `key`, `SSHd` creates an ECDSA host key of + its own, new for each server. Before, it read the private key of the user, + `~/.ssh/id_rsa`, and failed where that file was missing, e.g. with only an Ed25519 key + or in CI, or needed a passphrase. For a fixed host key, pass `key`. - **Client Handling:** `SSHd` and `Telnetd` keep listening after a client disconnects, allowing sequential connections. They share the supplied emulator device; concurrent clients are not supported. diff --git a/docs/migration.rst b/docs/migration.rst index 92d51f95..cd80405a 100644 --- a/docs/migration.rst +++ b/docs/migration.rst @@ -247,6 +247,9 @@ Emulators and test servers - The test servers ``SSHd`` and ``Telnetd`` run as threads; stop them with ``exit()`` and ``join()``. Their ``buf`` attribute and the ``encoding`` argument of ``Telnetd`` are removed. +- ``SSHd`` without ``key`` creates a host key of its own instead of using + ``~/.ssh/id_rsa``. To keep a fixed host key, e.g. for ``known_hosts``, + pass ``key=PrivateKey.from_file(...)``. - ``Exscript.servers.HTTPd`` and its ``RequestHandler`` are removed; their digest authentication never worked on Python 3. Use ``http.server`` from the standard library or a web framework. diff --git a/src/Exscript/servers/sshd.py b/src/Exscript/servers/sshd.py index adc2a43e..159eb8f7 100644 --- a/src/Exscript/servers/sshd.py +++ b/src/Exscript/servers/sshd.py @@ -26,7 +26,6 @@ import base64 import logging -import os import socket import threading @@ -142,15 +141,16 @@ def __init__( encoding, which is UTF-8. :param key: The host key; the server reads the RSA key from its file. - Defaults to the key in ~/.ssh/id_rsa. + Without it, the server creates an ECDSA key of its own, which is + new for each server. """ Server.__init__(self, host, port, device) self._logger = _logger - if key: - keyfile = key.get_filename() - else: - keyfile = os.path.expanduser('~/.ssh/id_rsa') - self.host_key = paramiko.RSAKey(filename=keyfile) + # A device emulator has no use for the private key of the user, which + # may be missing or need a passphrase. + self.host_key: paramiko.PKey = ( + paramiko.RSAKey(filename=key.get_filename()) if key else paramiko.ECDSAKey.generate() + ) def _recvline(self, channel: paramiko.Channel, buf: bytes) -> tuple[str | None, bytes]: """ diff --git a/tests/Exscript/servers/SSHdTest.py b/tests/Exscript/servers/SSHdTest.py index 5cd48a8a..c954b507 100644 --- a/tests/Exscript/servers/SSHdTest.py +++ b/tests/Exscript/servers/SSHdTest.py @@ -69,13 +69,19 @@ def connect(self): return conn def testDefaultKey(self): + # Without a key, the server creates its own, and does not read the + # private key of the user, which may be missing. home = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, home) - os.mkdir(os.path.join(home, '.ssh')) - shutil.copy(KEYFILE, os.path.join(home, '.ssh', 'id_rsa')) with mock.patch.dict(os.environ, {'HOME': home, 'USERPROFILE': home}): - daemon = SSHd('127.0.0.1', 0, self.device) - self.assertEqual(daemon.host_key, paramiko.RSAKey(filename=KEYFILE)) + self.daemon = SSHd('127.0.0.1', 0, self.device) + self.assertIsInstance(self.daemon.host_key, paramiko.ECDSAKey) + self.assertNotEqual(self.daemon.host_key, SSHd('127.0.0.1', 0, self.device).host_key) + self.device.add_command('ls', 'file') + self.start() + conn = self.connect() + conn.execute('ls') + self.assertEqual(conn.response, 'file') def testOnlySessionChannels(self): server = sshd._ParamikoServer() From 52cd679fb3961721c4c9de568f0b1404a94099d7 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 11:22:53 +0100 Subject: [PATCH 232/250] Link the ncclient issue of the NETCONF framing workaround --- src/Exscript/protocols/netconf.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Exscript/protocols/netconf.py b/src/Exscript/protocols/netconf.py index 69ba4a9d..bd447315 100644 --- a/src/Exscript/protocols/netconf.py +++ b/src/Exscript/protocols/netconf.py @@ -86,8 +86,9 @@ class _Framer: 1.1). A character whose bytes arrive in two reads, or that a device splits across two chunks, raises a UnicodeDecodeError in the IO thread, which ends the session. Remove this class when ncclient frames the - messages on bytes, and let _init_ncclient() keep the parser of ncclient - again (session.parser = device_handler.get_xml_parser(session)). + messages on bytes (https://github.com/ncclient/ncclient/issues/671), and + let _init_ncclient() keep the parser of ncclient again + (session.parser = device_handler.get_xml_parser(session)). """ def __init__(self, session: Any) -> None: From 56bb41e04f2446f4a015c3413f8b67a7db4efbdb Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 11:22:53 +0100 Subject: [PATCH 233/250] Read the output of the exscript command as UTF-8 on Windows Before Python 3.15, a pipe on Windows gets the locale encoding, so the copyright sign in the help did not decode as UTF-8. --- tests/Exscript/ScriptsTest.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/Exscript/ScriptsTest.py b/tests/Exscript/ScriptsTest.py index fcedbd73..ced79fe8 100644 --- a/tests/Exscript/ScriptsTest.py +++ b/tests/Exscript/ScriptsTest.py @@ -16,8 +16,13 @@ class ScriptsTest(unittest.TestCase): def setUp(self): self.tmpdir = tempfile.TemporaryDirectory() self.addCleanup(self.tmpdir.cleanup) + # UTF-8 output also where the locale encoding is another, e.g. on Windows. self.env = dict( - os.environ, HOME=self.tmpdir.name, USERPROFILE=self.tmpdir.name, NO_COLOR='1' + os.environ, + HOME=self.tmpdir.name, + USERPROFILE=self.tmpdir.name, + NO_COLOR='1', + PYTHONIOENCODING='utf-8', ) def run_exscript(self, *args): From 60b5d872433f393626e3b99b31ade6219f43c4b8 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 11:49:56 +0100 Subject: [PATCH 234/250] Close the connection to the SSH agent after the login Each login that tried the agent left its socket open until the garbage collector closed it, with a ResourceWarning. --- CHANGES.md | 4 + src/Exscript/protocols/ssh2.py | 33 ++++---- tests/Exscript/protocols/SSH2Test.py | 117 +++++++++++++++++++++++++++ 3 files changed, 140 insertions(+), 14 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 592ab707..e20ad8fa 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -196,6 +196,10 @@ marked with **BREAKING**; a migration guide will accompany the release. - **SSH Keys Without Files:** `PrivateKey.from_string()` creates a key from its text, in the format of a key file, e.g. a key from a secret store, which then never needs to be written to a file (upstream issue #209). The type is detected unless it is given. +- **SSH Agent Connection Closed:** `SSH2` and `Netconf` close their connection to the + SSH agent (`SSH_AUTH_SOCK`) as soon as the login is done with its keys. Before, the + garbage collector closed it, with a `ResourceWarning`, and after a login with a key + of the agent not before the end of the session. - **BREAKING - Terminal Size:** SSH and Telnet sessions get a terminal of 511 columns and 1000 lines; the new options `term_width` and `term_height` (also `Host` options) choose another size. Before, Exscript used the size of the terminal that diff --git a/src/Exscript/protocols/ssh2.py b/src/Exscript/protocols/ssh2.py index 18bf599f..05091ab2 100644 --- a/src/Exscript/protocols/ssh2.py +++ b/src/Exscript/protocols/ssh2.py @@ -287,26 +287,31 @@ def _paramiko_auth_agent(self, username: str, password: str | None = None) -> li Authenticates with the keys of the SSH agent, one after the other until the server accepts one; the password is not used. Raises the error of the last key, or AuthenticationException if the agent has no - keys. + keys. Closes the connection to the agent before it returns or raises. :return: After a partial success, the authentication types that the server allows next; else an empty list. """ - keys = paramiko.Agent().get_keys() - if not keys: - raise AuthenticationException('auth agent found no keys') + agent = paramiko.Agent() + try: + keys = agent.get_keys() + if not keys: + raise AuthenticationException('auth agent found no keys') - saved_exception: SSHException = AuthenticationException( - 'Failed to authenticate with given username' - ) + saved_exception: SSHException = AuthenticationException( + 'Failed to authenticate with given username' + ) - for key in keys: - try: - self._dbg(2, 'Trying SSH agent key %s' % key.fingerprint) - return cast(paramiko.Transport, self._transport).auth_publickey(username, key) - except SSHException as e: - saved_exception = e - raise saved_exception + for key in keys: + try: + self._dbg(2, 'Trying SSH agent key %s' % key.fingerprint) + return cast(paramiko.Transport, self._transport).auth_publickey(username, key) + except SSHException as e: + saved_exception = e + raise saved_exception + finally: + # Not before: the keys sign through this connection. + agent.close() def _paramiko_auth_key( self, username: str, keys: Iterable[tuple[type[paramiko.PKey], str]], password: str | None diff --git a/tests/Exscript/protocols/SSH2Test.py b/tests/Exscript/protocols/SSH2Test.py index 6b3d5fc0..dadca8ff 100644 --- a/tests/Exscript/protocols/SSH2Test.py +++ b/tests/Exscript/protocols/SSH2Test.py @@ -1,3 +1,4 @@ +import contextlib import io import logging import os @@ -5,9 +6,12 @@ import shutil import socket import stat +import struct +import sys import tempfile import threading import unittest +import warnings from unittest import mock import paramiko @@ -827,6 +831,118 @@ def testNoShell(self): self.login() +class FakeAgent: + """ + An SSH agent on a UNIX socket that holds the given keys. It records the + type of each request, and sets the event closed when a client closes its + connection. + """ + + def __init__(self, test, *keys): + self.keys = keys + self.requests = [] + self.closed = threading.Event() + tmpdir = tempfile.TemporaryDirectory() + test.addCleanup(tmpdir.cleanup) + self.path = os.path.join(tmpdir.name, 'agent') + self._listener = socket.socket(socket.AF_UNIX) + self._listener.bind(self.path) + self._listener.listen() + self._listener.settimeout(0.1) + self._connections = [] + self._threads = [] + self._closing = threading.Event() + self._thread = threading.Thread(target=self._accept, daemon=True) + self._thread.start() + test.addCleanup(self.close) + + def _accept(self): + while not self._closing.is_set(): + try: + conn, _ = self._listener.accept() + except TimeoutError: + continue + self._connections.append(conn) + thread = threading.Thread(target=self._serve, args=(conn,), daemon=True) + self._threads.append(thread) + thread.start() + + def _serve(self, conn): + with conn, conn.makefile('rb') as fp: + while header := fp.read(4): + request = paramiko.Message(fp.read(struct.unpack('>I', header)[0])) + kind = request.get_byte()[0] + self.requests.append(kind) + reply = paramiko.Message() + if kind == 11: # SSH2_AGENTC_REQUEST_IDENTITIES + reply.add_byte(bytes([12])) + reply.add_int(len(self.keys)) + for key in self.keys: + reply.add_string(key.asbytes()) + reply.add_string('test key') + elif kind == 13: # SSH2_AGENTC_SIGN_REQUEST + blob, data = request.get_binary(), request.get_binary() + [key] = [key for key in self.keys if key.asbytes() == blob] + reply.add_byte(bytes([14])) + reply.add_string(key.sign_ssh_data(data).asbytes()) + else: + reply.add_byte(bytes([5])) # SSH_AGENT_FAILURE + conn.sendall(struct.pack('>I', len(reply.asbytes())) + reply.asbytes()) + self.closed.set() + + def close(self): + self._closing.set() + self._thread.join() + self._listener.close() + for conn in self._connections: + with contextlib.suppress(OSError): # closed already + conn.shutdown(socket.SHUT_RDWR) + for thread in self._threads: + thread.join() + + +@unittest.skipIf( + sys.platform == 'win32' or not hasattr(socket, 'AF_UNIX'), + 'paramiko finds the agent through SSH_AUTH_SOCK only with UNIX sockets', +) +class AgentTest(unittest.TestCase): + """The connection to the SSH agent is closed when the login is done with its keys.""" + + def setUp(self): + home = tempfile.TemporaryDirectory() # no key files + self.addCleanup(home.cleanup) + self.home = home.name + + def environ(self, agent): + env = {'SSH_AUTH_SOCK': agent.path, 'HOME': self.home, 'USERPROFILE': self.home} + return mock.patch.dict(os.environ, env) + + def testLoginWithAnAgentKey(self): + device = VirtualDevice(HOST, login_type=VirtualDevice.LOGIN_TYPE_NONE, strict=False) + daemon = SSHd(HOST, 0, device, key=key) + daemon.start() + self.addCleanup(daemon.join) + self.addCleanup(daemon.exit) + agent = FakeAgent(self, paramiko.ECDSAKey.generate()) + conn = SSH2(timeout=10, connect_rate_limit=0, verify_fingerprint=False) + self.addCleanup(conn.close, force=True) + with self.environ(agent): + conn.connect(HOST, daemon.port) + conn.login(Account('user', PASSWORD, needs_lock=False)) + self.assertEqual(agent.requests, [11, 13]) # the key signed the login + self.assertTrue(agent.closed.wait(5)) # while the session is open + + def testAgentWithoutKeys(self): + agent = FakeAgent(self) + conn = SSH2() + with self.environ(agent), warnings.catch_warnings(record=True) as caught: + warnings.simplefilter('always', ResourceWarning) + with self.assertRaisesRegex(paramiko.AuthenticationException, 'found no keys'): + conn._paramiko_auth_agent('admin') + self.assertEqual([str(warning.message) for warning in caught], []) # closed, not collected + self.assertTrue(agent.closed.wait(5)) + + class SessionTest(unittest.TestCase): """Sending to and closing a session that the host ended or that has pending output.""" @@ -897,6 +1013,7 @@ def suite(): loader.loadTestsFromTestCase(ConnectTest), loader.loadTestsFromTestCase(TerminalSizeTest), loader.loadTestsFromTestCase(AuthenticationTest), + loader.loadTestsFromTestCase(AgentTest), loader.loadTestsFromTestCase(SessionTest), ] ) From 02042fa42dd4c5d7efa69a02d5819ae40220d733 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 11:56:22 +0100 Subject: [PATCH 235/250] Keep the prompt after login(flush=False) with nxos, bigip and rios login() consumed the prompt before it called the auto_authorize() of these drivers, which sends nothing and so waits for no new prompt. --- CHANGES.md | 7 +++++++ docs/drivers.rst | 4 +++- docs/migration.rst | 6 ++++++ src/Exscript/protocols/drivers/bigip.py | 6 ++++++ src/Exscript/protocols/drivers/driver.py | 6 +++++- src/Exscript/protocols/drivers/nxos.py | 6 ++++++ src/Exscript/protocols/drivers/rios.py | 6 ++++++ src/Exscript/protocols/protocol.py | 7 ++++++- tests/Exscript/protocols/drivers/bigipTest.py | 7 +++++++ tests/Exscript/protocols/drivers/iosTest.py | 8 ++++++++ tests/Exscript/protocols/drivers/nxosTest.py | 7 +++++++ tests/Exscript/protocols/drivers/riosTest.py | 7 +++++++ 12 files changed, 74 insertions(+), 3 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index e20ad8fa..570f9a11 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -466,6 +466,13 @@ development versions; "before" refers to them. - **`disable_driver()`:** `Exscript.protocols.drivers.disable_driver()` works; it raised a `TypeError` after it had removed the driver only from `driver_map`, so the OS guesser still chose it. +- **BREAKING - `login(flush=False)` With `nxos`, `bigip` and `rios`:** + `login(flush=False)` leaves the prompt in the buffer with these drivers, as with the + others. Their `supports_auto_authorize()` returns `False`, as their + `auto_authorize()` sends nothing. Before, `login()` consumed the prompt before it + called their `auto_authorize()`, so that e.g. `expect_prompt()` after the login + waited until the timeout. To migrate, consume the prompt before the first command, + e.g. with `expect_prompt()`, or omit `flush=False`. #### Class `Exscript.protocols.drivers.Driver` diff --git a/docs/drivers.rst b/docs/drivers.rst index 7f2b14ac..fb16918e 100644 --- a/docs/drivers.rst +++ b/docs/drivers.rst @@ -143,7 +143,9 @@ And it can override these methods: ``auto_authorize(conn, account, flush, bailout)`` Gets the privileges that the commands need, for example with ``enable`` on IOS. ``login()`` calls it through - ``conn.auto_app_authorize()`` after the authentication. + ``conn.auto_app_authorize()`` after the authentication. If the devices + need no such command, override it with a method that does nothing, and + ``supports_auto_authorize()`` with one that returns ``False``. ``logout(conn)`` Ends the session; ``close()`` calls it. The default sends ``exit``. diff --git a/docs/migration.rst b/docs/migration.rst index cd80405a..242affd3 100644 --- a/docs/migration.rst +++ b/docs/migration.rst @@ -105,6 +105,12 @@ NX-OS errors commands that NX-OS rejects with them; 2.x returned their response. Catch the exception where a command may fail. +Login without flush + With the ``nxos``, ``bigip`` and ``rios`` drivers, + ``login(flush=False)`` leaves the prompt in the buffer, as with the + other drivers; 2.x consumed it. Consume it, e.g. with + ``expect_prompt()``, before the first command, or omit ``flush=False``. + Templates --------- diff --git a/src/Exscript/protocols/drivers/bigip.py b/src/Exscript/protocols/drivers/bigip.py index 546fa5d6..09cec778 100644 --- a/src/Exscript/protocols/drivers/bigip.py +++ b/src/Exscript/protocols/drivers/bigip.py @@ -75,6 +75,12 @@ def init_terminal(self, conn: 'Protocol') -> None: """ conn.execute('modify cli preference pager disabled') + def supports_auto_authorize(self) -> bool: + """ + Returns False, as auto_authorize() sends nothing. + """ + return False + def auto_authorize( self, conn: 'Protocol', diff --git a/src/Exscript/protocols/drivers/driver.py b/src/Exscript/protocols/drivers/driver.py index 78a285e9..8d3b12b8 100644 --- a/src/Exscript/protocols/drivers/driver.py +++ b/src/Exscript/protocols/drivers/driver.py @@ -242,7 +242,11 @@ def init_terminal(self, conn: 'Protocol') -> None: def supports_auto_authorize(self) -> bool: """ - Returns whether the driver overrides auto_authorize(). + Returns whether auto_authorize() sends a command that starts the + authorization, such as "enable", so that login() consumes the prompt + before it calls auto_authorize(). By default, this is whether the + driver overrides auto_authorize(); a driver whose auto_authorize() + sends nothing returns False. """ return self.__class__.auto_authorize is not Driver.auto_authorize diff --git a/src/Exscript/protocols/drivers/nxos.py b/src/Exscript/protocols/drivers/nxos.py index cd9422e1..7b6b9817 100644 --- a/src/Exscript/protocols/drivers/nxos.py +++ b/src/Exscript/protocols/drivers/nxos.py @@ -116,6 +116,12 @@ def init_terminal(self, conn: 'Protocol') -> None: """ conn.execute('term len 0') + def supports_auto_authorize(self) -> bool: + """ + Returns False, as auto_authorize() sends nothing. + """ + return False + def auto_authorize( self, conn: 'Protocol', diff --git a/src/Exscript/protocols/drivers/rios.py b/src/Exscript/protocols/drivers/rios.py index 30a29283..99078734 100644 --- a/src/Exscript/protocols/drivers/rios.py +++ b/src/Exscript/protocols/drivers/rios.py @@ -70,6 +70,12 @@ def init_terminal(self, conn: 'Protocol') -> None: conn.execute('term len 0') conn.execute('enable') + def supports_auto_authorize(self) -> bool: + """ + Returns False, as auto_authorize() sends nothing. + """ + return False + def auto_authorize( self, conn: 'Protocol', diff --git a/src/Exscript/protocols/protocol.py b/src/Exscript/protocols/protocol.py index 7e237cec..06e3c87d 100644 --- a/src/Exscript/protocols/protocol.py +++ b/src/Exscript/protocols/protocol.py @@ -1184,7 +1184,12 @@ def login( if driver.supports_auto_authorize(): self._dbg(4, '%s supports auto-authorization.' % repr(driver)) self.expect_prompt() - self.auto_app_authorize(app_account, flush=flush) + self.auto_app_authorize(app_account, flush=flush) + else: + # The driver may do nothing, so the prompt is consumed here. + self.auto_app_authorize(app_account, flush=False) + if flush: + self.expect_prompt() def authenticate( self, diff --git a/tests/Exscript/protocols/drivers/bigipTest.py b/tests/Exscript/protocols/drivers/bigipTest.py index 27f6bc5a..db9d7da9 100644 --- a/tests/Exscript/protocols/drivers/bigipTest.py +++ b/tests/Exscript/protocols/drivers/bigipTest.py @@ -34,6 +34,13 @@ def testNoAuthorization(self): conn.auto_app_authorize(ACCOUNT) self.assertEqual(device.received, []) + def testLoginWithoutFlush(self): + # The prompt stays in the buffer. + device = Device(PROMPT) + conn = connect(self, device, 'bigip', prompt=False) + conn.login(ACCOUNT, flush=False) + self.assertEqual(conn.expect_prompt()[1].group(), PROMPT) + def testInitTerminal(self): device = Device(PROMPT) conn = connect(self, device, 'bigip') diff --git a/tests/Exscript/protocols/drivers/iosTest.py b/tests/Exscript/protocols/drivers/iosTest.py index f749bcaf..3ce6bad9 100644 --- a/tests/Exscript/protocols/drivers/iosTest.py +++ b/tests/Exscript/protocols/drivers/iosTest.py @@ -46,6 +46,14 @@ def testLoginAndEnable(self): self.assertEqual(device.received, ['admin', 'login-secret', 'enable', 'enable-secret']) self.assertTrue(conn.is_app_authorized()) + def testLoginAndEnableWithoutFlush(self): + # The prompt after "enable" stays in the buffer. + device = Device('Router>', login=('Username: ', 'Password: '), enabled_prompt='Router#') + conn = connect(self, device, 'ios', prompt=False) + conn.login(ACCOUNT, flush=False) + self.assertEqual(conn.expect_prompt()[1].group().strip(), 'Router#') + self.assertEqual(device.received, ['admin', 'login-secret', 'enable', 'enable-secret']) + def testInitTerminal(self): device = Device('Router#') conn = connect(self, device, 'ios') diff --git a/tests/Exscript/protocols/drivers/nxosTest.py b/tests/Exscript/protocols/drivers/nxosTest.py index e4dfe7d3..ba5ff971 100644 --- a/tests/Exscript/protocols/drivers/nxosTest.py +++ b/tests/Exscript/protocols/drivers/nxosTest.py @@ -78,6 +78,13 @@ def testLoginWithoutEnable(self): conn.auto_app_authorize(ACCOUNT) self.assertEqual(device.received, ['admin', 'login-secret']) + def testLoginWithoutFlush(self): + # The prompt stays in the buffer. + device = Device('N7K-1# ', login=('Username: ', 'Password: ')) + conn = connect(self, device, 'nxos', prompt=False) + conn.login(ACCOUNT, flush=False) + self.assertEqual(conn.expect_prompt()[1].group().strip(), 'N7K-1#') + def testLoginPrompt(self): # The login works at each user name prompt of NX-OS, also without the host name. for prompt in 'switch login: ', 'login: ', ' login: ', 'Username: ': diff --git a/tests/Exscript/protocols/drivers/riosTest.py b/tests/Exscript/protocols/drivers/riosTest.py index 193ad775..504e4294 100644 --- a/tests/Exscript/protocols/drivers/riosTest.py +++ b/tests/Exscript/protocols/drivers/riosTest.py @@ -32,6 +32,13 @@ def testNoAuthorization(self): conn.auto_app_authorize(ACCOUNT) self.assertEqual(device.received, []) + def testLoginWithoutFlush(self): + # The prompt stays in the buffer. + device = Device('amnesiac > ') + conn = connect(self, device, 'rios', prompt=False) + conn.login(ACCOUNT, flush=False) + self.assertEqual(conn.expect_prompt()[1].group().strip(), 'amnesiac >') + def testErrors(self): device = Device('amnesiac # ') device.add_command( From ebde3972e9fad8870dfaafaf7c5b7de18e032221 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 12:01:26 +0100 Subject: [PATCH 236/250] Silence paramiko per connection instead of disabling its logger Below debug level 5, SSH2 disabled the logger paramiko.transport, which silenced paramiko in the whole process, also for other code. --- CHANGES.md | 5 +++++ src/Exscript/protocols/ssh2.py | 8 +++++++- tests/Exscript/protocols/SSH2Test.py | 16 ++++++++++++++++ 3 files changed, 28 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 570f9a11..95f65a28 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -163,6 +163,11 @@ marked with **BREAKING**; a migration guide will accompany the release. - **SSH Key Fingerprints in Debug Logs:** The keys that are tried for a login are logged with their SHA256 fingerprints, as `ssh-add -l` and `ssh-keygen -l` show them. Before, the MD5 fingerprint was logged as a Python bytes literal. +- **paramiko's Logger Stays On:** `SSH2` and `Netconf` connections with a debug level + below 5 keep paramiko's messages about them out of the logs with a logger of their + own, `Exscript.protocols.ssh2.paramiko`, which logs nothing. Before, they turned off + paramiko's logger `paramiko.transport` for the whole process, so that other code that + uses paramiko, and connections with debug level 5, logged nothing either. - **SSH Banners in Another Encoding:** An authentication banner or a version string that is not valid in the encoding of the connection no longer fails the login; the invalid bytes become U+FFFD. Before, e.g. a latin-1 banner with `encoding='utf-8'` diff --git a/src/Exscript/protocols/ssh2.py b/src/Exscript/protocols/ssh2.py index 05091ab2..5bf3e072 100644 --- a/src/Exscript/protocols/ssh2.py +++ b/src/Exscript/protocols/ssh2.py @@ -49,6 +49,8 @@ _logger = logging.getLogger(__name__) _logger.addHandler(logging.NullHandler()) +# paramiko's log channel for connections with a debug level below 5; it logs nothing. +_QUIET_LOG_CHANNEL = __name__ + '.paramiko' _known_hosts_lock = threading.Lock() _READ_SIZE = 65536 @@ -95,6 +97,8 @@ def __init__(self, **kwargs: Any) -> None: Constructor. Takes the keyword arguments of :class:`Protocol`. If preferred_authentications is empty or not given, the methods publickey, keyboard-interactive and password are tried, in this order. + From debug level 5 on, the logger paramiko.transport logs paramiko's + messages about the connection, too. """ Protocol.__init__(self, **kwargs) # Module-Level @@ -206,7 +210,9 @@ def _paramiko_connect(self) -> paramiko.Transport: t.banner_timeout = self.banner_timeout t.name = cast(str, self.host) # connect() checks that the host is set if self.debug < 5: - t.logger.disabled = True + # Not paramiko's own channel, whose level would apply to all users of paramiko. + t.set_log_channel(_QUIET_LOG_CHANNEL) + t.logger.setLevel(logging.CRITICAL + 1) # No packet dumps (paramiko's set_hexdump()): they show passwords in clear text. # The known keys are loaded before the key exchange, so that the key is diff --git a/tests/Exscript/protocols/SSH2Test.py b/tests/Exscript/protocols/SSH2Test.py index dadca8ff..171ec804 100644 --- a/tests/Exscript/protocols/SSH2Test.py +++ b/tests/Exscript/protocols/SSH2Test.py @@ -660,6 +660,22 @@ def testNoPacketDumps(self): conn.close(force=True) self.assertEqual([line for line in logs.output if PASSWORD in line], []) + def testParamikoLogsFromDebugLevel5(self): + # paramiko's own logger keeps its state, so other users of paramiko, such as the + # server, still log. + self.serve() + shared = logging.getLogger('paramiko.transport') + self.addCleanup(setattr, shared, 'disabled', shared.disabled) + for debug in 0, 4, 5: + with self.subTest(debug=debug): + with self.assertLogs(level=logging.DEBUG) as logs: + self.login(debug=debug).close(force=True) + self.assertFalse(shared.disabled) + messages = [(record.name, record.getMessage()) for record in logs.records] + client = [name for name, message in messages if 'client mode' in message] + self.assertEqual(client, ['paramiko.transport'] if debug == 5 else []) + self.assertTrue(any('server mode' in message for _, message in messages)) + def testKeyboardInteractiveWithAnEmptyRound(self): # Some servers send a round without prompts first. rounds = [[], [('Password: ', False)]] From 1115688181164ea61eb1b8fba7449204a75e8973 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 12:03:45 +0100 Subject: [PATCH 237/250] Remove the unused good_pub_key of the SSHd server interface The server accepts any public key. --- src/Exscript/servers/sshd.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/src/Exscript/servers/sshd.py b/src/Exscript/servers/sshd.py index 159eb8f7..94dbe9dd 100644 --- a/src/Exscript/servers/sshd.py +++ b/src/Exscript/servers/sshd.py @@ -24,7 +24,6 @@ An SSH2 server. """ -import base64 import logging import socket import threading @@ -49,17 +48,6 @@ class _ParamikoServer(ServerInterface): password, any public key, and any shell and pseudo-terminal request. """ - # Must match the public key of the test suite! - data = ( - b'AAAAB3NzaC1yc2EAAAADAQABAAABAQChbnHaPrmxa5NpvtKXSMfMJEklUlinrnaNc' - b'VsLZeQv+3DfZBrdR4pTwJVIeda60JS6IwqtAl+pkSgZxdgW1TTxteMYCNz/zCpJSC' - b'KQ6oVP3/DVyZYODkmZaLz44INFDQp6cOfFzrhjvyi1SsUAKHvJIhS74u1swGm7TbD' - b'Nw56WAhjeQqfk+mOcLdzwwSBYiw4q7TdJ/iVT3DFFcoikBajegUpbaG8GVypBkluN' - b'sW3FmKG3WH0+V6Zi8HrV2t+kh1jaSSOsZfN4QVNYKlDs+pn9ZJc/nhQ40wFPgWDSW' - b'rasB8FNmAy7rpRlfQJ6Mnv3ez2lbrhieMSsiwLoyPpqxn87' - ) - good_pub_key = paramiko.RSAKey(data=base64.decodebytes(data)) - def __init__(self) -> None: """ Constructor. From 5ea63229fe20060b9c3d610933d31307d852cad4 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 12:13:56 +0100 Subject: [PATCH 238/250] Show colors and a progress display in the commands with rich The new extra cli installs rich. Without it, in pipes and files, and with NO_COLOR, the output stays plain text; the library does not import rich. --- CHANGES.md | 8 + README.md | 9 + docs/cli_tutorial.rst | 5 + docs/install.rst | 5 + pyproject.toml | 9 +- src/Exscript/cli.py | 340 +++++++++++++++++++++++++++++++++++--- tests/Exscript/cliTest.py | 255 +++++++++++++++++++++++++++- uv.lock | 21 ++- 8 files changed, 630 insertions(+), 22 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 95f65a28..fada57a6 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -35,6 +35,14 @@ marked with **BREAKING**; a migration guide will accompany the release. 130. Before, on Linux and macOS, it ran the connections in a child process, which it killed on Ctrl+C before it exited with status 1; on Windows, Ctrl+C took effect only after the running connections ended. +- **Colored Command Line Output (optional):** With the new `cli` extra + (`Exscript[cli]`), which installs rich, `exscript` and `otp` write their errors in + red, warnings and `Interrupted.` in yellow, and the one-time password in bold. With + more than one connection (`-c`) at the default verbosity, `exscript` shows a progress + bar with the numbers of done, all and failed hosts and the names of the running ones + instead of its text status bar, and a line for each host above it. Without rich, + when stdout or stderr is no terminal, and with `NO_COLOR`, the output stays plain + text as before. Only the commands use rich, not the library. - **NETCONF Protocol:** The new adapter `Exscript.protocols.Netconf` (scheme `netconf://`, port 830) sends RPCs over SSH with ncclient, and returns the replies as text or as lxml elements; an `` raises `RPCError`. The IOS XR and Junos diff --git a/README.md b/README.md index 8b57b807..053df2b7 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,15 @@ Exscript is what you are looking for. Additionally any system that supports SSH Exscript is supported by [Procedure 8](https://procedure8.com). Get in touch if you need anything! +## Installation + +```shell +pip install Exscript +``` + +The `cli` extra adds colors and a progress display to the command line tools +(`pip install "Exscript[cli]"`), and the `otel` extra adds tracing with OpenTelemetry. + ## Method 1: Using Exscript with Python ### Simple diff --git a/docs/cli_tutorial.rst b/docs/cli_tutorial.rst index bcf89489..094460eb 100644 --- a/docs/cli_tutorial.rst +++ b/docs/cli_tutorial.rst @@ -16,6 +16,11 @@ terminal; if this prints the version number, your installation is complete. ``python -m Exscript`` runs the tool, too, e.g. if the ``exscript`` command is not on your ``PATH``. +With the ``cli`` extra, e.g. ``pip install "Exscript[cli]"``, the tool +writes its errors and warnings in colors on a terminal, and shows the +progress of the hosts that it runs in parallel. In pipes and files, and if +the environment variable ``NO_COLOR`` is set, its output is plain text. + Getting started --------------- diff --git a/docs/install.rst b/docs/install.rst index a8f64cbe..30dc9740 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -22,6 +22,11 @@ or, in a project managed with uv:: To trace sessions with OpenTelemetry, install the ``otel`` extra, e.g. ``pip install "Exscript[otel]"``. +For colored messages and a progress display in the command line tools +``exscript`` and ``otp``, install the ``cli`` extra, which adds rich, e.g. +``pip install "Exscript[cli]"``, or ``uv tool install "Exscript[cli]"`` for +the commands alone. The library itself does not need it. + Installing from GitHub ---------------------- diff --git a/pyproject.toml b/pyproject.toml index a3dfd8c6..eff92430 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,6 +48,9 @@ otel = [ "opentelemetry-api>=1.27", "opentelemetry-instrumentation>=0.48b0", ] +cli = [ + "rich>=13", +] [project.scripts] exscript = "Exscript.cli:main" @@ -71,7 +74,11 @@ types = ["mypy>=2.4", "types-lxml>=2026.2", "types-paramiko>=5.1"] docs = ["sphinx>=9", "furo>=2025.12", "myst-parser>=5"] # Not a dependency of Exscript: CI checks that applications can run it under gevent. gevent = [{ include-group = "test" }, "gevent>=26"] -dev = [{ include-group = "test" }, { include-group = "types" }] +dev = [ + { include-group = "test" }, + { include-group = "types" }, + "rich>=15.0.0", +] [tool.hatch.version] source = "vcs" diff --git a/src/Exscript/cli.py b/src/Exscript/cli.py index 74573c79..ce745c1f 100644 --- a/src/Exscript/cli.py +++ b/src/Exscript/cli.py @@ -27,10 +27,17 @@ ``exscript``, too. ``exscript`` runs a template on hosts; see the :doc:`/cli_tutorial` and ``exscript --help``. ``otp`` asks for a password, and prints the S/Key one-time password for a sequence number and a salt. + +With rich, which the extra ``cli`` installs (``pip install "Exscript[cli]"``), +both write their messages in colors if stdout and stderr are terminals, and +with more than one connection at the default verbosity, ``exscript`` shows a +progress display instead of its text status bar. Otherwise, or if the +environment variable NO_COLOR is set, the output is plain text. """ import argparse import ast +import contextlib import getpass import importlib.util import os @@ -39,9 +46,10 @@ import sys import time from collections.abc import Sequence +from functools import partial from pathlib import Path from types import FrameType -from typing import NoReturn +from typing import TYPE_CHECKING, NoReturn import Exscript.protocols.drivers from Exscript import Account, Host, PrivateKey, Queue, __version__ @@ -61,6 +69,9 @@ from Exscript.util.log import log_to_file from Exscript.workqueue.job import Thread +if TYPE_CHECKING: + from rich.console import Console + _bracket_expression = r"\{([^\]]*)\}" _define_expression = rf"^\S+=(?:\S+|{_bracket_expression})$" _bracket_expression_re = re.compile(rf"^{_bracket_expression}$") @@ -145,17 +156,304 @@ def _expand_host_variables(host: Host) -> None: host.set(key, value) -def _exit_on_sigint(signum: int, frame: FrameType | None) -> NoReturn: +# The styles of the messages on a terminal, see _RichOutput. +_ERROR = 'red' +_WARNING = 'yellow' +_SUCCESS = 'green' + + +def _print(console: 'Console', text: str, style: str, end: str = '\n') -> None: + """ + Prints the given text as it is in the given style on the given rich + console: without markup, highlighting, emoji codes and line wrapping. + """ + from rich.text import Text + + # Styles the text only; the style argument of print() also styles the + # progress display, which the console prints after the text. + console.print(Text(text, style=style), end=end, soft_wrap=True) + + +class _Output: + """ + The output of the commands as plain text: where rich is not installed or + cannot show colors, or NO_COLOR is set; see :func:`_create_output`. + """ + + def warning(self, text: str) -> None: + """ + Writes the given warning to stdout. + """ + print(text) + + def error(self, text: str) -> None: + """ + Writes the given error to stderr. + """ + print(text, file=sys.stderr) + + def interrupted(self) -> None: + """ + Writes "Interrupted." to stderr, after Ctrl+C. + """ + sys.stderr.write('Interrupted.\n') + sys.stderr.flush() + + def password(self, text: str) -> None: + """ + Writes the given one-time password to stdout. + """ + print(text) + + def queue(self, domain: str, host_driver: str | None, verbose: int, max_threads: int) -> Queue: + """ + Returns a new queue with the given options, see :class:`Queue`. + """ + return Queue( + domain=domain, host_driver=host_driver, verbose=verbose, max_threads=max_threads + ) + + def progress(self) -> contextlib.AbstractContextManager[object]: + """ + Returns the context in which the queue of :meth:`queue` runs its jobs. + """ + return contextlib.nullcontext() + + +class _ErrorWriter: + """ + The error channel of a queue on a rich console: prints what the queue + writes to it in the style of errors, above the progress display if it + is shown. + """ + + def __init__(self, console: 'Console') -> None: + """ + Constructor. + """ + self._console = console + + def write(self, text: str) -> int: + """ + Prints the given text, and returns its length. + """ + _print(self._console, text, _ERROR, end='') + return len(text) + + def flush(self) -> None: + """ + Does nothing; write() prints the text at once. + """ + + +class _Progress: + """ + A progress display of the jobs of a queue on a rich console, instead of + the status bar of the queue: the number of hosts that are done, of all + hosts and of those that failed, and the names of the running jobs. + Above it, it writes a line for each host that succeeded. It is shown + while the context of the object lasts. + """ + + def __init__(self, console: 'Console', queue: Queue) -> None: + """ + Constructor. + + :param console: The console of the display. + :param queue: The queue, whose events update the display. + """ + from rich.progress import BarColumn, MofNCompleteColumn, Progress, TextColumn + from rich.table import Column + + self._console = console + self._queue = queue + self._running: dict[Thread, None] = {} # in the order in which the jobs started + names = Column(no_wrap=True, overflow='ellipsis', ratio=1) + self._display = Progress( + BarColumn(), + MofNCompleteColumn(), + TextColumn('hosts'), + TextColumn('{task.fields[failed]}', style=_ERROR), + TextColumn('{task.fields[running]}', markup=False, table_column=names), + console=console, + auto_refresh=False, + expand=True, + ) + self._task = self._display.add_task('', total=0, failed='', running='') + queue.workqueue.job_started_event.listen(self._on_job_started) + queue.workqueue.job_error_event.listen(self._on_job_ended) + queue.workqueue.job_succeeded_event.listen(self._on_job_succeeded) + queue.workqueue.job_aborted_event.listen(self._on_job_ended) + + def __enter__(self) -> None: + """ + Shows the display. + """ + with self._queue.lock: + self._update() + self._display.start() + + def __exit__(self, *exc_info: object) -> None: + """ + Shows the final numbers, and stops the display. + """ + with self._queue.lock: + self._running.clear() + self._update() + self.stop() + + def stop(self) -> None: + """ + Stops the display if it is shown; its last state stays on the console. + """ + if self._display.live.is_started: + self._display.stop() + + def _update(self) -> None: + """ + Shows the numbers of the queue and the names of the running jobs. The + caller holds the lock of the queue, under which the queue counts. + """ + failed = self._queue.failed + self._display.update( + self._task, + total=self._queue.total, + completed=self._queue.completed, + failed=f'{failed} failed' if failed else '', + running=', '.join(job.name for job in self._running), + refresh=True, + ) + + def _on_job_started(self, job: Thread) -> None: + """ + Shows the given job as running. + """ + with self._queue.lock: + self._running[job] = None + self._update() + + def _on_job_ended(self, job: Thread, exc_info: object = None) -> None: + """ + Shows that the given job is no longer running after an attempt that + failed, and the number of failed hosts after its last attempt. + """ + with self._queue.lock: + self._running.pop(job, None) + self._update() + + def _on_job_succeeded(self, job: Thread) -> None: + """ + Writes that the given job succeeded, and shows it as done. + """ + with self._queue.lock: + self._running.pop(job, None) + _print(self._console, job.name + ' succeeded.', _SUCCESS) + self._update() + + +class _RichOutput(_Output): + """ + The output of the commands on terminals with rich: the messages in + colors, and a progress display instead of the status bar of the queue. + """ + + def __init__(self, stdout: 'Console', stderr: 'Console') -> None: + """ + Constructor. + + :param stdout: The console of stdout. + :param stderr: The console of stderr, which also shows the progress. + """ + self._stdout = stdout + self._stderr = stderr + self._progress: _Progress | None = None + + def warning(self, text: str) -> None: + """ + Writes the given warning to stdout, in yellow. + """ + _print(self._stdout, text, _WARNING) + + def error(self, text: str) -> None: + """ + Writes the given error to stderr, in red. + """ + _print(self._stderr, text, _ERROR) + + def interrupted(self) -> None: + """ + Stops the progress display if it is shown, and writes "Interrupted." + to stderr, in yellow. + """ + if self._progress is not None: + self._progress.stop() + _print(self._stderr, 'Interrupted.', _WARNING) + + def password(self, text: str) -> None: + """ + Writes the given one-time password to stdout, in bold. + """ + _print(self._stdout, text, 'bold') + + def queue(self, domain: str, host_driver: str | None, verbose: int, max_threads: int) -> Queue: + """ + Returns a new queue with the given options, see :class:`Queue`, which + writes its errors in red. With more than one thread at verbosity 1, + the default, the progress display replaces its status bar. + """ + # At verbosity 0, the queue writes the same as at 1 but the status bar. + # From 2 on, it logs debug messages, which would garble the display. + progress = verbose == 1 and max_threads > 1 + queue = Queue( + domain=domain, + host_driver=host_driver, + verbose=0 if progress else verbose, + max_threads=max_threads, + stderr=_ErrorWriter(self._stderr), + ) + if progress: + self._progress = _Progress(self._stderr, queue) + return queue + + def progress(self) -> contextlib.AbstractContextManager[object]: + """ + Returns the context in which the queue of :meth:`queue` runs its + jobs: the progress display, if the queue has one. + """ + return self._progress or contextlib.nullcontext() + + +def _create_output() -> _Output: + """ + Returns the output of the commands: :class:`_RichOutput` if rich is + installed and stdout and stderr are terminals that show colors, unless + the environment variable NO_COLOR is set; else :class:`_Output`, as + plain text. + """ + try: + from rich.console import Console + except ImportError: # the optional "cli" extra is not installed + return _Output() + stdout, stderr = Console(), Console(stderr=True) + # No color system: no terminal, or a dumb one. + if stdout.no_color or stdout.color_system is None or stderr.color_system is None: + return _Output() + return _RichOutput(stdout, stderr) + + +def _exit_on_sigint( + signum: int, frame: FrameType | None, output: _Output | None = None +) -> NoReturn: """ The handler of SIGINT (Ctrl+C) while the template runs: writes - "Interrupted." to stderr, and ends the process at once with exit status - 130. A KeyboardInterrupt would end it only after the threads of the - running connections, which cannot be interrupted. + "Interrupted." to stderr through the given output, plain by default, and + ends the process at once with exit status 130. A KeyboardInterrupt would + end it only after the threads of the running connections, which cannot + be interrupted. """ try: sys.stdout.flush() - sys.stderr.write('Interrupted.\n') - sys.stderr.flush() + (output or _Output()).interrupted() finally: os._exit(130) @@ -172,7 +470,7 @@ def _join(queue: Queue) -> None: queue.join() -def _run(parser: argparse.ArgumentParser, args: argparse.Namespace) -> int: +def _run(parser: argparse.ArgumentParser, args: argparse.Namespace, output: _Output) -> int: """ Runs the template on the hosts as the given command line arguments say, and returns the number of hosts on which it failed. Invalid arguments @@ -182,6 +480,7 @@ def _run(parser: argparse.ArgumentParser, args: argparse.Namespace) -> int: :param parser: The parser of the arguments, see :func:`_create_parser`. :type args: argparse.Namespace :param args: The parsed arguments. + :param output: The output of the messages and the progress. :rtype: int :return: The number of failed actions. """ @@ -241,7 +540,7 @@ def mkhosts_from_csv(filename: Path) -> list[Host]: except OSError as e: parser.error(str(e)) if not csv_hosts: - print(f"Warning: {args.csv_hosts} is empty.") + output.warning(f"Warning: {args.csv_hosts} is empty.") hosts += csv_hosts # If a filename containing hostnames was given, read it. @@ -254,7 +553,7 @@ def mkhosts_from_csv(filename: Path) -> list[Host]: except ValueError as e: parser.error(str(e)) if not txt_hosts: - print(f"Warning: {args.hosts} is empty.") + output.warning(f"Warning: {args.hosts} is empty.") hosts += txt_hosts # If a filename of an Exscript driver was given, import it. @@ -270,7 +569,7 @@ def mkhosts_from_csv(filename: Path) -> list[Host]: except OSError as e: parser.error(str(e)) except Exception: - sys.stderr.write(f"Error in driver {args.load_driver}\n") + output.error(f"Error in driver {args.load_driver}") raise for name, obj in list(driver_module.__dict__.items()): if Exscript.protocols.drivers.isdriver(obj): @@ -328,7 +627,7 @@ def mkhosts_from_csv(filename: Path) -> list[Host]: template.debug = 0 # Create Exscript. - queue = Queue( + queue = output.queue( domain=args.default_domain, host_driver=args.use_driver, verbose=args.verbose, @@ -345,7 +644,7 @@ def mkhosts_from_csv(filename: Path) -> list[Host]: except OSError as e: parser.error(str(e)) if args.account_pool and default_pool.n_accounts() == 0: - print(f"WARNING: No accounts found in account pool file ({args.account_pool})!") + output.warning(f"WARNING: No accounts found in account pool file ({args.account_pool})!") # Read username and password. if args.non_interactive or default_pool.n_accounts() > 0: @@ -444,7 +743,7 @@ def function(job: Thread, host: Host, conn: Protocol, **kwargs: object) -> objec # Ctrl+C ends the process at once from here on, see _exit_on_sigint(). # Not before: while getpass() reads a password, the terminal does not # echo, and only a KeyboardInterrupt lets getpass() turn the echo on again. - sigint_handler = signal.signal(signal.SIGINT, _exit_on_sigint) + sigint_handler = signal.signal(signal.SIGINT, partial(_exit_on_sigint, output=output)) try: # Wait until the specified time. if args.sleep: @@ -453,8 +752,9 @@ def function(job: Thread, host: Host, conn: Protocol, **kwargs: object) -> objec print("time expired, starting script.") # Run the template. - queue.run(hosts, function, attempts=args.retry + 1) - _join(queue) + with output.progress(): + queue.run(hosts, function, attempts=args.retry + 1) + _join(queue) finally: signal.signal(signal.SIGINT, sigint_handler) failed = queue.failed @@ -931,9 +1231,11 @@ def main(argv: list[str] | None = None) -> int: process through SystemExit, as usual with argparse. """ parser = _create_parser() - failed = _run(parser, parser.parse_args(argv)) + args = parser.parse_args(argv) + output = _create_output() + failed = _run(parser, args, output) if failed != 0: - print(f"error: {failed} actions failed.", file=sys.stderr) + output.error(f"error: {failed} actions failed.") return 1 return 0 @@ -965,5 +1267,5 @@ def otp_main(argv: list[str] | None = None) -> int: parser.error('first argument must be a number') password = getpass.getpass('Password: ') - print(otp(password, salt, seq)) + _create_output().password(otp(password, salt, seq)) return 0 diff --git a/tests/Exscript/cliTest.py b/tests/Exscript/cliTest.py index c57f0047..89c216e4 100644 --- a/tests/Exscript/cliTest.py +++ b/tests/Exscript/cliTest.py @@ -1,22 +1,45 @@ import contextlib import io import os +import re import runpy import signal +import subprocess +import sys import tempfile +import textwrap import unittest from unittest import mock from Exscript import Account, Host, PrivateKey -from Exscript.cli import _duration, _exit_on_sigint, _expand_bracket, main, otp_main +from Exscript.cli import ( + _create_output, + _duration, + _exit_on_sigint, + _expand_bracket, + _Output, + _RichOutput, + main, + otp_main, +) +from Exscript.interpreter.exception import FailException from Exscript.parselib.exception import CompileError from Exscript.protocols import Dummy, drivers from Exscript.util import template from Exscript.util.crypt import otp +from Exscript.workqueue.job import Thread + +try: + from rich.console import Console +except ImportError: # the optional "cli" extra is not installed + Console = None + +requires_rich = unittest.skipIf(Console is None, 'requires the "cli" extra') KEYFILE = os.path.join(os.path.dirname(__file__), 'protocols', 'id_rsa') HELLO = '{sys.message("hello")}' POOL = '[account-pool]\nalice = c2VjcmV0\n' # the password is "secret" +RED, YELLOW, GREEN, BOLD, RESET = '\x1b[31m', '\x1b[33m', '\x1b[32m', '\x1b[1m', '\x1b[0m' LIB = ''' def hello(scope, name): @@ -37,6 +60,45 @@ def __init__(self): ''' +class Tty(io.StringIO): + """A file that claims to be a terminal.""" + + def isatty(self): + return True + + +@contextlib.contextmanager +def environ(**values): + """Sets the given environment variables, and removes the others that rich reads to + detect a terminal and its colors.""" + with mock.patch.dict(os.environ, values): + names = {'NO_COLOR', 'FORCE_COLOR', 'TTY_COMPATIBLE', 'TTY_INTERACTIVE', 'TERM'} + for name in names - values.keys(): + os.environ.pop(name, None) + yield + + +def terminal(): + """Returns a rich console that writes to a StringIO like to a terminal with 16 colors + and 100 columns, whatever the environment says.""" + return Console( + file=io.StringIO(), + width=100, + force_terminal=True, + force_interactive=True, + color_system='standard', + no_color=False, + legacy_windows=False, + ) + + +def shown(output): + """Returns what a terminal shows of the given output: without the states of the + progress display that are erased, and without styles and other control sequences.""" + output = re.sub(r'[^\n]*\r\x1b\[2K', '', output) + return re.sub(r'\x1b\[[0-9;?]*[A-Za-z]', '', output) + + class cliTest(unittest.TestCase): def setUp(self): self.tmpdir = tempfile.TemporaryDirectory() @@ -82,6 +144,15 @@ def unload_driver(self, name): if name in drivers.driver_map: drivers.disable_driver(name) + def rich_output(self): + """Lets main() and otp_main() write with rich to terminals, and returns the output + and the files of its stdout and stderr.""" + self.start(mock.patch.dict(os.environ, TERM='xterm')) # not a dumb terminal + stdout, stderr = terminal(), terminal() + output = _RichOutput(stdout, stderr) + self.start(mock.patch('Exscript.cli._create_output', return_value=output)) + return output, stdout.file, stderr.file + def testTemplateFile(self): path = self.write('test.exscript', '{sys.message(__filename__)}\n') code, stdout, stderr = self.run_main('-n', '-i', path, 'dummy://r1') @@ -588,3 +659,185 @@ def testOtpWithoutNumber(self): self.assertEqual(cm.exception.code, 2) self.assertIn('first argument must be a number', stderr.getvalue()) getpass.assert_not_called() + + def testOutputOnTerminalsOnly(self): + rich = _RichOutput if Console else _Output + cases = ( + (Tty, Tty, {}, rich), + (io.StringIO, io.StringIO, {}, _Output), + (Tty, io.StringIO, {}, _Output), + (io.StringIO, Tty, {}, _Output), + (Tty, Tty, {'NO_COLOR': '1'}, _Output), + (Tty, Tty, {'TERM': 'dumb'}, _Output), + (io.StringIO, io.StringIO, {'FORCE_COLOR': '1'}, rich), # any file is a terminal + ) + for stdout, stderr, variables, cls in cases: + with ( + self.subTest(stdout=stdout, stderr=stderr, environ=variables), + environ(**variables), + contextlib.redirect_stdout(stdout()), + contextlib.redirect_stderr(stderr()), + ): + self.assertIs(type(_create_output()), cls) + + def testPlainOutputWithoutRich(self): + # rich takes any output for a terminal, but it is not installed. + path = self.write('hosts.txt', '') + with environ(FORCE_COLOR='1'), mock.patch.dict(sys.modules, {'rich.console': None}): + self.assertIs(type(_create_output()), _Output) + code, stdout, stderr = self.run_main( + '-n', '-i', '-c', '2', '--hosts', path, '-e', '{fail "boom"}', 'dummy://r1' + ) + self.assertEqual(code, 1) + self.assertEqual(stdout, f'Warning: {path} is empty.\n') + self.assertEqual(stderr, 'error: 1 actions failed.\n') + + def testPlainOutputWithoutTerminal(self): + path = self.write('hosts.txt', '') + with environ(): + code, stdout, stderr = self.run_main( + '-n', '-i', '-c', '2', '--hosts', path, '-e', '{fail "boom"}', 'dummy://r1' + ) + self.assertEqual(code, 1) + self.assertEqual(stdout, f'Warning: {path} is empty.\n') + self.assertEqual(stderr, 'error: 1 actions failed.\n') + + def testLibraryDoesNotImportRich(self): + # Applications that use the library do not need rich; only the commands use it. + script = textwrap.dedent(''' + import importlib, pkgutil, sys + import Exscript + for module in pkgutil.walk_packages(Exscript.__path__, 'Exscript.'): + if module.name not in ('Exscript.cli', 'Exscript.__main__'): + importlib.import_module(module.name) + print(sorted(name for name in sys.modules if name.split('.')[0] == 'rich')) + ''') + proc = subprocess.run( # noqa: S603 - runs Python with a fixed script + [sys.executable, '-c', script], capture_output=True, encoding='utf-8', timeout=60 + ) + self.assertEqual(proc.returncode, 0, proc.stderr) + self.assertEqual(proc.stdout, '[]\n') + + @requires_rich + def testRichMessages(self): + output, stdout, stderr = self.rich_output() + hosts = self.write('hosts.txt', '') + pool = self.write('pool.cfg', '[account-pool]\n') + args = '--hosts', hosts, '--account-pool', pool, '-e', '{fail "boom"}', 'dummy://r1' + code, out, err = self.run_main('-n', '-i', *args) + self.assertEqual(code, 1) + self.assertEqual((out, err), ('', '')) + self.assertEqual( + stdout.getvalue(), + f'{YELLOW}Warning: {hosts} is empty.{RESET}\n' + f'{YELLOW}WARNING: No accounts found in account pool file ({pool})!{RESET}\n', + ) + # One connection: no progress display, and the errors of the queue are red, too. + self.assertEqual( + stderr.getvalue(), + f'{RED}r1 error: boom{RESET}\n' + f'{RED}r1 finally failed.{RESET}\n' + f'{RED}error: 1 actions failed.{RESET}\n', + ) + + @requires_rich + def testRichDriverError(self): + output, stdout, stderr = self.rich_output() + path = self.write('broken.py', 'x = \n') + with contextlib.redirect_stdout(io.StringIO()), self.assertRaises(SyntaxError): + main(['-n', '-i', '--load-driver', path, '-e', HELLO, 'dummy://r1']) + self.assertEqual(stderr.getvalue(), f'{RED}Error in driver {path}{RESET}\n') + + @requires_rich + def testRichProgress(self): + output, stdout, stderr = self.rich_output() + script = '{sys.message(__hostname__)}{fail "boom" if __hostname__ is "r2"}' + code, out, err = self.run_main( + '-n', '-i', '-c', '2', '-e', script, 'dummy://r1', 'dummy://r2' + ) + self.assertEqual(code, 1) + self.assertEqual((out, err, stdout.getvalue()), ('', '', '')) + # Above the display, the output of the template and a line for each host; the + # connections run in parallel, so in any order. + lines = [line.rstrip() for line in shown(stderr.getvalue()).splitlines()] + expected = ['r1', 'r1 succeeded.', 'r2', 'r2 error: boom', 'r2 finally failed.'] + self.assertCountEqual(lines[:-2], expected) + self.assertEqual(lines[-2:], ['━' * 40 + ' 2/2 hosts 1 failed', 'error: 1 actions failed.']) + self.assertIn(f'{GREEN}r1 succeeded.{RESET}\n', stderr.getvalue()) + self.assertIn(f'{RED}r2 finally failed.{RESET}\n', stderr.getvalue()) + + @requires_rich + def testRichProgressShowsRunningJobs(self): + # The events of the queue as two jobs run: r1 succeeds, r2 fails twice. + output, stdout, stderr = self.rich_output() + queue = output.queue(domain='', host_driver=None, verbose=1, max_threads=2) + self.addCleanup(queue.destroy) + events = queue.workqueue + r1, r2 = Thread(None, None, 'r1', {}), Thread(None, None, 'r2', {}) # attempts of jobs + error = FailException, FailException('boom'), '' + queue.total = 2 + + def display(): + """Returns the words of the display after the bar.""" + return shown(stderr.getvalue()).rsplit('\n', 1)[-1].split()[1:] + + with output.progress(): + self.assertEqual(display(), ['0/2', 'hosts']) + events.job_started_event(r1) + events.job_started_event(r2) + self.assertEqual(display(), ['0/2', 'hosts', 'r1,', 'r2']) + events.job_succeeded_event(r1) + self.assertEqual(display(), ['1/2', 'hosts', 'r2']) + events.job_error_event(r2, error) + self.assertEqual(display(), ['1/2', 'hosts']) + events.job_started_event(r2) + self.assertEqual(display(), ['1/2', 'hosts', 'r2']) + events.job_error_event(r2, error) + events.job_aborted_event(r2) + self.assertEqual(display(), ['2/2', 'hosts', '1', 'failed']) + lines = [line.rstrip() for line in shown(stderr.getvalue()).splitlines()] + expected = ['r1 succeeded.', 'r2 error: boom', 'r2 error: boom', 'r2 finally failed.'] + self.assertEqual(lines, [*expected, '━' * 40 + ' 2/2 hosts 1 failed']) + # The styles of the lines above stay out of the display, which replaces the status bar. + self.assertNotIn(f'{GREEN}hosts', stderr.getvalue()) + self.assertNotIn(f'{RED}hosts', stderr.getvalue()) + self.assertIs(queue.channel_map['status_bar'], queue.devnull) + + @requires_rich + def testRichDebugOutputWithoutProgress(self): + # From verbosity 2 on, the queue logs debug messages, so it keeps its status bar. + output, stdout, stderr = self.rich_output() + args = '-n', '-i', '-c', '2', '-v', '2', '-e', '{fail "boom"}', 'dummy://r1' + code, out, err = self.run_main(*args) + self.assertEqual(code, 1) + self.assertNotIn('━', stderr.getvalue()) + self.assertTrue(stderr.getvalue().startswith(f'{RED}r1 error: boom{RESET}\n')) + self.assertTrue(stderr.getvalue().endswith(f'{RED}error: 1 actions failed.{RESET}\n')) + + @requires_rich + def testRichInterrupted(self): + # Without a progress display, e.g. with one connection. + output, stdout, stderr = self.rich_output() + with mock.patch('os._exit') as exit: + _exit_on_sigint(signal.SIGINT, None, output=output) + exit.assert_called_once_with(130) + self.assertEqual(stderr.getvalue(), f'{YELLOW}Interrupted.{RESET}\n') + + # The display stops, shows the cursor again, and keeps its last state. + output, stdout, stderr = self.rich_output() + queue = output.queue(domain='', host_driver=None, verbose=1, max_threads=2) + self.addCleanup(queue.destroy) + with output.progress(), mock.patch('os._exit') as exit: + _exit_on_sigint(signal.SIGINT, None, output=output) + exit.assert_called_once_with(130) + self.assertTrue(stderr.getvalue().endswith(f'\x1b[?25h{YELLOW}Interrupted.{RESET}\n')) + self.assertEqual( + shown(stderr.getvalue()).split(), ['━' * 40, '0/0', 'hosts', 'Interrupted.'] + ) + + @requires_rich + def testRichOtp(self): + output, stdout, stderr = self.rich_output() + with mock.patch('getpass.getpass', return_value='password'): + self.assertEqual(otp_main(['9999', 'abc123']), 0) + self.assertEqual(stdout.getvalue(), f'{BOLD}ACTA AMMO CAR WEB BIN YAP{RESET}\n') diff --git a/uv.lock b/uv.lock index d174c3b8..3854f779 100644 --- a/uv.lock +++ b/uv.lock @@ -617,6 +617,9 @@ dependencies = [ ] [package.optional-dependencies] +cli = [ + { name = "rich" }, +] otel = [ { name = "opentelemetry-api" }, { name = "opentelemetry-instrumentation" }, @@ -630,6 +633,7 @@ dev = [ { name = "opentelemetry-sdk" }, { name = "pytest" }, { name = "pytest-cov" }, + { name = "rich" }, { name = "types-lxml" }, { name = "types-paramiko" }, ] @@ -668,8 +672,9 @@ requires-dist = [ { name = "opentelemetry-instrumentation", marker = "extra == 'otel'", specifier = ">=0.48b0" }, { name = "paramiko", specifier = ">=3.4,<6" }, { name = "pycryptodomex" }, + { name = "rich", marker = "extra == 'cli'", specifier = ">=13" }, ] -provides-extras = ["otel"] +provides-extras = ["cli", "otel"] [package.metadata.requires-dev] dev = [ @@ -679,6 +684,7 @@ dev = [ { name = "opentelemetry-sdk", specifier = ">=1.27" }, { name = "pytest", specifier = ">=9.2" }, { name = "pytest-cov", specifier = ">=7.1" }, + { name = "rich", specifier = ">=15.0.0" }, { name = "types-lxml", specifier = ">=2026.2" }, { name = "types-paramiko", specifier = ">=5.1" }, ] @@ -1697,6 +1703,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0", size = 73075, upload-time = "2026-05-14T19:25:26.443Z" }, ] +[[package]] +name = "rich" +version = "15.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markdown-it-py" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c0/8f/0722ca900cc807c13a6a0c696dacf35430f72e0ec571c4275d2371fca3e9/rich-15.0.0.tar.gz", hash = "sha256:edd07a4824c6b40189fb7ac9bc4c52536e9780fbbfbddf6f1e2502c31b068c36", size = 230680, upload-time = "2026-04-12T08:24:00.75Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/82/3b/64d4899d73f91ba49a8c18a8ff3f0ea8f1c1d75481760df8c68ef5235bf5/rich-15.0.0-py3-none-any.whl", hash = "sha256:33bd4ef74232fb73fe9279a257718407f169c09b78a87ad3d296f548e27de0bb", size = 310654, upload-time = "2026-04-12T08:24:02.83Z" }, +] + [[package]] name = "roman-numerals" version = "4.1.0" From 61aa86f63b7a290790b88d75924a00d281d22b53 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 15:42:17 +0100 Subject: [PATCH 239/250] Add command files for recording NX-OS sessions --- tests/recordings/commands/nxos-basic.txt | 22 +++++++++++++++++++++ tests/recordings/commands/nxos-operator.txt | 6 ++++++ tests/recordings/commands/nxos-pager.txt | 8 ++++++++ 3 files changed, 36 insertions(+) create mode 100644 tests/recordings/commands/nxos-basic.txt create mode 100644 tests/recordings/commands/nxos-operator.txt create mode 100644 tests/recordings/commands/nxos-pager.txt diff --git a/tests/recordings/commands/nxos-basic.txt b/tests/recordings/commands/nxos-basic.txt new file mode 100644 index 00000000..1bc399ef --- /dev/null +++ b/tests/recordings/commands/nxos-basic.txt @@ -0,0 +1,22 @@ +# Cisco NX-OS (R2, lab devices only), with --os nxos. +# Log in as a network-admin. Changes the running config: creates and removes loopback999. +terminal length 0 +show version +show running-config +show interface brief +show ip route +show inventory +show nonexistent +show i +show interface ethernet +show interface ethernet 99/99 +configure terminal +interface loopback999 +description Exscript test, safe to remove +end +show running-config interface loopback999 +configure terminal +no interface loopback999 +end +show this-command-is-much-longer-than-the-terminal-width | include something-that-does-not-match-anything-at-all +exit diff --git a/tests/recordings/commands/nxos-operator.txt b/tests/recordings/commands/nxos-operator.txt new file mode 100644 index 00000000..7ff1ac10 --- /dev/null +++ b/tests/recordings/commands/nxos-operator.txt @@ -0,0 +1,6 @@ +# Cisco NX-OS (R2, optional), with --os nxos, as a user with the role network-operator: +# the error of a command that the role does not allow. +terminal length 0 +show version +configure terminal +exit diff --git a/tests/recordings/commands/nxos-pager.txt b/tests/recordings/commands/nxos-pager.txt new file mode 100644 index 00000000..39f79e51 --- /dev/null +++ b/tests/recordings/commands/nxos-pager.txt @@ -0,0 +1,8 @@ +# Cisco NX-OS (R2): output with the pager on. +# The recorder asks for 1000 lines, so set the pager length explicitly. +terminal length 24 +show running-config +!send \x20 +!send \x20 +!send q +exit From 0420b6141bfdda372d811946c43e7667fe5ba44d Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 16:24:24 +0100 Subject: [PATCH 240/250] Document the API changes against 2.6.32 in the changelog and guide Add the changes against 2.6.32 that were not documented, and correct entries that described the 2.9 development versions as the previous release. --- CHANGES.md | 165 ++++++++++++++++++++++++++++++++------------- docs/migration.rst | 95 +++++++++++++++++++++++--- docs/templates.rst | 14 ++++ 3 files changed, 219 insertions(+), 55 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index fada57a6..fbcc7965 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,9 +2,9 @@ ## 3.0.0 (unreleased) -Work in progress: the entries below cover the changes merged from the 2.9 development -line and are completed against Exscript 2.6.32 before the release. Breaking changes are -marked with **BREAKING**; a migration guide will accompany the release. +Work in progress: the entries below describe the changes against Exscript 2.6.32, +including those of the 2.9 development line. Breaking changes are marked with +**BREAKING**; the migration guide says what to change. ## General @@ -46,20 +46,33 @@ marked with **BREAKING**; a migration guide will accompany the release. - **NETCONF Protocol:** The new adapter `Exscript.protocols.Netconf` (scheme `netconf://`, port 830) sends RPCs over SSH with ncclient, and returns the replies as text or as lxml elements; an `` raises `RPCError`. The IOS XR and Junos - drivers recognize their devices by the NETCONF capabilities. + drivers recognize their devices by the NETCONF capabilities. Templates send RPCs in + `` elements. - **BREAKING - Internal Attributes and Helpers:** Attributes and helpers that were meant for internal use are private or removed; the migration guide lists their public replacements. Among them are the attributes `buffer`, `auto_driver`, - `manual_driver`, `manual_*_re`, `*_authenticated`, `last_account`, `cancel` and `log` - of the protocol adapters, `SSH2.client`, `shell` and `sock`, and `Telnet.tn`; - `OsGuesser.data_received()` and `protocol_info()`, now `check_received_data()` and - `check_protocol_info()`; `Exscript.util.template.test_secure()`, now + `manual_driver`, `manual_*_re`, `*_authenticated`, `driver_replaced`, + `last_account`, `cancel` and `log` of the protocol adapters, `Dummy.init_done`, + `SSH2.client`, `shell` and `sock`, and `Telnet.tn`; `os_guesser` of the protocol + adapters is read-only; `OsGuesser.data_received()` and `protocol_info()`, now + `check_received_data()` and `check_protocol_info()`, and its attributes `info`, + `os_map`, `auth_os_map`, `protocol_os_map` and `auth_buffer`; + `HPProCurveDriver.clean_res_re`; `Exscript.util.template.test_secure()`, now `test(secure=True)`; `InputHistory.file`, now `filename`; the `debug` argument of - `WorkQueue`, whose `set_debug()` argument is now `level`; the `type` argument of - `parselib.Lexer.expect()` and `current_is()`, now `types`; and the `dump1()` methods - of the template interpreter, `ExpressionNode.parent_node`, the `prio` argument of - `Expression.prioritize()`, and `lft`, `rgt`, `op` and `priority()` of `Term`, which - only the old ordering of operators used. + `WorkQueue`, whose `set_debug()` argument is now `level`, which is required; the + `type` argument of `parselib.Lexer.expect()` and `current_is()`, now `types`; + `string_re` of `Exscript.interpreter.string`, now `varstring_re`; `bracket_re` and + `Regex.n_groups` of `Exscript.interpreter.regex`, now `Regex.groups`; and the + `dump1()` methods of the template interpreter, `ExpressionNode.parent_node`, the + `prio` argument of `Expression.prioritize()`, and `lft`, `rgt`, `op` and + `priority()` of `Term`, which only the old ordering of operators used. +- **BREAKING - Debug Messages Through `logging`:** The protocol adapters write the + messages of their `debug` level to the logger of their module, e.g. + `Exscript.protocols.ssh2`, at the level `DEBUG`, and the queue those of its + `verbose` level 2 or more to `Exscript.queue` at the level `INFO`. Before, the + adapters wrote them to their `stderr`, which they no longer use, and the queue to + its `stdout`. Configure `logging` to see them, e.g. with + `logging.basicConfig(level=logging.DEBUG)`; `exscript` shows them on standard error. - **OpenTelemetry Tracing (optional):** Install the `otel` extra (`Exscript[otel]`) and call `Exscript.contrib.otel.ExscriptInstrumentor().instrument()`. Every call of `connect()`, `login()`, `authenticate()`, `send()`, `execute()` and `close()` creates @@ -125,10 +138,13 @@ marked with **BREAKING**; a migration guide will accompany the release. entities, loading DTDs, or accessing the network, also with `huge_tree`. lxml 5.0 or later is required, which no longer resolves external entities by default, also in the parsers of ncclient. -- **CLI Without `eval()` and Shell:** A value entered at a `-d VAR={#"PROMPT"}` prompt - is parsed as a Python literal, such as a number or a list, instead of being run as - Python code. `--sleep` no longer runs the `sleep` command in a shell; it accepts - seconds with an optional suffix `m`, `h` or `d`, also on Windows. +- **BREAKING - CLI Without `eval()` and Shell:** A value entered at a + `-d VAR={"PROMPT"}` prompt is text, and one entered at a `-d VAR={#"PROMPT"}` prompt + is parsed as a Python literal, such as a number or a list. Before, the value of a + `{"PROMPT"}` prompt was run as Python code, so text had to be entered in quotes; the + 2.9 development versions ran the value of a `{#"PROMPT"}` prompt. `--sleep` no longer + runs the `sleep` command in a shell; it accepts seconds with an optional suffix `m`, + `h` or `d`, also on Windows. ## Protocols @@ -149,10 +165,13 @@ marked with **BREAKING**; a migration guide will accompany the release. ### Connecting -- **Connect Rate Limit Per Host:** `connect_rate_limit` no longer delays connects to - other hosts. The limiter slept while it held a process-wide lock, so one waiting - connect blocked all others, by default for up to 1.1 seconds. Concurrent connects - to the same host are now spaced, too; before, they could start together. +- **Connect Rate Limit Per Host:** Connects to the same host, also concurrent ones, + are spaced by 60 / `connect_rate_limit` + 0.1 seconds. The new option + `connect_rate_limit` (also a `Host` option) is 60 connects per minute by default; 0 + turns the limit off. Connects to other hosts are not delayed. 2.6 had no limit. The + limiter of the 2.9 development versions slept while it held a process-wide lock, so + one waiting connect blocked all others, by default for up to 1.1 seconds, and + concurrent connects to the same host could start together. - **Fewer SSH Reconnects:** Drivers that reconnect between authentication methods (`ios`) now reconnect only after a method sent a request that failed. Before, they reconnected before every method, also before the public key methods when there was @@ -168,6 +187,11 @@ marked with **BREAKING**; a migration guide will accompany the release. `PreferredAuthentications`. The default order is `publickey`, `keyboard-interactive`, `password`; methods that the server does not offer are skipped. +- **BREAKING - Order of the Arguments of `Protocol`:** The new arguments + `connect_rate_limit`, before `connect_timeout`, and `disabled_algorithms` and + `preferred_authentications`, before `verify_fingerprint`, move the later arguments + of `Protocol.__init__()`, so adapters that subclass `Protocol` must pass them by + keyword. The adapters take keyword arguments only, as before. - **SSH Key Fingerprints in Debug Logs:** The keys that are tried for a login are logged with their SHA256 fingerprints, as `ssh-add -l` and `ssh-keygen -l` show them. Before, the MD5 fingerprint was logged as a Python bytes literal. @@ -227,6 +251,13 @@ marked with **BREAKING**; a migration guide will accompany the release. closed the connection first, are only logged (debug level 1); before, errors of the logout were ignored silently, and `close(force=True)` could raise an `EOFError`. +- **BREAKING - `connect()` While Connected:** `connect()` raises a `ProtocolException` + if the adapter is connected already. Before, it opened another connection and left + the first one open. +- **BREAKING - Output and Logfile With `\r`:** The `stdout` stream and the `logfile` of + the protocol adapters get the data as it was received, also the `\r` of `\r\n` line + breaks. Before, `\r` was removed. The logs of `Exscript.util.log` and + `exscript --logdir` are unchanged. ### Interactive Shell @@ -268,6 +299,16 @@ marked with **BREAKING**; a migration guide will accompany the release. `ProtocolException`, and Telnet an `EOFError`, or a `TimeoutException` if unmatched data was left. The new exception is a `ProtocolException` and an `EOFError`, so only code that expects a `TimeoutException` for a closed connection must change. +- **BREAKING - `expect()` Checks for Errors:** `expect()` raises an + `InvalidCommandException` if a line of the response matches an error pattern, as + `expect_prompt()` and `execute()` do; the data is consumed all the same. Before, only + `expect_prompt()` checked. `waitfor()` does not check. +- **BREAKING - `response` Without `\r` Before the Prompt:** The prompt patterns of the + drivers match a `\r\n` before the prompt as a whole, so after `execute()` and + `expect_prompt()`, `response` no longer ends with the `\r` of that line break. +- **BREAKING - `send()` Clears `response`:** `send()` sets `response` to `None`, and + `get_response()` returns an empty string, until the next `expect()`, `waitfor()` or + `expect_prompt()`. Before, `send()` kept the response to the previous command. - **Login Errors After Sending Only:** `login()`, `app_authenticate()` and `app_authorize()` raise `LoginFailure` only if the driver's login error patterns match data that arrived after Exscript sent something, e.g. the password. Before, @@ -323,8 +364,9 @@ marked with **BREAKING**; a migration guide will accompany the release. `read_eager()`, `read_very_eager()`, `read_lazy()`, `read_very_lazy()`, `read_all()`, `expect()`, `waitfor()`, `interact()` and `mt_interact()` with their helpers, `sock_avail()`, `get_socket()`, `fileno()` and `set_receive_callback()`; the - attribute `cancel_expect`; the function `test()`; the option constants other than - those Exscript negotiates; and the `stdout` and `stderr` arguments. The new method + attributes `cancel_expect`, `cookedq`, `irawq` and `data_callback_kwargs`; the + function `test()`; the option constants other than those Exscript negotiates; and + the `stdout` and `stderr` arguments and attributes. The new method `receive(timeout)` reads once and passes the text to the callback. The remaining constants are integers instead of bytes, and the options of the constructor are keyword-only. Use `Exscript.protocols.Telnet`. If you need a drop-in replacement for @@ -500,11 +542,11 @@ development versions; "before" refers to them. ### Class `Exscript.protocols.Protocol` -| Old | New | Comment | -| --- | --- | --- | -| - | `cleanup_response(response, ...)` | Static method to cleanup a response. Used by the `get_response` method. | -| `close(force=False, reconnecting=False)` | `close(force=False)` | **BREAKING:** `reconnecting` removed. `force=False` now performs a graceful, driver-driven logout before closing. | -| - | `get_response(*, strip_command, strip_chars, strip_ansi)` | Should be the first choice to retrieve a clean response. `Protocol.response` still exists with the raw received data. | +| Old | New | Comment | +| --- | --- | --- | +| - | `cleanup_response(response, ...)` | Static method to cleanup a response. Used by the `get_response` method. | +| `close(force=False)` | `close(force=False)` | **BREAKING:** `force=False` now performs a graceful, driver-driven logout before closing. | +| - | `get_response(*, strip_command, strip_chars, strip_ansi)` | Should be the first choice to retrieve a clean response. `Protocol.response` still exists with the raw received data. | ## Emulators @@ -531,11 +573,18 @@ development versions; "before" refers to them. - **BREAKING - Last Line Without a Line Break:** A template also sends its last line if the line does not end with a line break, e.g. `show version` in `template.eval(conn, 'ls\nshow version')`. Before, such a line was ignored. -- **BREAKING - Escape Sequences Out of Range:** A Unicode sequence above U+10FFFF, - such as `\UFFFFFFFF`, and an octal sequence above `\377`, such as `\400`, in a - command or a string raise a `CompileError` that shows their position. Before, the - former raised a `SyntaxError` of Python, and the latter gave the characters U+0100 - to U+01FF with a `SyntaxWarning` of Python. +- **BREAKING - Escape Sequences:** Commands and strings evaluate the escape sequences + of Python: `\n`, `\r`, `\t`, `\0`, `\a`, `\b`, `\f`, `\v`, `\e` (escape), octal + sequences of three digits such as `\101`, `\xhh`, `\uhhhh` and `\Uhhhhhhhh`; e.g. + `show \x41` sends `show A`, and `\n` in a command sends a line break. Before, a + backslash made the next character literal, so `show \x41` sent `show x41`, and only + `\n` and `\r` in strings were line breaks. The new argument `no_escaping=True` of + `eval()`, `eval_file()`, `paste()` and `paste_file()` of `Exscript.util.template`, + and the new option `--no-escaping` of `exscript`, keep the old behavior, except that + `\t` in a string is a tab. A Unicode sequence above U+10FFFF, such as `\UFFFFFFFF`, + and an octal sequence above `\377`, such as `\400`, raise a `CompileError` that shows + their position; the 2.9 development versions raised a `SyntaxError` of Python for the + former, and gave the characters U+0100 to U+01FF for the latter. - **BREAKING - `extract` Requires a Regular Expression:** `extract` raises a `CompileError` if no regular expression between slashes follows, e.g. in `{extract "(a)" as y from "a"}` or `{extract x as y}`. Before, the character after @@ -562,15 +611,28 @@ development versions; "before" refers to them. - **`is not` and `not in` With Several Blanks:** `is not` and `not in` with more than one blank or a line break between the words mean the same as `is not` and `not in`. Before, they gave `None`, and as a condition a `TypeError` of Python. -- **BREAKING - Lines With Only Whitespace:** A line with only whitespace is ignored, - also if the whitespace is not a space or a tab, e.g. a no-break space. Before, such - whitespace was sent at the start of the command in the next line. -- **BREAKING - Escaped Characters Start No Comment:** An escaped hash, or a hash - behind an escaped character, is part of the command: `\# x` sends `# x`, - `show \}# x` sends `show }# x`, and `show a\n# b` sends `# b` in its second line. - Before, the hash started a comment there, so `\# x` sent an empty line, and - `show \}# x` took the next line into the command. A hash behind an `` element - starts a comment, as behind a code block; before, it sent an empty line. +- **Lines With Only Whitespace:** A line with only whitespace is ignored, also if the + whitespace is not a space or a tab, e.g. a no-break space, as in 2.6. The 2.9 + development versions sent such whitespace at the start of the command in the next + line. +- **BREAKING - Escaped Characters Start No Comment:** A hash behind an escaped + character is part of the command: `show \}# x` sends `show }# x`, and `show a\n# b` + sends `# b` in its second line. Before, the hash started a comment there, so these + sent `show }` and `show an`. `\# x` sends `# x`, as before. A hash behind an `` + element starts a comment, as behind a code block. The 2.9 development versions sent + an empty line for `\# x` and for a hash behind an `` element, and took the next + line into the command for `show \}# x`. +- **BREAKING - Undefined Variables Raise `NameError`:** A template that uses an + undefined variable in an expression, e.g. in `{if x is "a"}`, raises a `NameError` + when it is compiled. Before, it raised a `ValueError`. +- **BREAKING - `` Elements:** A template sends the content of an `` element, + e.g. `...`, as a command of its own, without the + `rpc` tags, and checks its XML syntax when it is compiled; `Netconf` sends it as an + RPC. A `` without `` raises a `CompileError`. Before, `` and + `` were text; `\` and `\` still are. +- **`__username__` Without an Account:** The template variable `__username__` is + `undefined` if the connection has no account, like `__hostname__` without a + connection. Before, it was `None`. - **Errors at Runtime:** A division by zero (`1 / 0`, `1 % 0`), an operand of `/`, `ge`, `gt`, `le` or `lt` that is not a number (`"a" / 2`), an operand of an arithmetic operator without a value, such as a group that `extract` did not match, @@ -630,6 +692,9 @@ The names below refer to functions callable from templates, not protocol methods | Old | New | Comment | | --- | --- | --- | | `__response__` | `__response__` | **BREAKING:** Connection helpers now clean responses using `get_response()`, stripping command echoes where applicable, control/ANSI codes, and normalizing CR/LF. | +| - | `connection.add_error(error)` | Adds an error pattern to the current ones; `connection.set_error()` replaces them. | +| - | `connection.add_prompt(prompt)` | Adds a prompt pattern to the current ones; `connection.set_prompt()` replaces them. | +| - | `connection.authenticate_user(...)` | Like `connection.authenticate()`, but with the given user name and password. | | - | `connection.get_protocol()` | Returns the protocol name, e.g. `ssh`, `telnet`, or `netconf`. | | - | `connection.expect(prompt)` | Consumes the response and matched prompt. | | `connection.wait_for(prompt)` | `connection.wait_for(prompt)` | **BREAKING:** No longer consumes the response/prompt. Use `connection.expect()` for the previous behavior. | @@ -722,6 +787,16 @@ The names below refer to functions callable from templates, not protocol methods `exscript --overwrite-logs` overwrite the log file of an earlier run and keep all output of the job. Before, every write overwrote the file, so it kept only the last piece of the output, e.g. the last prompt. +- **BREAKING - `first_match()` and `any_match()` With a Connection:** + `Exscript.util.match.first_match()` and `any_match()` search `conn.get_response()` + when they get a connection: the response without the echo of the command, control + characters and terminal escape sequences, and with `\n` line ends. Before, they + searched `conn.response`. +- **BREAKING - Passwords of `get_accounts_from_file()`:** The passwords in the file + are decoded as UTF-8 after base64, or with the encoding of the new argument + `password_encoding`. A password that is no valid base64 or text in that encoding + raises an `Exception` that names the user. Before, the passwords were decoded as + latin-1, and invalid base64 raised a `binascii.Error`. - **BREAKING - Removed Modules:** `Exscript.util.daemonize`, `Exscript.util.pidutil`, `Exscript.util.sigint` and `Exscript.util.sigintcatcher` are removed; `daemonize()` failed on Python 3 with `ValueError: can't have unbuffered text I/O`. Run long-lived scripts under a service manager such as systemd @@ -740,9 +815,9 @@ The names below refer to functions callable from templates, not protocol methods - **BREAKING - No Pipes and Proxies:** Jobs take their accounts from the account manager and write to their loggers directly, without a pipe and a thread per job. Removed: `Exscript.account.AccountProxy`, `Exscript.logger.LoggerProxy`, the duplicate - `Exscript.account.LoggerProxy`, `Exscript.workqueue.job.Process` and - `Queue._create_pipe()`. The queue, accounts and account pools use the locks of - `threading` instead of those of `multiprocessing`. + `Exscript.account.LoggerProxy`, `Exscript.workqueue.job.Process`, + `Queue.pipe_handlers` and `Queue._create_pipe()`. The queue, accounts and account + pools use the locks of `threading` instead of those of `multiprocessing`. - **`AccountPool.reset()` With Owners:** Releases the accounts that owners acquired, then removes all accounts. Before, it failed with `RuntimeError: dictionary changed size during iteration` if an account was acquired with an owner, as in diff --git a/docs/migration.rst b/docs/migration.rst index 242affd3..d37fe692 100644 --- a/docs/migration.rst +++ b/docs/migration.rst @@ -76,7 +76,38 @@ Key logins Responses ``conn.get_response()`` returns the response without the echo of the command, control characters and terminal escape sequences, and with - ``\n`` line ends. ``conn.response`` is unchanged. + ``\n`` line ends. ``conn.response`` no longer ends with the ``\r`` of + the line break before the prompt, and ``send()`` sets it to ``None``. + :func:`~Exscript.util.match.first_match` and + :func:`~Exscript.util.match.any_match` search ``conn.get_response()`` + when they get a connection; pass ``conn.response`` to search it as + before. + +Errors in ``expect()`` + ``expect()`` raises + :class:`~Exscript.protocols.exception.InvalidCommandException` if a + line of the response matches an error pattern, as ``expect_prompt()`` + does; 2.x checked only in ``expect_prompt()``. The data is consumed all + the same, so catch the exception where such a line is expected. + +Debug output + The messages of ``debug=`` and of ``Queue(verbose=2)`` and higher go to + Python's :mod:`logging`, to loggers such as ``Exscript.protocols.ssh2`` + and ``Exscript.queue``, instead of to ``stderr`` and ``stdout``. + Configure logging to see them, e.g. with + ``logging.basicConfig(level=logging.DEBUG)``. + +Output and log files + The ``stdout`` stream and the ``logfile`` of an adapter get the data as + received, also the ``\r`` of ``\r\n`` line breaks; 2.x removed it. + +``connect()`` + Raises a ``ProtocolException`` if the adapter is connected already; + call ``close()`` first. + +Connect rate limit + Connects to the same host are spaced by 1.1 seconds by default. Pass + ``connect_rate_limit=0`` to connect at once, as 2.x did. Generic prompt A match of the prompt pattern of the generic driver, which the @@ -132,6 +163,24 @@ Last line A template also sends its last line if the line does not end with a line break. 2.x ignored such a line. +Escape sequences + Commands and strings evaluate escape sequences such as ``\t``, + ``\x41`` or ``\u00e4``, e.g. ``show \x41`` sends ``show A``; in 2.x, a + backslash only made the next character literal. Escape the backslash, + as in ``\\x41``, or pass ``no_escaping=True`` to + :func:`~Exscript.util.template.eval`, ``eval_file()``, ``paste()`` or + ``paste_file()`` (command line: ``--no-escaping``) for the old + behavior; with it, ``\t`` in a string is still a tab. + +Comments + A hash behind an escaped character is text: ``show \}# x`` sends + ``show }# x``; 2.x sent ``show }``. + +```` elements + A template sends the content of an ```` element as a command of + its own, without the tags, for NETCONF. To send the tags as text, + escape them: ``\``, ``\``. + ``extract`` Requires a regular expression between slashes. 2.x also accepted a string in double quotes, e.g. ``{extract "(a)" as y}``. @@ -161,14 +210,28 @@ Values of the caller the tuple, and ``{x = t}`` copies the items instead of storing the tuple as one item. +Undefined variables + An undefined variable in an expression raises a ``NameError`` when the + template is compiled, instead of a ``ValueError``. + +Errors of functions + An exception of a function, other than a + :class:`~Exscript.protocols.exception.ProtocolException`, stops the + template with an :class:`~Exscript.parselib.exception.ExecuteError` + that shows the position of the call; the exception of the function is + its ``__context__``. 2.x raised the exception of the function. ``$f`` + for a function ``f`` in a command or a string raises an + ``ExecuteError``, too; 2.x inserted the text of the function object. + Command line tool ----------------- -Besides the defaults above: a value entered at a ``-d VAR={#"PROMPT"}`` -prompt is parsed as a Python literal, such as a number or a list, instead -of being run as Python code, and ``--sleep`` accepts seconds with an -optional suffix ``m``, ``h`` or ``d`` instead of running ``sleep`` in a -shell. +Besides the defaults above: a value entered at a ``-d VAR={"PROMPT"}`` +prompt is text instead of being run as Python code, so enter it without +quotes; for a number or a list, write ``-d VAR={#"PROMPT"}``, which +parses a Python literal. ``--sleep`` accepts seconds with an optional +suffix ``m``, ``h`` or ``d`` instead of running ``sleep`` in a shell. The +debug messages of ``-v 2`` and higher go to the standard error. Python API ---------- @@ -203,7 +266,8 @@ private. Use these instead: - ``conn.logfile``, the path of the log file; setting it switches to another file * - ``SSH2.client``, ``shell``, ``sock``, ``Telnet.tn``, - ``conn.last_account`` + ``conn.last_account``, ``conn.driver_replaced``, + ``Dummy.init_done`` - none; they are internal Other renamed or removed names: @@ -213,7 +277,10 @@ Other renamed or removed names: package ``standard-telnetlib`` as a replacement for the telnetlib module that Python 3.13 removed. - ``OsGuesser.data_received()`` and ``protocol_info()`` are - ``check_received_data()`` and ``check_protocol_info()``. + ``check_received_data()`` and ``check_protocol_info()``. Its attributes + ``info``, ``os_map``, ``auth_os_map``, ``protocol_os_map`` and + ``auth_buffer`` are private; use ``get()`` and ``set()``. + ``conn.os_guesser`` is read-only. - ``Exscript.util.template.test_secure()`` is ``test(secure=True)``. - ``Exscript.util.interact.InputHistory.file`` is ``filename``. - ``Exscript.util.daemonize``, ``Exscript.util.pidutil``, @@ -222,11 +289,12 @@ Other renamed or removed names: To stop the connections of a queue at once on Ctrl+C, as ``SigIntWatcher`` did, end the process with ``os._exit()`` in a handler of ``signal.SIGINT``. -- ``WorkQueue`` has no ``debug`` argument, and ``set_debug()`` takes +- ``WorkQueue`` has no ``debug`` argument, and ``set_debug()`` requires ``level``. Pass the other arguments by keyword. - ``Queue(mode='multiprocessing')`` raises a ``ValueError``: jobs run in threads only. For several CPU cores, see :doc:`platforms`. -- ``Exscript.account.AccountProxy`` and ``Exscript.logger.LoggerProxy`` are +- ``Exscript.account.AccountProxy``, ``Exscript.logger.LoggerProxy``, + ``Exscript.workqueue.job.Process`` and ``Queue.pipe_handlers`` are removed; jobs use the accounts and the loggers directly. - ``WorkQueue.queue_empty_event`` is emitted when the last job of the queue has ended, and no longer when the queue stops. Code that used it to learn @@ -239,6 +307,13 @@ Other renamed or removed names: ``ExpressionNode.parent_node``, the ``prio`` argument of ``Expression.prioritize()``, and ``lft``, ``rgt``, ``op`` and ``priority()`` of ``Term``. +- ``string_re`` of ``Exscript.interpreter.string`` is ``varstring_re``; + ``bracket_re`` and ``Regex.n_groups`` of ``Exscript.interpreter.regex`` + are replaced by ``Regex.groups``. ``HPProCurveDriver.clean_res_re`` is + removed. +- :func:`~Exscript.util.file.get_accounts_from_file` decodes the + passwords as UTF-8; 2.x decoded them as latin-1. For non-ASCII passwords + that were encoded as latin-1, pass ``password_encoding='latin-1'``. - Adapters that subclass ``Protocol`` must pass its arguments by keyword; new ones were inserted. diff --git a/docs/templates.rst b/docs/templates.rst index e192b574..462bbf72 100644 --- a/docs/templates.rst +++ b/docs/templates.rst @@ -28,6 +28,20 @@ that ends with the line, e.g. in ``{x = 1} # set x``. Escaped characters are text: ``\# text`` sends the command ``# text``, and ``show \}# x`` sends ``show }# x``. +Escape Sequences +---------------- + +A backslash makes the next character text, e.g. in ``\{``, ``\$`` or +``\\``. Commands and strings also evaluate the escape sequences of +Python: ``\n``, ``\r``, ``\t``, ``\0``, ``\a``, ``\b``, ``\f``, ``\v``, +``\e`` (escape), octal sequences of three digits such as ``\101``, +``\xhh``, ``\uhhhh`` and ``\Uhhhhhhhh``. For example, ``show \x41`` +sends ``show A``. With ``no_escaping=True`` (command line: +``--no-escaping``), only ``\n``, ``\r`` and ``\t`` in strings are +evaluated, and other escaped characters are text. In regular +expressions, a backslash keeps its meaning for Python's :mod:`re`, e.g. +in ``/\d+/``. + Using Variables --------------- From 6eb1c0e0a35e70bd87dea4a0913b1bbe0f4bac46 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 16:10:20 +0100 Subject: [PATCH 241/250] Fix the lower bounds of the dependencies and extras paramiko before 3.5.1 cannot load encrypted PEM keys with current cryptography. pycryptodomex 3.17 is the first version with wheels for Python 3.12 on all platforms, Apple Silicon included. The otel extra needs opentelemetry-api 1.30 for the bucket advisory of create_histogram(); opentelemetry-instrumentation 0.51b0 goes with it. --- CHANGES.md | 6 ++++-- pyproject.toml | 12 ++++++------ uv.lock | 20 ++++++++++---------- 3 files changed, 20 insertions(+), 18 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index fbcc7965..e0f0e3fd 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -24,8 +24,10 @@ including those of the 2.9 development line. Breaking changes are marked with `Any` on purpose (callbacks, options that are passed on, template variables) or comes from ncclient, which is not annotated. CI checks that this share does not drop. - **Dependencies:** `future` and `configparser`, backports for Python 2, are no longer - needed. paramiko 3.4 to 5.x is supported (before: 1.17 to 4.x). New: ncclient and - lxml 5.0 or later, for NETCONF. + needed. paramiko 3.5.1 to 5.x is supported (before: 1.17 to 4.x), and pycryptodomex + 3.17 or later (before: any version). New: ncclient and lxml 5.0 or later, for + NETCONF. The `otel` extra needs opentelemetry-api 1.30 and + opentelemetry-instrumentation 0.51b0 or later. - **Console Commands:** The commands `exscript` and `otp` keep their names, and are now console entry points of the package: `Exscript.cli.main()` and `otp_main()`. `python -m Exscript` runs `exscript`, too. On Windows, the commands are `.exe` diff --git a/pyproject.toml b/pyproject.toml index eff92430..8c94e1fe 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,14 +39,14 @@ classifiers = [ dependencies = [ "lxml>=5.0", # 5.0 stopped resolving external entities by default, also in ncclient "ncclient>=0.7,<1", - "paramiko>=3.4,<6", - "pycryptodomex", + "paramiko>=3.5.1,<6", # older versions cannot load encrypted PEM keys with current cryptography + "pycryptodomex>=3.17", # the first version with wheels for Python 3.12 also on Apple Silicon ] [project.optional-dependencies] otel = [ - "opentelemetry-api>=1.27", - "opentelemetry-instrumentation>=0.48b0", + "opentelemetry-api>=1.30", # create_histogram() with explicit_bucket_boundaries_advisory + "opentelemetry-instrumentation>=0.51b0", # the release for opentelemetry-api 1.30 ] cli = [ "rich>=13", @@ -66,8 +66,8 @@ Changelog = "https://github.com/knipknap/exscript/blob/master/CHANGES.md" test = [ "pytest>=9.2", "pytest-cov>=7.1", - "opentelemetry-sdk>=1.27", - "opentelemetry-instrumentation>=0.48b0", + "opentelemetry-sdk>=1.30", + "opentelemetry-instrumentation>=0.51b0", "hypothesis>=6.170.14", ] types = ["mypy>=2.4", "types-lxml>=2026.2", "types-paramiko>=5.1"] diff --git a/uv.lock b/uv.lock index 3854f779..3ef54248 100644 --- a/uv.lock +++ b/uv.lock @@ -668,10 +668,10 @@ types = [ requires-dist = [ { name = "lxml", specifier = ">=5.0" }, { name = "ncclient", specifier = ">=0.7,<1" }, - { name = "opentelemetry-api", marker = "extra == 'otel'", specifier = ">=1.27" }, - { name = "opentelemetry-instrumentation", marker = "extra == 'otel'", specifier = ">=0.48b0" }, - { name = "paramiko", specifier = ">=3.4,<6" }, - { name = "pycryptodomex" }, + { name = "opentelemetry-api", marker = "extra == 'otel'", specifier = ">=1.30" }, + { name = "opentelemetry-instrumentation", marker = "extra == 'otel'", specifier = ">=0.51b0" }, + { name = "paramiko", specifier = ">=3.5.1,<6" }, + { name = "pycryptodomex", specifier = ">=3.17" }, { name = "rich", marker = "extra == 'cli'", specifier = ">=13" }, ] provides-extras = ["cli", "otel"] @@ -680,8 +680,8 @@ provides-extras = ["cli", "otel"] dev = [ { name = "hypothesis", specifier = ">=6.170.14" }, { name = "mypy", specifier = ">=2.4" }, - { name = "opentelemetry-instrumentation", specifier = ">=0.48b0" }, - { name = "opentelemetry-sdk", specifier = ">=1.27" }, + { name = "opentelemetry-instrumentation", specifier = ">=0.51b0" }, + { name = "opentelemetry-sdk", specifier = ">=1.30" }, { name = "pytest", specifier = ">=9.2" }, { name = "pytest-cov", specifier = ">=7.1" }, { name = "rich", specifier = ">=15.0.0" }, @@ -696,15 +696,15 @@ docs = [ gevent = [ { name = "gevent", specifier = ">=26" }, { name = "hypothesis", specifier = ">=6.170.14" }, - { name = "opentelemetry-instrumentation", specifier = ">=0.48b0" }, - { name = "opentelemetry-sdk", specifier = ">=1.27" }, + { name = "opentelemetry-instrumentation", specifier = ">=0.51b0" }, + { name = "opentelemetry-sdk", specifier = ">=1.30" }, { name = "pytest", specifier = ">=9.2" }, { name = "pytest-cov", specifier = ">=7.1" }, ] test = [ { name = "hypothesis", specifier = ">=6.170.14" }, - { name = "opentelemetry-instrumentation", specifier = ">=0.48b0" }, - { name = "opentelemetry-sdk", specifier = ">=1.27" }, + { name = "opentelemetry-instrumentation", specifier = ">=0.51b0" }, + { name = "opentelemetry-sdk", specifier = ">=1.30" }, { name = "pytest", specifier = ">=9.2" }, { name = "pytest-cov", specifier = ">=7.1" }, ] From 757df61f012ecc71a9cf4bd7e708bd905db605be Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 16:12:37 +0100 Subject: [PATCH 242/250] Test the lowest dependency versions in CI --- .github/workflows/ci.yml | 21 +++++++++++++++++++++ docs/platforms.rst | 3 ++- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2448dfc9..48f9aa45 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -63,6 +63,27 @@ jobs: uv run --frozen --python "$PYTHON" pytest --cov=Exscript --cov-report=term-missing:skip-covered --cov-fail-under="$COV_FAIL_UNDER" + lowest: + name: Lowest dependency versions + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + fetch-depth: 0 + - uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3 + # Only the dependencies of Exscript and its extras get their lowest versions, the test + # tools the newest that fit. uv pip neither uses nor changes uv.lock. + - name: Install the lowest versions that pyproject.toml allows, on Python 3.12 + run: | + uv venv --python 3.12 + uv pip compile pyproject.toml --all-extras --resolution lowest-direct --python-version 3.12 -o lowest.txt + uv pip install -r lowest.txt -e . --group test + # The tests of the extras skip instead of failing if these imports fail. + uv run --no-sync python -c 'import Exscript.contrib.otel, rich.console' + - name: Run tests + run: uv run --no-sync pytest -p no:cacheprovider + build: name: Build distributions runs-on: ubuntu-latest diff --git a/docs/platforms.rst b/docs/platforms.rst index bfb9c6ec..fca16662 100644 --- a/docs/platforms.rst +++ b/docs/platforms.rst @@ -4,7 +4,8 @@ Platforms Exscript runs on Linux, macOS and Windows with Python 3.12 or later. CI runs the tests with every supported Python version on Linux, with the latest one on macOS and Windows, and with the free-threaded build of -Python 3.14. +Python 3.14. On Python 3.12, it also runs them with the lowest versions +of the dependencies and extras that Exscript allows. Most of Exscript works the same on every platform: for example, it reads and writes text files such as templates, host lists and logs as UTF-8 From 130ed2d40c686bfedf0eb6ae68b621d742272cc1 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 16:24:52 +0100 Subject: [PATCH 243/250] State the required type coverage in CHANGES.md, not a measured one --- CHANGES.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index e0f0e3fd..f8097a97 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -20,9 +20,9 @@ including those of the 2.9 development line. Breaking changes are marked with versions before 3.12 are no longer supported. - **Type Annotations:** Exscript is annotated and ships the `py.typed` marker (PEP 561), so type checkers such as mypy and pyright check the code that uses it. All modules - pass `mypy --strict`. mypy knows the type of 95.7 % of the expressions; the rest is - `Any` on purpose (callbacks, options that are passed on, template variables) or comes - from ncclient, which is not annotated. CI checks that this share does not drop. + pass `mypy --strict`. CI requires that mypy knows the type of at least 95 % of the + expressions; the rest is `Any` on purpose (callbacks, options that are passed on, + template variables) or comes from ncclient, which is not annotated. - **Dependencies:** `future` and `configparser`, backports for Python 2, are no longer needed. paramiko 3.5.1 to 5.x is supported (before: 1.17 to 4.x), and pycryptodomex 3.17 or later (before: any version). New: ncclient and lxml 5.0 or later, for From 7200bd4e4d3803dfd7819f6b312ed17fd4692a62 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 16:43:16 +0100 Subject: [PATCH 244/250] Accept values with blanks in exscript -d The pattern for NAME=VALUE ended the value at the first blank, so -d "NAME=VALUE WITH BLANKS" failed with "invalid value". --- src/Exscript/cli.py | 3 ++- tests/Exscript/cliTest.py | 11 +++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/src/Exscript/cli.py b/src/Exscript/cli.py index ce745c1f..1fcaa203 100644 --- a/src/Exscript/cli.py +++ b/src/Exscript/cli.py @@ -73,7 +73,8 @@ from rich.console import Console _bracket_expression = r"\{([^\]]*)\}" -_define_expression = rf"^\S+=(?:\S+|{_bracket_expression})$" +# NAME=VALUE, where the value may contain blanks but must not start with one. +_define_expression = r"^\S+=\S" _bracket_expression_re = re.compile(rf"^{_bracket_expression}$") _define_expression_re = re.compile(_define_expression) _duration_re = re.compile(r"(\d+(?:\.\d*)?|\.\d+)([smhd]?)") diff --git a/tests/Exscript/cliTest.py b/tests/Exscript/cliTest.py index 89c216e4..02b4d777 100644 --- a/tests/Exscript/cliTest.py +++ b/tests/Exscript/cliTest.py @@ -283,6 +283,8 @@ def testDefinePrompts(self): def testDefineErrors(self): cases = ( ('a', 'invalid value: a'), + ('a=', 'invalid value: a='), + ('a= b', 'invalid value: a= b'), ('__filename__=x', "Assignment to internal variable '__filename__'"), ) for value, error in cases: @@ -296,6 +298,15 @@ def testDefineValueWithEqualSign(self): self.assertEqual(code, 0, stderr) self.assertEqual(stdout, 'b=c\n') + def testDefineValueWithBlanks(self): + args = '-d', 'a=b c ', '-d', 'd={Enter d}' + script = '{sys.message(a)}{sys.message(d)}' + with mock.patch('builtins.input', return_value='e f') as input: + code, stdout, stderr = self.run_main('-n', '-i', *args, '-e', script, 'dummy://r1') + self.assertEqual(code, 0, stderr) + self.assertEqual(stdout, 'b c \ne f\n') + input.assert_called_once_with('Please enter Enter d: ') + def testLoginPrompt(self): with ( mock.patch('builtins.input', return_value='joe') as input, From f2ac1359d5eb58d6d343a987bb88254d52254d6c Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 16:45:11 +0100 Subject: [PATCH 245/250] Raise binascii.Error and ValueError for undecodable account passwords get_accounts_from_file() raised a plain Exception, whose message and chained exception showed parts of the password. --- CHANGES.md | 6 +++--- src/Exscript/util/file.py | 21 ++++++++++++++------- tests/Exscript/util/fileTest.py | 20 ++++++++++++++++++-- 3 files changed, 35 insertions(+), 12 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index f8097a97..23c55b22 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -796,9 +796,9 @@ The names below refer to functions callable from templates, not protocol methods searched `conn.response`. - **BREAKING - Passwords of `get_accounts_from_file()`:** The passwords in the file are decoded as UTF-8 after base64, or with the encoding of the new argument - `password_encoding`. A password that is no valid base64 or text in that encoding - raises an `Exception` that names the user. Before, the passwords were decoded as - latin-1, and invalid base64 raised a `binascii.Error`. + `password_encoding`; before, as latin-1. A password that is not valid in that + encoding raises a `ValueError`. Its message, and that of the `binascii.Error` for + invalid base64, names the user, but not the password. - **BREAKING - Removed Modules:** `Exscript.util.daemonize`, `Exscript.util.pidutil`, `Exscript.util.sigint` and `Exscript.util.sigintcatcher` are removed; `daemonize()` failed on Python 3 with `ValueError: can't have unbuffered text I/O`. Run long-lived scripts under a service manager such as systemd diff --git a/src/Exscript/util/file.py b/src/Exscript/util/file.py index d87983bd..ae21cc70 100644 --- a/src/Exscript/util/file.py +++ b/src/Exscript/util/file.py @@ -59,7 +59,9 @@ def get_accounts_from_file( python3 -c 'import base64; print(base64.b64encode("thepassword".encode()).decode())' Raises OSError if the file cannot be opened, e.g. FileNotFoundError if - it does not exist, and an Exception if a password can not be decoded. + it does not exist, binascii.Error if a password is not valid base64, and + ValueError if it is not valid in the password encoding. Their messages + name the user, but not the password. :type filename: str|os.PathLike :param filename: The name of the file containing the list of accounts. @@ -77,14 +79,19 @@ def get_accounts_from_file( cast(Any, parser).optionxform = str with open(filename, encoding=encoding) as fp: # read() skips files it cannot open parser.read_file(fp) - password: str | bytes # read as a string, then decoded to bytes for user, password in parser.items('account-pool'): + # The errors of the decoders show parts of the password: not chained. try: - password = base64.decodebytes(password.encode('ascii')) - accounts.append(Account(user, password.decode(password_encoding))) - except (UnicodeDecodeError, binascii.Error) as exc: - msg = 'Could not decode password for user: %s (%s)' % (user, exc) - raise Exception(msg) + data = base64.decodebytes(password.encode('ascii')) + except (UnicodeEncodeError, binascii.Error): + msg = 'Could not decode password for user: %s (invalid base64)' % user + raise binascii.Error(msg) from None + try: + text = data.decode(password_encoding) + except UnicodeDecodeError: + msg = 'Could not decode password for user: %s (invalid %s)' % (user, password_encoding) + raise ValueError(msg) from None + accounts.append(Account(user, text)) return accounts diff --git a/tests/Exscript/util/fileTest.py b/tests/Exscript/util/fileTest.py index e0d3e16c..2c7cd811 100644 --- a/tests/Exscript/util/fileTest.py +++ b/tests/Exscript/util/fileTest.py @@ -1,7 +1,9 @@ import base64 +import binascii import os.path import re import tempfile +import traceback import unittest from tempfile import NamedTemporaryFile @@ -104,12 +106,26 @@ def testGetAccountsFromFileWithOtherSections(self): def testGetAccountsFromFileWithInvalidPasswords(self): from Exscript.util.file import get_accounts_from_file - for password in 'not base64!', base64.b64encode(b'\xffpassword').decode('ascii'): + undecodable = base64.b64encode(b'\xffS3cr3t').decode('ascii') + cases = ( + ('S3cr3t!', binascii.Error, 'invalid base64'), + ('S3cr3t-\u00e4', binascii.Error, 'invalid base64'), + (undecodable, ValueError, 'invalid utf-8'), + ) + for password, cls, reason in cases: with self.subTest(password=password): data = '[account-pool]\nuser1 = %s\n' % password filename = self.write_tempfile(data.encode('utf8')).name - with self.assertRaisesRegex(Exception, 'Could not decode password for user: user1'): + with self.assertRaises(cls) as cm: get_accounts_from_file(filename) + self.assertIs(type(cm.exception), cls) + message = 'Could not decode password for user: user1 (%s)' % reason + self.assertEqual(str(cm.exception), message) + # Neither the message nor the traceback shows the password or its bytes. + self.assertTrue(cm.exception.__suppress_context__) + output = ''.join(traceback.format_exception(cm.exception)) + for secret in password, 'S3cr3t', '0xff', '\\xff': + self.assertNotIn(secret, output) def testGetAccountsFromFileWithPasswordEncoding(self): from Exscript.util.file import get_accounts_from_file From 6240a85acf9c65a69ecfeb9dc30ca77557231ee9 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 16:47:12 +0100 Subject: [PATCH 246/250] Raise a ValueError and NameError for undefined template variables An undefined variable in an expression raised a NameError; handlers for the ValueError of 2.6 missed it. UndefinedVariableError is both. --- CHANGES.md | 7 ++++--- docs/migration.rst | 4 ---- src/Exscript/interpreter/exception.py | 10 ++++++++++ src/Exscript/interpreter/term.py | 7 ++++--- tests/Exscript/interpreter/ExpressionTest.py | 5 ++++- 5 files changed, 22 insertions(+), 11 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 23c55b22..a602d962 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -624,9 +624,10 @@ development versions; "before" refers to them. element starts a comment, as behind a code block. The 2.9 development versions sent an empty line for `\# x` and for a hash behind an `` element, and took the next line into the command for `show \}# x`. -- **BREAKING - Undefined Variables Raise `NameError`:** A template that uses an - undefined variable in an expression, e.g. in `{if x is "a"}`, raises a `NameError` - when it is compiled. Before, it raised a `ValueError`. +- **Undefined Variables:** A template that uses an undefined variable in an + expression, e.g. in `{if x is "a"}`, raises the new + `Exscript.interpreter.exception.UndefinedVariableError` when it is compiled. It is a + `ValueError`, as before, and a `NameError`. - **BREAKING - `` Elements:** A template sends the content of an `` element, e.g. `...`, as a command of its own, without the `rpc` tags, and checks its XML syntax when it is compiled; `Netconf` sends it as an diff --git a/docs/migration.rst b/docs/migration.rst index d37fe692..2f705cf3 100644 --- a/docs/migration.rst +++ b/docs/migration.rst @@ -210,10 +210,6 @@ Values of the caller the tuple, and ``{x = t}`` copies the items instead of storing the tuple as one item. -Undefined variables - An undefined variable in an expression raises a ``NameError`` when the - template is compiled, instead of a ``ValueError``. - Errors of functions An exception of a function, other than a :class:`~Exscript.protocols.exception.ProtocolException`, stops the diff --git a/src/Exscript/interpreter/exception.py b/src/Exscript/interpreter/exception.py index b1ffd6c3..75ef1a64 100644 --- a/src/Exscript/interpreter/exception.py +++ b/src/Exscript/interpreter/exception.py @@ -48,3 +48,13 @@ class PermissionError(ExscriptException): """ pass + + +class UndefinedVariableError(NameError, ValueError): + """ + Raised if an expression of a template uses a variable that is not + defined when the template is compiled. It is a NameError and a + ValueError. + """ + + pass diff --git a/src/Exscript/interpreter/term.py b/src/Exscript/interpreter/term.py index b2d65ea7..35e2c7aa 100644 --- a/src/Exscript/interpreter/term.py +++ b/src/Exscript/interpreter/term.py @@ -27,6 +27,7 @@ from typing import TYPE_CHECKING, Any, cast from ..parselib import Lexer, Token +from .exception import UndefinedVariableError from .functioncall import FunctionCall from .number import Number from .regex import Regex @@ -47,8 +48,8 @@ class Term(Token): def __init__(self, lexer: Lexer, parser: 'Parser', parent: Scope) -> None: """ - Constructor. Parses the term. Raises a NameError if a variable is - not defined. + Constructor. Parses the term. Raises an UndefinedVariableError, a + NameError and a ValueError, if a variable is not defined. :param lexer: The lexer that reads the template. :param parser: The parser of the template. @@ -63,7 +64,7 @@ def __init__(self, lexer: Lexer, parser: 'Parser', parent: Scope) -> None: if not parent.is_defined(token): self.mark_start() self.mark_end(distance=len(token)) - lexer.error('Undefined variable %s' % repr(token), self, NameError) + lexer.error('Undefined variable %s' % repr(token), self, UndefinedVariableError) self.term = Variable(lexer, parser, parent) elif lexer.current_is('open_function_call'): self.term = FunctionCall(lexer, parser, parent) diff --git a/tests/Exscript/interpreter/ExpressionTest.py b/tests/Exscript/interpreter/ExpressionTest.py index ae39c6d7..0186ee51 100644 --- a/tests/Exscript/interpreter/ExpressionTest.py +++ b/tests/Exscript/interpreter/ExpressionTest.py @@ -1,5 +1,6 @@ import unittest +from Exscript.interpreter.exception import UndefinedVariableError from Exscript.parselib.exception import CompileError, ExecuteError from .helpers import TemplateTestCase @@ -26,8 +27,10 @@ def testListVariable(self): self.assertValues('{a = list.new("x", "y")}{b = a}', b=['x', 'y']) def testUndefinedVariable(self): - with self.assertRaises(NameError) as cm: + with self.assertRaises(UndefinedVariableError) as cm: self.evaluate('ls\n{x = nothing}') + self.assertIsInstance(cm.exception, NameError) + self.assertIsInstance(cm.exception, ValueError) message = "Undefined variable 'nothing' in undefined:2" self.assertError(cm.exception, ['{x = nothing}'], ' ^^^^^^^', message) self.assertSent() From 57b662967960e2d4da4b282473eb4d6ab7d2fa6a Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 16:54:21 +0100 Subject: [PATCH 247/250] Remove CR from the stdout and logfile output of Dummy As in 2.6, the Dummy adapter writes the received data without CR to its stdout stream and its logfile; the other adapters, interact() and data_received_event keep the data as received. --- CHANGES.md | 4 ---- docs/migration.rst | 4 ---- src/Exscript/protocols/dummy.py | 5 +++-- src/Exscript/protocols/protocol.py | 12 +++++++----- tests/Exscript/protocols/DummyTest.py | 23 +++++++++++++++++++++++ 5 files changed, 33 insertions(+), 15 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index a602d962..8668c3c4 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -256,10 +256,6 @@ including those of the 2.9 development line. Breaking changes are marked with - **BREAKING - `connect()` While Connected:** `connect()` raises a `ProtocolException` if the adapter is connected already. Before, it opened another connection and left the first one open. -- **BREAKING - Output and Logfile With `\r`:** The `stdout` stream and the `logfile` of - the protocol adapters get the data as it was received, also the `\r` of `\r\n` line - breaks. Before, `\r` was removed. The logs of `Exscript.util.log` and - `exscript --logdir` are unchanged. ### Interactive Shell diff --git a/docs/migration.rst b/docs/migration.rst index 2f705cf3..aefc8978 100644 --- a/docs/migration.rst +++ b/docs/migration.rst @@ -97,10 +97,6 @@ Debug output Configure logging to see them, e.g. with ``logging.basicConfig(level=logging.DEBUG)``. -Output and log files - The ``stdout`` stream and the ``logfile`` of an adapter get the data as - received, also the ``\r`` of ``\r\n`` line breaks; 2.x removed it. - ``connect()`` Raises a ``ProtocolException`` if the adapter is connected already; call ``close()`` first. diff --git a/src/Exscript/protocols/dummy.py b/src/Exscript/protocols/dummy.py index 8a97ab95..3b39d344 100644 --- a/src/Exscript/protocols/dummy.py +++ b/src/Exscript/protocols/dummy.py @@ -75,9 +75,10 @@ def is_dummy(self) -> bool: def _say(self, string: str) -> None: """ Handles the given text as if the device had sent it: passes it to - _receive_cb() and appends it to the buffer. + _receive_cb(), which writes it to stdout and the logfile without CR + characters, and appends it to the buffer. """ - self._receive_cb(string) + self._receive_cb(string, remove_cr=True) self._buffer.append(string) def _connect(self) -> bool: diff --git a/src/Exscript/protocols/protocol.py b/src/Exscript/protocols/protocol.py index 06e3c87d..a58ad10a 100644 --- a/src/Exscript/protocols/protocol.py +++ b/src/Exscript/protocols/protocol.py @@ -577,15 +577,17 @@ def _reset(self) -> None: self.stdout.seek(0) self._init() - def _receive_cb(self, data: str) -> None: + def _receive_cb(self, data: str, remove_cr: bool = False) -> None: """ Handles data that was received from the host: writes it to stdout and - to the logfile, updates the OS guess and the driver, and emits - data_received_event. + to the logfile, without CR characters if remove_cr is True, updates + the OS guess and the driver, and emits data_received_event with the + data as received. """ + text = data.replace('\r', '') if remove_cr else data with self._write_lock: - self._write_to_streams(data) - self._write_to_logs(data) + self._write_to_streams(text) + self._write_to_logs(text) # Check whether a better driver is found based on the incoming data. self._os_guesser.check_received_data(data, self.is_app_authenticated()) diff --git a/tests/Exscript/protocols/DummyTest.py b/tests/Exscript/protocols/DummyTest.py index 1428161f..985a52c6 100644 --- a/tests/Exscript/protocols/DummyTest.py +++ b/tests/Exscript/protocols/DummyTest.py @@ -1,3 +1,4 @@ +import io import os import re import tempfile @@ -172,12 +173,34 @@ def testErrorMessageShowsOnlyTheErrorLine(self): conn.close(force=True) +class OutputTest(unittest.TestCase): + def testStdoutAndLogfileWithoutCarriageReturns(self): + # The stdout stream and the logfile get the received data without CR, the + # listeners of data_received_event get it as the device sent it. + tmpdir = self.enterContext(tempfile.TemporaryDirectory()) + logfile = os.path.join(tmpdir, 'session.log') + stdout = io.StringIO() + received = [] + conn = Dummy(device=VirtualDevice('r1', strict=False), stdout=stdout, logfile=logfile) + conn.data_received_event.connect(received.append) + conn.connect('r1') + conn.login(Account('admin', 'password', needs_lock=False)) + conn.execute('ls') + self.assertEqual(conn.response, 'ls') + conn.close(force=True) + self.assertIn('ls\r\n', ''.join(received)) + self.assertEqual(stdout.getvalue(), ''.join(received).replace('\r', '')) + with open(logfile, encoding='utf-8', newline='') as fp: + self.assertEqual(fp.read(), stdout.getvalue().replace('\n', os.linesep)) + + def suite(): loader = unittest.TestLoader() return unittest.TestSuite( [ loader.loadTestsFromTestCase(DummyTest), loader.loadTestsFromTestCase(SecretHygieneTest), + loader.loadTestsFromTestCase(OutputTest), ] ) From f8c7db2d5a6c570f71a6f3ec40a876bb3ac548ce Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 17:02:47 +0100 Subject: [PATCH 248/250] Describe the changes against 2.6.32 in the changelog Several entries described the 2.9 development versions as the previous release; entries without a change against 2.6.32 are removed. --- CHANGES.md | 130 ++++++++++++++++----------------------------- docs/migration.rst | 1 + 2 files changed, 48 insertions(+), 83 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 8668c3c4..79b04618 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -115,11 +115,7 @@ including those of the 2.9 development line. Breaking changes are marked with `host_key_policy='auto-add'` or `host_key_policy='warn'`. - **Passwords in Debug Logs:** Passwords and S/Key responses sent by the app-level login are logged as ``. Before, debug level 4 logged them in clear text. - Debug levels above 5 no longer turn on paramiko's packet dumps, which showed the - password of an SSH login in clear text. - Listeners of `data_sent_event`, and the output and the logfile of a protocol whose - host does not echo, get `` instead of the password and the S/Key response, - too; before, `data_sent_event` passed them in clear text. + The listeners of the new `data_sent_event` get ``, too. - **BREAKING - `InvalidCommandException` Message:** The message contains only the line that matched an error pattern, for example `Error in the response from the device: % Invalid input detected`. Before, it contained the full response, which @@ -131,8 +127,8 @@ including those of the 2.9 development line. Breaking changes are marked with `sys.env`, and `file.*` except `file.exists`. Templates that call them raise `Exscript.interpreter.exception.PermissionError` when they are compiled. Functions that the caller passes as template variables are allowed. Before, secure mode was - available only for `test()` and off by default. To migrate, pass `secure=False`, - or use the new CLI option `--insecure`. + available only through `test_secure()`. To migrate, pass `secure=False`, or use the + new CLI option `--insecure`. - **BREAKING - `sys.exec` Without a Shell:** The template function `sys.exec` runs the command without a shell and waits for it to finish. Pipes, redirections and shell variables no longer work; use for example `sys.exec("sh -c '...'")`. @@ -143,10 +139,9 @@ including those of the 2.9 development line. Breaking changes are marked with - **BREAKING - CLI Without `eval()` and Shell:** A value entered at a `-d VAR={"PROMPT"}` prompt is text, and one entered at a `-d VAR={#"PROMPT"}` prompt is parsed as a Python literal, such as a number or a list. Before, the value of a - `{"PROMPT"}` prompt was run as Python code, so text had to be entered in quotes; the - 2.9 development versions ran the value of a `{#"PROMPT"}` prompt. `--sleep` no longer - runs the `sleep` command in a shell; it accepts seconds with an optional suffix `m`, - `h` or `d`, also on Windows. + `{"PROMPT"}` prompt was run as Python code, so text had to be entered in quotes. + `--sleep` no longer runs the `sleep` command in a shell; it accepts seconds with an + optional suffix `m`, `h` or `d`, also on Windows. ## Protocols @@ -170,10 +165,7 @@ including those of the 2.9 development line. Breaking changes are marked with - **Connect Rate Limit Per Host:** Connects to the same host, also concurrent ones, are spaced by 60 / `connect_rate_limit` + 0.1 seconds. The new option `connect_rate_limit` (also a `Host` option) is 60 connects per minute by default; 0 - turns the limit off. Connects to other hosts are not delayed. 2.6 had no limit. The - limiter of the 2.9 development versions slept while it held a process-wide lock, so - one waiting connect blocked all others, by default for up to 1.1 seconds, and - concurrent connects to the same host could start together. + turns the limit off. Connects to other hosts are not delayed. - **Fewer SSH Reconnects:** Drivers that reconnect between authentication methods (`ios`) now reconnect only after a method sent a request that failed. Before, they reconnected before every method, also before the public key methods when there was @@ -197,15 +189,16 @@ including those of the 2.9 development line. Breaking changes are marked with - **SSH Key Fingerprints in Debug Logs:** The keys that are tried for a login are logged with their SHA256 fingerprints, as `ssh-add -l` and `ssh-keygen -l` show them. Before, the MD5 fingerprint was logged as a Python bytes literal. -- **paramiko's Logger Stays On:** `SSH2` and `Netconf` connections with a debug level - below 5 keep paramiko's messages about them out of the logs with a logger of their - own, `Exscript.protocols.ssh2.paramiko`, which logs nothing. Before, they turned off - paramiko's logger `paramiko.transport` for the whole process, so that other code that - uses paramiko, and connections with debug level 5, logged nothing either. -- **SSH Banners in Another Encoding:** An authentication banner or a version string - that is not valid in the encoding of the connection no longer fails the login; the - invalid bytes become U+FFFD. Before, e.g. a latin-1 banner with `encoding='utf-8'` - raised a `UnicodeDecodeError`. +- **paramiko's Logger:** Importing Exscript leaves paramiko's logger `paramiko` as it + is. Before, importing Exscript set the level of this logger to `DEBUG` and gave it a + handler that wrote to `os.devnull`, so the handlers of the root logger got all debug + messages of paramiko. `SSH2` and `Netconf` connections with a debug level below 5 + keep paramiko's messages about them out of the logs with a logger of their own, + `Exscript.protocols.ssh2.paramiko`, which logs nothing. +- **BREAKING - SSH Banners:** `SSH2.get_banner()` returns the authentication banner as + text instead of bytes; bytes that are not valid in the encoding of the connection + become U+FFFD. The banner is also part of the received data, e.g. of the log, and + helps to guess the OS. - **Keyboard-Interactive Logins:** Every prompt gets an answer: password prompts the password, S/Key challenges the one-time password, other prompts, e.g. for a verification code, an empty string. Before, other prompts got no answer, and @@ -250,9 +243,7 @@ including those of the 2.9 development line. Breaking changes are marked with previous behavior without a terminal, pass `term_width=80, term_height=25`. - **Reliable `close()`:** `close()` closes the connection and resets the object even if the logout fails. Errors of the logout and while closing, e.g. because the host - closed the connection first, are only logged (debug level 1); before, errors of - the logout were ignored silently, and `close(force=True)` could raise an - `EOFError`. + closed the connection first, are only logged (debug level 1). - **BREAKING - `connect()` While Connected:** `connect()` raises a `ProtocolException` if the adapter is connected already. Before, it opened another connection and left the first one open. @@ -328,18 +319,10 @@ including those of the 2.9 development line. Breaking changes are marked with received data instead of a copy of all of it for each read: 8 MiB of output in 1 KiB reads take 0.015 seconds instead of 9.4. The positions in the match object that the callback gets refer to the searched text, not to the whole buffer. -- **NETCONF 1.1 Chunks:** `Netconf.reassemble_chunks()` takes linear time: a 16 MiB - reply in 4 KiB chunks takes 0.02 seconds instead of 5.2. -- **NETCONF Threads:** ncclient's thread appends the received data to the buffer that - the caller searches. The buffer is locked now; before, a search could move the - position at which the thread wrote. Exscript still sends its `` and RPCs - pretty-printed, but no longer makes other users of ncclient in the same process do - so (changes against the 2.9 development versions). -- **Free-Threaded Python:** `import Exscript` no longer imports lxml and ncclient; - NETCONF sessions and templates with RPCs load them when they need them. lxml - enables the GIL of free-threaded Python (3.14t), so before, Exscript always ran with - the GIL there. XML is parsed with a new parser for each call, so threads no longer - wait for each other. +- **Free-Threaded Python:** `import Exscript` does not import lxml and ncclient, + which NETCONF uses; NETCONF sessions and templates with RPCs import them when they + need them, as lxml enables the GIL of free-threaded Python (3.14t). XML is parsed + with a new parser for each call, so threads do not wait for each other. ### Telnet @@ -371,15 +354,20 @@ including those of the 2.9 development line. Breaking changes are marked with the telnetlib module that Python 3.13 removed, use the package `standard-telnetlib`. - **Closing While Receiving:** If `close()` of `Exscript.protocols.telnetlib.Telnet` runs in another thread while `receive()` waits for data, the connection stays - closed, and `receive()` returns `False`. Before, data that arrived in the meantime - cleared the end-of-file flag, and the next `receive()` failed with - `AttributeError: 'NoneType' object has no attribute 'settimeout'`. + closed, and `receive()` returns `False`. ### NETCONF The `Netconf` adapter is new in 3.0, so these fixes matter only to users of the 2.9 development versions; "before" refers to them. +- **NETCONF 1.1 Chunks:** `Netconf.reassemble_chunks()` takes linear time: a 16 MiB + reply in 4 KiB chunks takes 0.02 seconds instead of 5.2. +- **NETCONF Threads:** ncclient's thread appends the received data to the buffer that + the caller searches. The buffer is locked now; before, a search could move the + position at which the thread wrote. Exscript still sends its `` and RPCs + pretty-printed, but no longer makes other users of ncclient in the same process do + so. - **`close()` Ends ncclient's Thread:** `close()` ends the thread in which ncclient reads the session, also with `force=True` or if the server does not answer ``. If a callback, e.g. of a monitor, keeps the thread busy during @@ -425,16 +413,10 @@ development versions; "before" refers to them. constructor ignored such a name silently and guessed the driver, so a typo, e.g. in an inventory, went unnoticed. To migrate, correct the name; `drivers.driver_map` lists the names. -- **Junos and MRV Prompts in Linear Time:** The prompt patterns of the `junos` and `mrv` - drivers backtracked exponentially on line breaks that no prompt follows: 22 empty - lines took 41 seconds to check with `junos`, and every further line doubled the time, - so a device that sent a block of empty lines could stall the session. They now match - each run of line breaks once, in linear time, and match the same texts at the same - positions, verified against the old patterns. - **Generic Prompt Pattern in Linear Time:** The prompt pattern of the generic driver, which other drivers inherit, backtracked catastrophically on long lines that are no - prompt: a line with a run of 100 word characters took 1.3 seconds to check, 200 - characters 25 seconds, so hex or base64 output could stall a session. The pattern + prompt: a line with a run of 100 word characters took 0.6 seconds to check, 200 + characters 9 seconds, so hex or base64 output could stall a session. The pattern matches the same strings, verified against the old one, in linear time. A test checks every driver pattern for this. The isam driver's prompt pattern, which the OS guesser runs on all received data, matches the same strings 3.5 times faster. @@ -504,10 +486,10 @@ development versions; "before" refers to them. mode waited for the prompt until the timeout. - **NX-OS Over NETCONF:** A NETCONF session with NX-OS is guessed as `nxos` by the capability of its device model, `http://cisco.com/ns/yang/cisco-nx-os-device`, as - sessions with IOS XR and Junos are guessed by theirs. Before, the guess was - `unknown`. The driver keeps ncclient's default device handler: the `nexus` handler - replaces the base capability of RFC 6241 in the hello of the client with the one of - the old XML agent, which a server may reject. + sessions with IOS XR and Junos are guessed by theirs. The driver keeps ncclient's + default device handler: the `nexus` handler replaces the base capability of RFC 6241 + in the hello of the client with the one of the old XML agent, which a server may + reject. - **HP ProCurve Configuration Prompts:** The `hp_pro_curve` driver recognizes the prompts of the configuration contexts, e.g. `HP-2920(config)# ` or `HP-2920(vlan-10)# `. Before, a command that entered such a context, e.g. @@ -581,8 +563,7 @@ development versions; "before" refers to them. and the new option `--no-escaping` of `exscript`, keep the old behavior, except that `\t` in a string is a tab. A Unicode sequence above U+10FFFF, such as `\UFFFFFFFF`, and an octal sequence above `\377`, such as `\400`, raise a `CompileError` that shows - their position; the 2.9 development versions raised a `SyntaxError` of Python for the - former, and gave the characters U+0100 to U+01FF for the latter. + their position. - **BREAKING - `extract` Requires a Regular Expression:** `extract` raises a `CompileError` if no regular expression between slashes follows, e.g. in `{extract "(a)" as y from "a"}` or `{extract x as y}`. Before, the character after @@ -609,21 +590,16 @@ development versions; "before" refers to them. - **`is not` and `not in` With Several Blanks:** `is not` and `not in` with more than one blank or a line break between the words mean the same as `is not` and `not in`. Before, they gave `None`, and as a condition a `TypeError` of Python. -- **Lines With Only Whitespace:** A line with only whitespace is ignored, also if the - whitespace is not a space or a tab, e.g. a no-break space, as in 2.6. The 2.9 - development versions sent such whitespace at the start of the command in the next - line. - **BREAKING - Escaped Characters Start No Comment:** A hash behind an escaped character is part of the command: `show \}# x` sends `show }# x`, and `show a\n# b` sends `# b` in its second line. Before, the hash started a comment there, so these - sent `show }` and `show an`. `\# x` sends `# x`, as before. A hash behind an `` - element starts a comment, as behind a code block. The 2.9 development versions sent - an empty line for `\# x` and for a hash behind an `` element, and took the next - line into the command for `show \}# x`. + sent `show }` and `show an`. A hash behind an `` element starts a comment, as + behind a code block. - **Undefined Variables:** A template that uses an undefined variable in an expression, e.g. in `{if x is "a"}`, raises the new `Exscript.interpreter.exception.UndefinedVariableError` when it is compiled. It is a - `ValueError`, as before, and a `NameError`. + `ValueError`, as before, and a `NameError`. Its message says `Undefined variable 'x'` + instead of `Undeclared variable x`. - **BREAKING - `` Elements:** A template sends the content of an `` element, e.g. `...`, as a command of its own, without the `rpc` tags, and checks its XML syntax when it is compiled; `Netconf` sends it as an @@ -671,18 +647,12 @@ development versions; "before" refers to them. function like to a variable, e.g. `"$f"` with a function `f` that the caller passes, raises an `ExecuteError`, as the interpreter intends. Before, it contained the text of the function object, such as ``. -- **Syntax Errors of Custom Grammars:** If no expression of the grammar of - `Exscript.parselib.Lexer` matches, and the next 50 characters contain no blank and - not the end of the input, the lexer raises a `CompileError` that shows these 50 - characters. Before, it raised an `AttributeError`. The grammars of the template - language were not affected. -- **BREAKING - Errors of the Device in Functions:** If a function of a template raises - a `ProtocolException`, e.g. `connection.execline()` an `InvalidCommandException` for - a command that the device rejects, the exception stops the template unchanged, like - the error of a command, and `try` ignores it. Other exceptions of functions raise an - `ExecuteError` that shows the position of the call. Before, all of them raised an - `ExecuteError`, which `try` does not ignore; Exscript 2.6 raised all of them - unchanged. +- **BREAKING - Errors of Functions:** If a function of a template raises a + `ProtocolException`, e.g. `connection.execline()` an `InvalidCommandException` for a + command that the device rejects, the exception stops the template unchanged, like + the error of a command, and `try` ignores it, as before. Other exceptions of + functions raise an `ExecuteError` that shows the position of the call; before, they + were raised unchanged. ### Standard Library @@ -732,9 +702,6 @@ The names below refer to functions callable from templates, not protocol methods for the current user. If the name of the current user is unknown, it adds none, as without a default user name. Before, the CLI failed with the `OSError` of `getpass.getuser()`, or with a `KeyError` before Python 3.13. -- **`exscript --version`:** Prints the version of Exscript, as in 2.6 and as the - tutorials say. The 2.9 development versions failed with - `exscript: error: unrecognized arguments: --version`. - **`exscript --hosts` and `--csv-hosts` With `~` and Variables:** Expand `~` and environment variables in the name of the file, as `--account-pool` and `--load-driver` do, e.g. in `--hosts '~/$SITE/hosts.txt'`. Before, they read the @@ -745,9 +712,6 @@ The names below refer to functions callable from templates, not protocol methods `-i`; it never did. `-n` skips the automatic login, and unless `-i` is given, the command still asks for a username and password, so that the template can log in itself with them. -- **`exscript -e` and `__filename__`:** With `-e`, the template variable `__filename__` - is `undefined`, as for other templates that are not read from a file. Before, it was - the first host, which `-e` takes from the place of the file name. - **`Exscript.util.mail.send()`:** Ends the SMTP session with `QUIT` and closes the connection, also if sending fails; before, it left the connection open. It connects only after the mail and its attachments are ready. diff --git a/docs/migration.rst b/docs/migration.rst index aefc8978..7fd47b7a 100644 --- a/docs/migration.rst +++ b/docs/migration.rst @@ -306,6 +306,7 @@ Other renamed or removed names: - :func:`~Exscript.util.file.get_accounts_from_file` decodes the passwords as UTF-8; 2.x decoded them as latin-1. For non-ASCII passwords that were encoded as latin-1, pass ``password_encoding='latin-1'``. +- ``SSH2.get_banner()`` returns the banner as text instead of bytes. - Adapters that subclass ``Protocol`` must pass its arguments by keyword; new ones were inserted. From 6a783d20666188d500b5dad7be7764f16734d9a7 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 18:24:55 +0100 Subject: [PATCH 249/250] Use a terminal width of 510, since BusyBox's telnetd misreads 511 In 511 (0x01FF), the Telnet window size contains the byte 255, which is sent doubled. BusyBox's telnetd does not undouble it and passes a byte to the login, so 15 of 15 logins failed. --- CHANGES.md | 2 +- docs/migration.rst | 2 +- src/Exscript/protocols/protocol.py | 10 ++++++---- tests/Exscript/protocols/SSH2Test.py | 2 +- tests/Exscript/protocols/TelnetTest.py | 9 +++++++-- tests/recordings/README.md | 2 +- tools/record_session.py | 2 +- 7 files changed, 18 insertions(+), 11 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 79b04618..fd0c0e7b 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -232,7 +232,7 @@ including those of the 2.9 development line. Breaking changes are marked with SSH agent (`SSH_AUTH_SOCK`) as soon as the login is done with its keys. Before, the garbage collector closed it, with a `ResourceWarning`, and after a login with a key of the agent not before the end of the session. -- **BREAKING - Terminal Size:** SSH and Telnet sessions get a terminal of 511 columns +- **BREAKING - Terminal Size:** SSH and Telnet sessions get a terminal of 510 columns and 1000 lines; the new options `term_width` and `term_height` (also `Host` options) choose another size. Before, Exscript used the size of the terminal that ran the script, or 80×25 without one (cron, services), so the same script could diff --git a/docs/migration.rst b/docs/migration.rst index 7fd47b7a..d2832fc9 100644 --- a/docs/migration.rst +++ b/docs/migration.rst @@ -48,7 +48,7 @@ Connections ----------- Terminal size - Sessions get a terminal of 511 columns and 1000 lines, instead of the + Sessions get a terminal of 510 columns and 1000 lines, instead of the size of the terminal that runs the script, or 80×25 without one. Devices whose driver does not set the width themselves, e.g. Linux hosts, no longer wrap lines at 80 characters. For the old size, pass diff --git a/src/Exscript/protocols/protocol.py b/src/Exscript/protocols/protocol.py index a58ad10a..768838a9 100644 --- a/src/Exscript/protocols/protocol.py +++ b/src/Exscript/protocols/protocol.py @@ -260,7 +260,7 @@ def __init__( *, host_key_policy: str = 'reject', # SSH2 only known_hosts: str | None = None, # SSH2 only - term_width: int = 511, + term_width: int = 510, term_height: int = 1000, ) -> None: """ @@ -297,9 +297,11 @@ def __init__( :keyword term_width: The width of the terminal on the remote host, in characters, for SSH and Telnet. Hosts wrap or cut longer lines unless the driver changes the width, e.g. on Linux or NX-OS. The - default, 511, is the widest that many devices accept. The size - does not depend on the terminal that the script runs in; - interact() changes it to the size of that terminal. + default, 510, is about the widest that many devices accept. Avoid + sizes with a byte 255, such as 511: some Telnet servers, e.g. + BusyBox's, misread them. The size does not depend on the terminal + that the script runs in; interact() changes it to the size of that + terminal. :type term_height: int :keyword term_height: The height of the terminal on the remote host, in lines. Hosts may wait for a key press after this many lines diff --git a/tests/Exscript/protocols/SSH2Test.py b/tests/Exscript/protocols/SSH2Test.py index 171ec804..f25600ef 100644 --- a/tests/Exscript/protocols/SSH2Test.py +++ b/tests/Exscript/protocols/SSH2Test.py @@ -498,7 +498,7 @@ def requestedSize(self, **kwargs): return interface.sizes def testDefault(self): - self.assertEqual(self.requestedSize(), [(511, 1000)]) + self.assertEqual(self.requestedSize(), [(510, 1000)]) def testOptions(self): self.assertEqual(self.requestedSize(term_width=80, term_height=25), [(80, 25)]) diff --git a/tests/Exscript/protocols/TelnetTest.py b/tests/Exscript/protocols/TelnetTest.py index 30b05c3c..9b2ce33b 100644 --- a/tests/Exscript/protocols/TelnetTest.py +++ b/tests/Exscript/protocols/TelnetTest.py @@ -53,10 +53,15 @@ def serve(): return bytes(received) def testDefault(self): - # 511 columns = 0x01FF, so the byte 255 is doubled; 1000 rows = 0x03E8. - naws = bytes([IAC, SB, NAWS, 0x01, 0xFF, 0xFF, 0x03, 0xE8, IAC, SE]) + # 510 columns = 0x01FE, 1000 rows = 0x03E8. + naws = bytes([IAC, SB, NAWS, 0x01, 0xFE, 0x03, 0xE8, IAC, SE]) self.assertEqual(self.sentSize(), bytes([IAC, WILL, NAWS]) + naws) + def testByte255IsDoubled(self): + # 511 columns = 0x01FF. + naws = bytes([IAC, SB, NAWS, 0x01, 0xFF, 0xFF, 0x03, 0xE8, IAC, SE]) + self.assertEqual(self.sentSize(term_width=511), bytes([IAC, WILL, NAWS]) + naws) + def testOptions(self): naws = bytes([IAC, SB, NAWS, 0, 80, 0, 25, IAC, SE]) self.assertEqual( diff --git a/tests/recordings/README.md b/tests/recordings/README.md index 775ca7e5..b631fbd5 100644 --- a/tests/recordings/README.md +++ b/tests/recordings/README.md @@ -28,7 +28,7 @@ report in the job summary and fails if fewer sessions are guessed right than bef - The protocol is `ssh`, `telnet` or `netconf`. - `--os` is the ground truth: the Exscript driver name that `guess_os()` should return (`unknown` if no driver fits). - - Keep the default terminal type `dumb` and size 511×1000. They are Exscript's defaults, and + - Keep the default terminal type `dumb` and size 510×1000. They are Exscript's defaults, and the replay test uses the recorded values. - Use `--auth password` for SSH servers that drop the connection after a failed keyboard-interactive attempt. diff --git a/tools/record_session.py b/tools/record_session.py index 2fa37deb..73982547 100755 --- a/tools/record_session.py +++ b/tools/record_session.py @@ -324,7 +324,7 @@ def main(): '--term', default='dumb', help="terminal type (default: Exscript's default)" ) parser.add_argument( - '--width', type=int, default=511, help='terminal width (default: 511, like Exscript)' + '--width', type=int, default=510, help='terminal width (default: 510, like Exscript)' ) parser.add_argument( '--height', type=int, default=1000, help='terminal height (default: 1000, like Exscript)' From 24ddd098ed39dbbd122e7564d60044417a3e8b66 Mon Sep 17 00:00:00 2001 From: Marcel Vilas Date: Wed, 11 Nov 2026 18:24:55 +0100 Subject: [PATCH 250/250] Make the lab command files safe to record and publish They show no configuration, routes or hardware tables, edit a private candidate on Junos, and answer the confirm of clear counters with n. login-only.txt records only the login, and --login-fails marks a recording of a failed login. The sanitizer report shows no part of a secret. --- tests/Exscript/protocols/ReplayTest.py | 8 ++--- tests/recordings/README.md | 35 ++++++++++++++----- tests/recordings/commands/ios-basic.txt | 13 +++---- tests/recordings/commands/ios-pager.txt | 4 +-- tests/recordings/commands/ios_xr-basic.txt | 14 +++++--- tests/recordings/commands/junos-basic.txt | 14 ++++---- tests/recordings/commands/junos-pager.txt | 4 +-- tests/recordings/commands/login-only.txt | 2 ++ tests/recordings/commands/netconf-junos.xml | 2 +- tests/recordings/commands/netconf.xml | 2 +- tests/recordings/commands/nxos-basic.txt | 11 +++--- tests/recordings/commands/nxos-pager.txt | 4 +-- ...usybox-1.37-ash-telnet-badpass.golden.json | 6 ++++ .../busybox-1.37-ash-telnet-badpass.jsonl | 19 ++++++++++ tools/record_session.py | 8 ++++- tools/sanitize_recording.py | 8 +++-- 16 files changed, 109 insertions(+), 45 deletions(-) create mode 100644 tests/recordings/commands/login-only.txt create mode 100644 tests/recordings/linux/busybox-1.37-ash-telnet-badpass.golden.json create mode 100644 tests/recordings/linux/busybox-1.37-ash-telnet-badpass.jsonl diff --git a/tests/Exscript/protocols/ReplayTest.py b/tests/Exscript/protocols/ReplayTest.py index 1ae32b44..e06efa35 100644 --- a/tests/Exscript/protocols/ReplayTest.py +++ b/tests/Exscript/protocols/ReplayTest.py @@ -135,10 +135,10 @@ class ReplayTest(unittest.TestCase): def _truth_test(path): def test(self): result = replay(path, 'recorded')[0] - self.assertEqual('ok', result['login']) - expected_os = Recording(path).header['device'].get('os') - if expected_os: - self.assertEqual(expected_os, result['os']) + device = Recording(path).header['device'] + self.assertEqual('LoginFailure' if device.get('login_fails') else 'ok', result['login']) + if device.get('os'): + self.assertEqual(device['os'], result['os']) return test diff --git a/tests/recordings/README.md b/tests/recordings/README.md index b631fbd5..4b363c70 100644 --- a/tests/recordings/README.md +++ b/tests/recordings/README.md @@ -6,8 +6,8 @@ to it. The replay server (`tests/Exscript/protocols/replay.py`) runs on 127.0.0. recorded device output, so the tests need no devices. Each recording runs twice: with the output as recorded, and split into small random pieces. Both -runs must give the same result. A third test checks the device's ground truth: login succeeds and -`guess_os()` returns `device.os`. `tools/osguess_report.py` compares the guesses for all recordings +runs must give the same result. A third test checks the device's ground truth: the login succeeds +(fails, if the recording has `device.login_fails`) and `guess_os()` returns `device.os`. `tools/osguess_report.py` compares the guesses for all recordings and for the banners in `tests/Exscript/protocols/banners` with their ground truth; CI shows its report in the job summary and fails if fewer sessions are guessed right than before. @@ -18,10 +18,12 @@ report in the job summary and fails if fewer sessions are guessed right than bef for (e.g. after `enable`), and `!send TEXT` sends keys without a line end (`!send \x20` answers a pager). For NETCONF, separate the RPC bodies with lines of `---`. -2. Record the session. The recorder asks for the password and never stores it: +2. Record the session. The recorder asks for the password and never stores it. Write the raw + recording outside the checkout: it contains host names, addresses and user names, and may + contain secrets, so never commit or share it. ```sh - uv run tools/record_session.py ssh 192.0.2.1 -u admin -c cmds.txt -o raw.jsonl \ + uv run tools/record_session.py ssh 192.0.2.1 -u admin -c cmds.txt -o ~/exscript-raw/raw.jsonl \ --os ios --os-version 15.9 --platform C2960X ``` @@ -32,18 +34,24 @@ report in the job summary and fails if fewer sessions are guessed right than bef the replay test uses the recorded values. - Use `--auth password` for SSH servers that drop the connection after a failed keyboard-interactive attempt. + - OS guessing uses only the login and the first prompt, which `commands/login-only.txt` + records. It is quick to record and to review, so use it for every further model or version. - See `--help` for the other options. 3. Sanitize the recording, and replace host names, user names and anything else identifying: ```sh - uv run tools/sanitize_recording.py raw.jsonl tests/recordings/ios/ios-15.9-c2960x-ssh.jsonl \ + uv run tools/sanitize_recording.py ~/exscript-raw/raw.jsonl \ + tests/recordings/ios/ios-15.9-c2960x-ssh-basic.jsonl \ --replace core-rtr-01=router1 --replace jdoe=admin ``` The sanitizer replaces addresses, MACs, secrets and serial numbers, and prints every replacement and the prompts it saw. It cannot know every identifier: read the sanitized file before - committing it. Never commit a raw recording. + committing it, and look in particular for host, domain, site and organization names, the names + of other users (e.g. in "Last login" lines), banners, AS numbers, phone numbers, e-mail + addresses, and serial numbers in tables, which have no label such as `SN:` before them. Never + commit a raw recording. 4. Create the golden file, review it, and commit both files: @@ -55,8 +63,18 @@ report in the job summary and fails if fewer sessions are guessed right than bef `ReplayTest.py` with the reason. If its results depend on read timing, add it to `TIMING_ISSUES` instead. -Name files `/---[-].jsonl`. The directory groups -recordings by device family. +Name files `/---[-].jsonl`, in lower case, with `_` +instead of `-` within a part: + +- `` groups recordings by device family, e.g. `ios` (IOS and IOS XE), `iosxr`, `junos` or + `nxos`. +- `` names the system, e.g. `ios`, `iosxe`, `cxr`, `exr` or `xr7` (IOS XR 32-bit, 64-bit and + XR7), `junos`, `evo` (Junos OS Evolved) or `nxos`. +- `` is the release, e.g. `17.9.4a` or `23.4r2`; `--os-version` takes the full version. +- `` is the model, e.g. `asr9903` or `n9k_c93180yc_fx`. +- `` is the command file without its OS prefix, e.g. `basic` or `pager`; `login` for + `login-only.txt`, and `badpass` for `login-only.txt` over Telnet with a wrong password and + `--login-fails`. ## Format (version 1) @@ -77,6 +95,7 @@ The first line is the header: - `ssh` (SSH and NETCONF only) holds the server's version string, the pre-login banner, the authentication methods it offered, and the keyboard-interactive query it sent, if any. - `terminal` (CLI sessions only) holds the terminal that the recorder requested. +- `device.login_fails: true` (from `--login-fails`) marks a session whose login fails. - `incomplete: true` marks a recording that ended with an error. Each following line is an event: diff --git a/tests/recordings/commands/ios-basic.txt b/tests/recordings/commands/ios-basic.txt index 14c9a0dc..a096a7b7 100644 --- a/tests/recordings/commands/ios-basic.txt +++ b/tests/recordings/commands/ios-basic.txt @@ -1,14 +1,14 @@ -# Cisco IOS and IOS XE (R2, lab devices only), with --os ios. +# Cisco IOS and IOS XE (lab devices only), with --os ios. # Log in as a user that lands in user EXEC (">"); drop the enable and !secret lines if you land in "#". -# Changes the running config: creates and removes Loopback999. +# Creates and removes Loopback999 in the running config; check first that it does not exist. +# Shows no configuration, routes or descriptions, so that there is little to sanitize. enable !secret terminal length 0 terminal width 0 show version -show running-config -show interfaces -show ip route +show processes cpu +show processes memory show ip interface brief show inventory show nonexistent @@ -22,7 +22,8 @@ show running-config interface Loopback999 configure terminal no interface Loopback999 end +# n answers the [confirm], so no counter is cleared. clear counters -!send \r +!send n show this-command-is-much-longer-than-the-terminal-width | include something-that-does-not-match-anything-at-all exit diff --git a/tests/recordings/commands/ios-pager.txt b/tests/recordings/commands/ios-pager.txt index 2be6dcb8..8e15d0a2 100644 --- a/tests/recordings/commands/ios-pager.txt +++ b/tests/recordings/commands/ios-pager.txt @@ -1,7 +1,7 @@ -# Cisco IOS, IOS XE and IOS XR (R2): output with the pager on. +# Cisco IOS, IOS XE and IOS XR: output with the pager on. # The recorder asks for 1000 lines, so set the pager length explicitly. terminal length 24 -show running-config +show processes memory !send \x20 !send \x20 !send q diff --git a/tests/recordings/commands/ios_xr-basic.txt b/tests/recordings/commands/ios_xr-basic.txt index 44077c7c..7a3cb805 100644 --- a/tests/recordings/commands/ios_xr-basic.txt +++ b/tests/recordings/commands/ios_xr-basic.txt @@ -1,11 +1,12 @@ -# Cisco IOS XR, cXR and eXR (R2, lab devices only), with --os ios_xr. -# Changes the running config: creates and removes Loopback999 (two commits). +# Cisco IOS XR: cXR, eXR and XR7 (lab devices only), with --os ios_xr. +# Creates and removes Loopback999 in the running config (two commits); check first that it does not +# exist. Shows no configuration, routes or descriptions, so that there is little to sanitize. terminal length 0 terminal width 0 show version -show running-config +show processes cpu +show processes memory show interfaces brief -show route show inventory show nonexistent show ip @@ -20,4 +21,9 @@ no interface Loopback999 commit end show this-command-is-much-longer-than-the-terminal-width | include something-that-does-not-match-anything-at-all +# The admin mode has a prompt of its own (eXR: sysadmin-vm:0_RP0#). Where the user may not enter it, +# or it does not exist, the first exit ends the session. +admin +show platform +exit exit diff --git a/tests/recordings/commands/junos-basic.txt b/tests/recordings/commands/junos-basic.txt index 316a7069..6eeecf69 100644 --- a/tests/recordings/commands/junos-basic.txt +++ b/tests/recordings/commands/junos-basic.txt @@ -1,19 +1,21 @@ -# Juniper Junos and Junos Evolved (R2, lab devices only), with --os junos. -# Enters configuration mode but commits nothing (commit check, then rollback 0). +# Juniper Junos OS and Junos OS Evolved (lab devices only), with --os junos. +# Edits a private candidate configuration and commits nothing (commit check, then rollback 0). +# Shows no configuration, routes or hardware, so that there is little to sanitize. set cli screen-length 0 set cli screen-width 0 show version -show configuration +show system statistics show interfaces terse -show route -show chassis hardware show nonexistent show i -configure +configure private set interfaces lo0 description "Exscript test" show | compare commit check rollback 0 exit configuration-mode show this-command-is-much-longer-than-the-terminal-width | match something-that-does-not-match-anything-at-all +# The shell has a prompt of its own. Where the user may not start it, the first exit ends the session. +start shell +exit exit diff --git a/tests/recordings/commands/junos-pager.txt b/tests/recordings/commands/junos-pager.txt index b500babd..21a39e57 100644 --- a/tests/recordings/commands/junos-pager.txt +++ b/tests/recordings/commands/junos-pager.txt @@ -1,7 +1,7 @@ -# Juniper Junos (R2): output with the pager on. +# Juniper Junos OS and Junos OS Evolved: output with the pager on. # The recorder asks for 1000 lines, so set the pager length explicitly. set cli screen-length 24 -show configuration +show system statistics !send \x20 !send \x20 !send q diff --git a/tests/recordings/commands/login-only.txt b/tests/recordings/commands/login-only.txt new file mode 100644 index 00000000..1dc73503 --- /dev/null +++ b/tests/recordings/commands/login-only.txt @@ -0,0 +1,2 @@ +# Any OS: records the login and the first prompt only; then the recorder closes the connection. +# That is what OS guessing uses, so record it for every further model, version or system. diff --git a/tests/recordings/commands/netconf-junos.xml b/tests/recordings/commands/netconf-junos.xml index a13c5127..a27428dc 100644 --- a/tests/recordings/commands/netconf-junos.xml +++ b/tests/recordings/commands/netconf-junos.xml @@ -1,6 +1,6 @@ --- - +ietf-netconf --- --- diff --git a/tests/recordings/commands/netconf.xml b/tests/recordings/commands/netconf.xml index 954197b1..8b98f9e8 100644 --- a/tests/recordings/commands/netconf.xml +++ b/tests/recordings/commands/netconf.xml @@ -1,4 +1,4 @@ - +ietf-netconf --- --- diff --git a/tests/recordings/commands/nxos-basic.txt b/tests/recordings/commands/nxos-basic.txt index 1bc399ef..f39b28a9 100644 --- a/tests/recordings/commands/nxos-basic.txt +++ b/tests/recordings/commands/nxos-basic.txt @@ -1,14 +1,15 @@ -# Cisco NX-OS (R2, lab devices only), with --os nxos. -# Log in as a network-admin. Changes the running config: creates and removes loopback999. +# Cisco NX-OS (lab devices only), with --os nxos. +# Log in as a network-admin. Creates and removes loopback999 in the running config; check first that +# it does not exist. Shows no configuration, routes or descriptions, so that there is little to sanitize. terminal length 0 show version -show running-config +show processes cpu +show processes memory show interface brief -show ip route show inventory show nonexistent show i -show interface ethernet +show vlan id show interface ethernet 99/99 configure terminal interface loopback999 diff --git a/tests/recordings/commands/nxos-pager.txt b/tests/recordings/commands/nxos-pager.txt index 39f79e51..994e8387 100644 --- a/tests/recordings/commands/nxos-pager.txt +++ b/tests/recordings/commands/nxos-pager.txt @@ -1,7 +1,7 @@ -# Cisco NX-OS (R2): output with the pager on. +# Cisco NX-OS: output with the pager on. # The recorder asks for 1000 lines, so set the pager length explicitly. terminal length 24 -show running-config +show processes memory !send \x20 !send \x20 !send q diff --git a/tests/recordings/linux/busybox-1.37-ash-telnet-badpass.golden.json b/tests/recordings/linux/busybox-1.37-ash-telnet-badpass.golden.json new file mode 100644 index 00000000..8ad87ab9 --- /dev/null +++ b/tests/recordings/linux/busybox-1.37-ash-telnet-badpass.golden.json @@ -0,0 +1,6 @@ +{ + "login": "LoginFailure", + "driver": "shell", + "os": "shell", + "commands": [] +} diff --git a/tests/recordings/linux/busybox-1.37-ash-telnet-badpass.jsonl b/tests/recordings/linux/busybox-1.37-ash-telnet-badpass.jsonl new file mode 100644 index 00000000..21894f4a --- /dev/null +++ b/tests/recordings/linux/busybox-1.37-ash-telnet-badpass.jsonl @@ -0,0 +1,19 @@ +{"format": "exscript-recording", "version": 1, "protocol": "telnet", "date": "2026-11-11", "sanitized": true, "device": {"os": "shell", "version": "BusyBox 1.37", "platform": "docker busybox:1.37, telnetd", "login_fails": true}, "terminal": {"type": "dumb", "width": 510, "height": 1000}} +{"t": 0.003, "dir": "recv", "data": "\u00ff\u00fd\u0001\u00ff\u00fd\u001f\u00ff\u00fb\u0001\u00ff\u00fb\u0003"} +{"t": 0.003, "dir": "send", "data": "\u00ff\u00fc\u0001", "phase": "negotiation"} +{"t": 0.003, "dir": "send", "data": "\u00ff\u00fb\u001f", "phase": "negotiation"} +{"t": 0.003, "dir": "send", "data": "\u00ff\u00fa\u001f\u0001\u00fe\u0003\u00e8\u00ff\u00f0", "phase": "negotiation"} +{"t": 0.003, "dir": "send", "data": "\u00ff\u00fd\u0001", "phase": "negotiation"} +{"t": 0.003, "dir": "send", "data": "\u00ff\u00fd\u0003", "phase": "negotiation"} +{"t": 0.004, "dir": "recv", "data": "\r\r\nrouter3 login: "} +{"t": 10.015, "dir": "send", "data": "admin\r", "phase": "login"} +{"t": 10.016, "dir": "recv", "data": "admin\r\nPassword: "} +{"t": 20.069, "dir": "send", "data": "", "secret": true, "phase": "login"} +{"t": 20.071, "dir": "recv", "data": "\r\nLogin incorrect\r\nrouter3 login: "} +{"t": 33.082, "dir": "send", "data": "admin\r", "phase": "login"} +{"t": 33.084, "dir": "recv", "data": "admin\r\nPassword: "} +{"t": 43.095, "dir": "send", "data": "", "secret": true, "phase": "login"} +{"t": 43.096, "dir": "recv", "data": "\r\nLogin incorrect\r\nrouter3 login: "} +{"t": 56.107, "dir": "send", "data": "admin\r", "phase": "login"} +{"t": 56.109, "dir": "recv", "data": "admin\r\nPassword: \r\r\nLogin timed out after 60 seconds\r\r\n"} +{"t": 60.017, "dir": "close", "data": ""} diff --git a/tools/record_session.py b/tools/record_session.py index 73982547..12b43c80 100755 --- a/tools/record_session.py +++ b/tools/record_session.py @@ -315,6 +315,9 @@ def main(): parser.add_argument('--os', help='ground truth: Exscript OS/driver name, e.g. ios') parser.add_argument('--os-version', help='ground truth: OS version') parser.add_argument('--platform', help='ground truth: hardware or image') + parser.add_argument( + '--login-fails', action='store_true', help='ground truth: the login fails (wrong password)' + ) parser.add_argument('--prompt', type=re.compile, help='regex that ends a read early') parser.add_argument( '--idle', type=float, default=2.0, help='seconds without data that end a read' @@ -345,6 +348,9 @@ def main(): password = None if args.no_password or args.key else getpass.getpass('Password: ') secret = getpass.getpass('Secret for !secret: ') if '!secret' in commands else None + device = {'os': args.os, 'version': args.os_version, 'platform': args.platform} + if args.login_fails: + device['login_fails'] = True recording = Recording( { 'format': 'exscript-recording', @@ -352,7 +358,7 @@ def main(): 'protocol': args.protocol, 'date': datetime.date.today().isoformat(), 'sanitized': False, - 'device': {'os': args.os, 'version': args.os_version, 'platform': args.platform}, + 'device': device, } ) try: diff --git a/tools/sanitize_recording.py b/tools/sanitize_recording.py index 16decf14..deb59b3a 100755 --- a/tools/sanitize_recording.py +++ b/tools/sanitize_recording.py @@ -16,8 +16,9 @@ Consecutive output events are merged first, so a value split between two reads is still found. NETCONF 1.1 chunks are re-framed with their original sizes. -The sanitizer prints every replacement and the prompts it saw, but it cannot -know every identifier: read the result before committing it. Example:: +The sanitizer prints every replacement (secrets without their values) and the +prompts it saw, but it cannot know every identifier: read the result before +committing it. Example:: uv run tools/sanitize_recording.py raw.jsonl tests/recordings/ios/ios-15.9-ssh.jsonl \\ --replace core-rtr-01=router1 --replace jdoe=admin @@ -151,7 +152,8 @@ def _remove(self, kind, value): quoted = len(value) > 1 and value[0] == value[-1] == '"' if value.strip('"') == REMOVED or value in self.keep: return value - self.report[kind, value[:2] + '...', REMOVED] += 1 + shown = value[:2] + '...' if kind == 'serial' else '...' # no part of a secret + self.report[kind, shown, REMOVED] += 1 return f'"{REMOVED}"' if quoted else REMOVED