openrvdas.logger.readers.http_reader

  1#!/usr/bin/env python3
  2"""
  3```
  4HTTPReader
  5==========
  6
  7This module defines the :class:`HTTPReader`, a polling reader for retrieving
  8data from HTTP or HTTPS endpoints at a fixed interval.
  9
 10The reader performs periodic HTTP GET or POST requests to a configured URL,
 11optionally including headers and a JSON payload. Responses are returned either
 12as decoded text or raw bytes, depending on the encoding configuration.
 13
 14Key features
 15------------
 16- Supports HTTP GET and POST methods
 17- Configurable request headers and JSON payloads
 18- Enforces a minimum polling interval between requests
 19- Thread-safe access using an internal lock
 20- Optional URL verification on first read
 21- Supports decoded text output or raw byte output
 22- Integrates with the OpenRVDAS Reader framework
 23
 24Typical usage
 25-------------
 26Example usage with a GET request:
 27
 28    reader = HTTPReader(
 29        url="https://example.com/data",
 30        interval=10
 31    )
 32
 33    record = reader.read()
 34
 35Example usage with a POST request and JSON payload:
 36
 37    reader = HTTPReader(
 38        url="https://example.com/api",
 39        method="post",
 40        headers={"Authorization": "Bearer TOKEN"},
 41        payload={"sensor": "temp"},
 42        interval=5
 43    )
 44
 45    record = reader.read()
 46
 47Output behavior
 48---------------
 49- If ``encoding`` is set (default: ``'utf-8'``), responses are decoded into
 50  strings using the specified encoding and error-handling strategy.
 51- If ``encoding`` is ``None`` or empty, raw response bytes are returned.
 52- On failure, ``read()`` returns ``None``.
 53
 54Error handling
 55--------------
 56- On the first read, the reader verifies that the configured URL is reachable.
 57  URL verification fails if:
 58  - The server returns HTTP 404 (resource not found), or
 59  - The server returns any HTTP status code >= 500, or
 60  - A network or request error occurs (e.g., timeout, DNS failure).
 61- HTTP 401, 403, and 405 responses are treated as valid during verification,
 62  since they indicate a reachable endpoint.
 63- If URL verification fails, ``read()`` returns ``None``.
 64- During normal operation, HTTP 4xx or 5xx responses and network errors are
 65  treated as read failures, reset URL verification, and cause ``read()``
 66  to return ``None``.
 67
 68Thread safety
 69-------------
 70All calls to ``read()`` are protected by an internal lock, making this reader
 71safe for use in multi-threaded environments.
 72
 73```
 74"""
 75
 76import threading
 77import time
 78import logging
 79import json
 80
 81from urllib.request import Request, urlopen
 82from urllib.error import HTTPError, URLError
 83
 84from logger.readers.reader import Reader  # noqa: E402
 85
 86
 87################################################################################
 88class HTTPReader(Reader):  # noqa: R0913
 89    """
 90    Read data from a URL at a set interval.
 91    """
 92
 93    _ALLOWED_METHODS = {"get", "post"}
 94
 95    ############################
 96    def __init__(self, url: str, method: str = 'get',  # noqa: R0902
 97                 headers: dict | None = None, payload: dict | None = None,
 98                 interval: float = 5.0, timeout: float = 2.0,
 99                 encoding: str | None = 'utf-8',
100                 encoding_errors: str = 'ignore', **kwargs):
101        """
102        ```
103        url - Network url to read, in protocol://host:port format (e.g.
104              'https://example.com' 'http://example.com:8000').
105
106        method - type of http request
107
108        headers - Headers to set for the request
109
110        payload - Payload to set for the request. Only applicable for
111                  POST requests
112
113        interval - Seconds between update requests, must be >=0
114
115        timeout - Max time to wait for device to respond AND read the response.
116
117        encoding - 'utf-8' by default. If empty or None, do not attempt any
118                decoding and return raw bytes. Other possible encodings are
119                listed in online documentation here:
120                https://docs.python.org/3/library/codecs.html#standard-encodings
121
122        encoding_errors - 'ignore' by default. Other error strategies are
123                'strict', 'replace', and 'backslashreplace', described here:
124                https://docs.python.org/3/howto/unicode.html#encodings
125        ```
126        """
127        super().__init__(encoding=encoding, encoding_errors=encoding_errors, **kwargs)
128
129        if interval < 0:
130            raise ValueError('Interval must be greater or equal to zero')
131
132        self.url = self._validate_url(url)
133        self.method = self._validate_method(method)
134        self.headers = headers or {}
135        self.payload = payload
136        self.interval = interval
137        self.timeout = timeout
138
139        self._read_lock = threading.Lock()
140        self._verified = False
141        self._next_read_time = 0.0
142
143    ############################
144    def _verify_url(self):
145        """Check URL reachability on first read."""
146        if self._verified:
147            return
148
149        def _check_status(code: int):
150            if code == 404 or code >= 500:
151                raise ValueError(
152                    f"invalid url: {self.url}; return code: {code}"
153                )
154
155        try:
156            req = Request(self.url, method="HEAD", headers=self.headers)
157            with urlopen(req, timeout=self.timeout) as resp:
158                _check_status(resp.status)
159                self._verified = True
160                return
161
162        except HTTPError as exc:
163            # Some servers block or mishandle HEAD
164            _check_status(exc.code)
165            self._verified = True
166            return
167
168        except URLError:
169            pass
170
171        # Fallback to GET (do not explicitly read body)
172        try:
173            req = Request(self.url, method="GET", headers=self.headers)
174            with urlopen(req, timeout=self.timeout) as resp:
175                _check_status(resp.status)
176                self._verified = True
177                return
178
179        except (HTTPError, URLError) as exc:
180            raise ValueError(f"invalid url: {self.url}, {exc}")
181
182    ############################
183    def _validate_url(self, url: str) -> str:
184        if not (url.startswith("http://") or url.startswith("https://")):
185            raise ValueError(f"Invalid URL scheme: {url}")
186        if len(url.split("://")[-1]) == 0:
187            raise ValueError(f"Invalid URL: {url}")
188        return url
189
190    ############################
191    def _validate_method(self, method: str) -> str:
192        method = method.lower()
193        if method not in self._ALLOWED_METHODS:
194            raise ValueError(f"Unsupported HTTP method: {method}")
195        return method
196
197    ############################
198    def _make_request(self):
199        """Perform a single HTTP request."""
200        headers = dict(self.headers)
201        data = None
202
203        if self.method == "post" and self.payload is not None:
204            data = json.dumps(self.payload).encode("utf-8")
205            headers.setdefault("Content-Type", "application/json")
206
207        req = Request(
208            self.url,
209            method=self.method.upper(),
210            headers=headers,
211            data=data,
212        )
213
214        return urlopen(req, timeout=self.timeout)
215
216    ############################
217    def _parse_response(self, response) -> str | bytes:
218        data = response.read()
219        if self.encoding:
220            return data.decode(self.encoding, errors=self.encoding_errors)
221        return data
222
223    ############################
224    def read(self) -> str | bytes | None:
225        """
226        Read from the HTTP endpoint. Returns immediately after the HTTP request
227        completes, while enforcing a minimum interval between requests.
228        """
229
230        with self._read_lock:
231            now = time.monotonic()
232
233            # Enforce polling interval BEFORE the request
234            if now < self._next_read_time:
235                time.sleep(self._next_read_time - now)
236
237            start = time.monotonic()
238
239            try:
240                self._verify_url()
241                with self._make_request() as resp:
242                    record = self._parse_response(resp)
243
244            except (HTTPError, URLError, ValueError) as exc:
245                self._verified = False
246                logging.warning(
247                    "HTTP read failed: %s %s: %s",
248                    self.method,
249                    self.url,
250                    exc,
251                )
252                record = None
253
254            finally:
255                self._next_read_time = start + self.interval
256
257            return record