diff --git a/README.md b/README.md index 3532f11a..b5728aac 100644 --- a/README.md +++ b/README.md @@ -23,15 +23,15 @@ pip install packetraven virtualenv packetraven_env ``` 2. activate your new virtual environment - - On Linux: + - On Linux: ```bash source packetraven_env/bin/activate ``` - - On Windows native command prompt (`cmd`): + - On Windows native command prompt (`cmd`): ```cmd .\packetraven_env\Scripts\activate.bat ``` - - On Windows PowerShell: + - On Windows PowerShell: ```cmd .\packetraven_env\Scripts\activate.ps1 ``` @@ -39,7 +39,7 @@ pip install packetraven ```bash pip install packetraven ``` - + # Usage ## Command-line Options diff --git a/packetraven/__main__.py b/packetraven/__main__.py index 86492b2b..6f1c1881 100644 --- a/packetraven/__main__.py +++ b/packetraven/__main__.py @@ -21,6 +21,7 @@ TimeIntervalError, ) from packetraven.connections.base import PacketSource +from packetraven.connections.internet import RockBLOCK from packetraven.packets import APRSPacket from packetraven.packets.tracks import APRSTrack, LocationPacketTrack from packetraven.packets.writer import write_packet_tracks @@ -44,6 +45,10 @@ def main(): help='comma-separated list of serial ports / text files of a TNC parsing APRS packets from analog audio to ASCII' ' (set to `auto` to use the first open serial port)', ) + args_parser.add_argument( + '--rockblock', help='listen to incoming POST requests (defaults to `localhost:80`)', + ) + args_parser.add_argument('--rockblock-imei', help='IMEI of RockBLOCK modem') args_parser.add_argument( '--database', help='PostGres database table `user@hostname:port/database/table`' ) @@ -110,6 +115,28 @@ def main(): if args.tnc is not None: kwargs['tnc'] = [tnc.strip() for tnc in args.tnc.split(',')] + if args.rockblock is not None: + try: + if Path(args.rockblock).exists(): + kwargs['rockblock_csv'] = args.rockblock + else: + raise FileNotFoundError + except FileNotFoundError: + listen_hostname = parse_hostname(args.rockblock) + + hostname = listen_hostname['hostname'] + port = listen_hostname['port'] + username = listen_hostname['username'] + password = listen_hostname['password'] + + if port is not None: + hostname = f'{hostname}:{port}' + + kwargs['rockblock_hostname'] = hostname + kwargs['rockblock_imei'] = args.rockblock_imei + kwargs['rockblock_username'] = username + kwargs['rockblock_password'] = password + if args.database is not None: database = parse_hostname(args.database) @@ -247,7 +274,9 @@ def main(): elif start_date is None and end_date is not None: filter_message += f' sent before {end_date:%Y-%m-%d %H:%M:%S}' elif start_date is not None and end_date is not None: - filter_message += f' sent between {start_date:%Y-%m-%d %H:%M:%S} and {end_date:%Y-%m-%d %H:%M:%S}' + filter_message += ( + f' sent between {start_date:%Y-%m-%d %H:%M:%S} and {end_date:%Y-%m-%d %H:%M:%S}' + ) if callsigns is not None: filter_message += f' from {len(callsigns)} callsigns: {callsigns}' LOGGER.info(filter_message) @@ -295,6 +324,30 @@ def main(): except ConnectionError as error: LOGGER.warning(f'{error.__class__.__name__} - {error}') + if 'rockblock_hostname' in kwargs: + rockblock_kwargs = { + key: kwargs[key] + for key in [ + 'rockblock_hostname', + 'rockblock_imei', + 'rockblock_username', + 'rockblock_password', + ] + if key in kwargs + } + + try: + rockblock = RockBLOCK( + imei=None, + username=rockblock_kwargs['rockblock_username'], + password=rockblock_kwargs['rockblock_password'], + ) + rockblock.start_listening(rockblock_kwargs['rockblock_hostname']) + LOGGER.info(f'connected to {rockblock.location}') + connections.append(rockblock) + except ConnectionError: + pass + if 'database_hostname' in kwargs: database_kwargs = { key: kwargs[key] diff --git a/packetraven/connections/file.py b/packetraven/connections/file.py index d2823120..f10acb4d 100644 --- a/packetraven/connections/file.py +++ b/packetraven/connections/file.py @@ -1,10 +1,13 @@ from datetime import datetime +from io import BytesIO, StringIO from os import PathLike from pathlib import Path +from typing import Union from urllib.parse import urlparse from dateutil.parser import parse as parse_date import geojson +import pandas import requests from packetraven.connections.base import ( @@ -14,9 +17,49 @@ TimeIntervalError, ) from packetraven.packets import APRSPacket, LocationPacket +from packetraven.packets.base import IridiumPacket -class RawAPRSTextFile(APRSPacketSource): +class WatchedFile: + def __init__(self, file: Union[PathLike, StringIO, BytesIO]): + if not isinstance(file, (StringIO, BytesIO)): + if not urlparse(str(file)).scheme in ['http', 'https', 'ftp', 'sftp']: + if not isinstance(file, Path): + if isinstance(file, str): + file = file.strip('"') + file = Path(file) + file = str(file) + + self.file = file + self.__parsed_lines = None + + def new_lines(self) -> [str]: + new_lines = [] + + if Path(self.file).exists(): + file_connection = open(Path(self.file).expanduser().resolve()) + lines = file_connection.readlines() + else: + file_connection = requests.get(self.file, stream=True) + lines = file_connection.iter_lines() + + for line in lines: + if len(line) > 0: + if isinstance(line, bytes): + line = line.decode() + if line not in self.__parsed_lines: + self.__parsed_lines.append(line) + new_lines.append(line) + + file_connection.close() + + return new_lines + + def close(self): + pass + + +class RawAPRSTextFile(APRSPacketSource, WatchedFile): def __init__(self, filename: PathLike = None, callsigns: str = None): """ read APRS packets from a given text file where each line consists of the time sent (`YYYY-MM-DDTHH:MM:SS`) followed by @@ -26,14 +69,9 @@ def __init__(self, filename: PathLike = None, callsigns: str = None): :param callsigns: list of callsigns to return from source """ - if not urlparse(str(filename)).scheme in ['http', 'https', 'ftp', 'sftp']: - if not isinstance(filename, Path): - if isinstance(filename, str): - filename = filename.strip('"') - filename = Path(filename) - filename = str(filename) + WatchedFile.__init__(self, filename) + APRSPacketSource.__init__(self, self.file, callsigns) - super().__init__(filename, callsigns) self.__last_access_time = None self.__parsed_lines = [] @@ -46,35 +84,23 @@ def packets(self) -> [APRSPacket]: f'interval {interval} less than minimum interval {self.interval}' ) - if Path(self.location).exists(): - file_connection = open(Path(self.location).expanduser().resolve()) - lines = file_connection.readlines() - else: - file_connection = requests.get(self.location, stream=True) - lines = file_connection.iter_lines() + new_lines = self.new_lines() packets = [] - for line in lines: - if len(line) > 0: - if isinstance(line, bytes): - line = line.decode() - if line not in self.__parsed_lines: - self.__parsed_lines.append(line) - try: - packet_time, raw_aprs = line.split(': ', 1) - packet_time = parse_date(packet_time) - except: - raw_aprs = line - packet_time = datetime.now() - raw_aprs = raw_aprs.strip() - try: - packets.append( - APRSPacket.from_frame(raw_aprs, packet_time, source=self.location) - ) - except Exception as error: - LOGGER.error(f'{error.__class__.__name__} - {error}') - - file_connection.close() + for line in new_lines: + try: + packet_time, raw_aprs = line.split(': ', 1) + packet_time = parse_date(packet_time) + except: + raw_aprs = line + packet_time = datetime.now() + raw_aprs = raw_aprs.strip() + try: + packets.append( + APRSPacket.from_frame(raw_aprs, packet_time, source=self.location) + ) + except Exception as error: + LOGGER.error(f'{error.__class__.__name__} - {error}') if self.callsigns is not None: packets = [packet for packet in packets if packet.from_callsign in self.callsigns] @@ -89,7 +115,52 @@ def __repr__(self): return f'{self.__class__.__name__}({repr(self.location)}, {repr(self.callsigns)})' -class PacketGeoJSON(PacketSource): +class RockBLOCKtoolsCSV(PacketSource, WatchedFile): + def __init__(self, csv_filename: PathLike = None): + """ + watch a CSV file being written to by a listening instance of `rockblock-tools` + ``` + rockblock listen csv localhost 80 + ``` + + :param csv_filename: path to CSV file + """ + + if not isinstance(csv_filename, Path): + csv_filename = Path(csv_filename) + + super().__init__(csv_filename) + self.__last_access_time = None + + @property + def packets(self) -> [LocationPacket]: + new_lines = self.new_lines() + + packets = [] + for line in new_lines: + line = [entry.strip() for entry in line.split(',')] + packet = IridiumPacket( + time=line[3], + x=float(line[1]), + y=float(line[0]), + device_type=line[2], + momsn=line[4], + imei=line[5], + serial=line[6], + data=line[7], + cep=line[8], + ) + packets.append(packet) + + self.__last_access_time = datetime.now() + + return packets + + def close(self): + pass + + +class PacketGeoJSON(PacketSource, WatchedFile): def __init__(self, filename: PathLike = None): """ read location packets from a given GeoJSON file @@ -97,14 +168,9 @@ def __init__(self, filename: PathLike = None): :param filename: path to GeoJSON file """ - if not urlparse(str(filename)).scheme in ['http', 'https', 'ftp', 'sftp']: - if not isinstance(filename, Path): - if isinstance(filename, str): - filename = filename.strip('"') - filename = Path(filename) - filename = str(filename) + WatchedFile.__init__(self, filename) + PacketSource.__init__(self, self.file) - super().__init__(filename) self.__last_access_time = None @property @@ -116,42 +182,39 @@ def packets(self) -> [LocationPacket]: f'interval {interval} less than minimum interval {self.interval}' ) - if Path(self.location).exists(): - with open(Path(self.location).expanduser().resolve()) as file_connection: - features = geojson.load(file_connection) - else: - response = requests.get(self.location, stream=True) - features = geojson.loads(response.text) - packets = [] - for feature in features['features']: - if feature['geometry']['type'] == 'Point': - properties = feature['properties'] - time = parse_date(properties['time']) - del properties['time'] - - if 'from' in properties: - from_callsign = properties['from'] - to_callsign = properties['to'] - del properties['from'], properties['to'] - - packet = APRSPacket( - from_callsign, - to_callsign, - time, - *feature['geometry']['coordinates'], - source=self.location, - **properties, - ) - else: - packet = LocationPacket( - time, - *feature['geometry']['coordinates'], - source=self.location, - **properties, - ) - - packets.append(packet) + new_lines = self.new_lines() + if len(new_lines) > 0: + features = geojson.loads(new_lines) + + for feature in features['features']: + if feature['geometry']['type'] == 'Point': + properties = feature['properties'] + time = parse_date(properties['time']) + del properties['time'] + + if 'from' in properties: + from_callsign = properties['from'] + to_callsign = properties['to'] + del properties['from'], properties['to'] + + packet = APRSPacket( + from_callsign, + to_callsign, + time, + *feature['geometry']['coordinates'], + source=self.location, + **properties, + ) + else: + packet = LocationPacket( + time, + *feature['geometry']['coordinates'], + source=self.location, + **properties, + ) + + packets.append(packet) self.__last_access_time = datetime.now() diff --git a/packetraven/connections/internet.py b/packetraven/connections/internet.py index 11068ec3..0f79a09c 100644 --- a/packetraven/connections/internet.py +++ b/packetraven/connections/internet.py @@ -1,12 +1,19 @@ +from argparse import Namespace from datetime import datetime, timedelta +from io import StringIO +from tempfile import NamedTemporaryFile, TemporaryFile from time import sleep from typing import Any, Sequence import aprslib +from flask import request import requests +import rockblock_tools +from rockblock_tools import listen +from rockblock_tools.formatter import CSVFormatter from shapely.geometry import Point from tablecrow import PostGresTable -from tablecrow.utilities import split_hostname_port +from tablecrow.utilities import parse_hostname, split_hostname_port from packetraven.connections.base import ( APRSPacketSink, @@ -18,6 +25,7 @@ PacketSource, TimeIntervalError, ) +from packetraven.connections.file import RockBLOCKtoolsCSV from packetraven.packets import APRSPacket, LocationPacket from packetraven.packets.parsing import InvalidPacketError from packetraven.utilities import read_configuration @@ -110,6 +118,93 @@ def __repr__(self): return f'{self.__class__.__name__}({repr(self.callsigns)}, {repr("****")})' +class RockBLOCK(PacketSource, NetworkConnection): + def __init__(self, imei: str = None, username: str = None, password: str = None): + """ + connect to RockBLOCK API + + :param imei: IMEI of RockBLOCK + :param username: RockBLOCK username + :param password: RockBLOCK password + """ + + url = 'https://rockblock.rock7.com' + NetworkConnection.__init__(self, url) + + configuration = read_configuration(CREDENTIALS_FILENAME) + if imei is None or imei == '': + if 'RockBLOCK' in configuration: + imei = configuration['RockBLOCK']['imei'] + if username is None or username == '': + if 'RockBLOCK' in configuration: + username = configuration['RockBLOCK']['username'] + if password is None or password == '': + if 'RockBLOCK' in configuration: + password = configuration['RockBLOCK']['password'] + + if not self.connected: + raise ConnectionError(f'no network connection') + + self.imei = imei + self.username = username + self.password = password + + self.__last_access_time = None + self.__parsed_lines = [] + + self.__csv_stream = NamedTemporaryFile() + formatter_options = Namespace() + setattr(formatter_options, 'data_format', 'raw') + setattr(formatter_options, 'csv_file', self.__csv_stream.name) + self.__csv_formatter = CSVFormatter(formatter_options) + self.__csv_parser = RockBLOCKtoolsCSV(self.__csv_stream.name) + + def start_listening(self, hostname: str = None, port: int = None): + """ + :param hostname: hostname on which to listen to POST requests from RockBLOCK + :param port: port on which to listen (must be available) + """ + + if hostname is None: + hostname = 'localhost' + else: + connection_info = parse_hostname(hostname) + hostname = connection_info['hostname'] + if port is None: + port = connection_info['port'] + if port is None: + port = 80 + + listen(hostname, port, self.__csv_formatter) + + @property + def packets(self) -> [LocationPacket]: + return self.__csv_parser.packets + + def close(self): + shutdown_function = request.environ.get('werkzeug.server.shutdown') + if shutdown_function is None: + raise RuntimeError('Not running with the Werkzeug Server') + shutdown_function() + + self.__csv_formatter.close() + self.__csv_stream.close() + + def send_message(self, message: str): + """ + :param message: message data + """ + + if self.imei is None: + raise ValueError('RockBLOCK IMEI not provided') + if self.username is None: + raise ValueError('RockBLOCK username not provided') + if self.password is None: + raise ValueError('RockBLOCK password not provided') + + rockblock_tools.send(self.imei, self.username, self.password, message) + + class PacketDatabaseTable(PostGresTable, PacketSource, PacketSink): __default_fields = { 'time': datetime, diff --git a/packetraven/gui/base.py b/packetraven/gui/base.py index 7e6e97d5..96ecf853 100644 --- a/packetraven/gui/base.py +++ b/packetraven/gui/base.py @@ -15,8 +15,8 @@ from packetraven import APRSDatabaseTable, APRSfi, RawAPRSTextFile, SerialTNC from packetraven.__main__ import DEFAULT_INTERVAL_SECONDS, LOGGER, retrieve_packets from packetraven.connections.base import available_serial_ports, next_open_serial_port -from packetraven.connections.file import PacketGeoJSON -from packetraven.connections.internet import APRSis +from packetraven.connections.file import PacketGeoJSON, RockBLOCKtoolsCSV +from packetraven.connections.internet import APRSis, RockBLOCK from packetraven.gui.plotting import LivePlot from packetraven.packets import APRSPacket, LocationPacket from packetraven.packets.tracks import LocationPacketTrack, PredictedTrajectory @@ -48,6 +48,13 @@ def __init__( self.__configuration = { 'aprs_fi': {'aprs_fi_key': None}, 'tnc': {'tnc': None}, + 'rockblock': { + 'rockblock_csv': None, + 'rockblock_hostname': None, + 'rockblock_imei': None, + 'rockblock_username': None, + 'rockblock_password': None, + }, 'database': { 'database_hostname': None, 'database_database': None, @@ -567,6 +574,37 @@ def toggle(self): except Exception as error: connection_errors.append(f'aprs.fi - {error}') + if ( + 'rockblock' in self.__configuration + and self.__configuration['rockblock']['rockblock_csv'] is not None + ): + try: + rockblock = RockBLOCKtoolsCSV( + self.__configuration['rockblock']['rockblock_csv'] + ) + LOGGER.info(f'connected to {rockblock.location}') + self.__connections.append(rockblock) + except ConnectionError as error: + connection_errors.append(f'rockblock - {error}') + + if ( + 'rockblock' in self.__configuration + and self.__configuration['rockblock']['rockblock_hostname'] is not None + ): + try: + rockblock = RockBLOCK( + imei=None, + username=self.__configuration['rockblock']['rockblock_username'], + password=self.__configuration['rockblock']['rockblock_password'], + ) + rockblock.start_listening( + self.__configuration['rockblock']['rockblock_hostname'] + ) + LOGGER.info(f'connected to {rockblock.location}') + self.__connections.append(rockblock) + except ConnectionError as error: + connection_errors.append(f'rockblock - {error}') + if ( 'database' in self.__configuration and self.__configuration['database']['database_hostname'] is not None diff --git a/packetraven/packets/base.py b/packetraven/packets/base.py index 54616910..4f3b0a84 100644 --- a/packetraven/packets/base.py +++ b/packetraven/packets/base.py @@ -316,3 +316,25 @@ def __repr__(self) -> str: f'{self.__class__.__name__}(from_callsign={repr(self.from_callsign)}, to_callsign={repr(self.to_callsign)}, time={repr(self.time)}, ' f'{coordinate_string}, crs={self.crs.__class__.__name__}.from_epsg({repr(self.crs.to_epsg())}), {attribute_string})' ) + + +class IridiumPacket(LocationPacket): + def __init__( + self, + time: datetime, + x: float, + y: float, + device_type: str, + momsn, + imei: str, + serial: str, + data: str, + cep: str, + ): + super().__init__(time, x, y) + self.device_type = device_type + self.momsn = momsn + self.imei = imei + self.serial = serial + self.data = data + self.cep = cep diff --git a/packetraven/predicts.py b/packetraven/predicts.py index 637fed8d..0f52d0d1 100644 --- a/packetraven/predicts.py +++ b/packetraven/predicts.py @@ -479,8 +479,8 @@ def get_predictions( prediction_float_end_time = None else: prediction_float_end_time = None - descent_only = ( - packet_track.falling or numpy.any(packet_track.ascent_rates[-2:] < 0) + descent_only = packet_track.falling or numpy.any( + packet_track.ascent_rates[-2:] < 0 ) try: diff --git a/setup.py b/setup.py index 17b746f5..0496e977 100644 --- a/setup.py +++ b/setup.py @@ -21,6 +21,7 @@ 'psycopg2-binary': [], 'pyproj': [], 'requests': [], + 'rockblock-tools': [], 'shapely': [], 'sshtunnel': [], 'tablecrow>=1.3.9': [],