Skip to content

Commit

Permalink
Replace NamedTuple with dataclass (AmpX-AI#24)
Browse files Browse the repository at this point in the history
Co-authored-by: Jakub Perniš <[email protected]>
  • Loading branch information
jakubpernis and Jakub Perniš authored Sep 24, 2022
1 parent 7983689 commit e165592
Show file tree
Hide file tree
Showing 3 changed files with 9 additions and 7 deletions.
8 changes: 5 additions & 3 deletions fsql/deser.py
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@
from enum import Enum, auto, unique
from functools import partial, reduce
from itertools import chain
from typing import Any, Generic, Iterable, NamedTuple, Tuple, TypeVar, Union
from typing import Any, Generic, Iterable, Tuple, TypeVar, Union

import pandas as pd
from fsspec.core import OpenFile
Expand Down Expand Up @@ -98,12 +98,14 @@ def from_url(cls, url: str):
DataObject = TypeVar("DataObject")


class DataObjectRich(NamedTuple, Generic[DataObject]):
@dataclass(frozen=True)
class DataObjectRich(Generic[DataObject]):
data: DataObject
failures: Iterable[PartitionReadFailure]


class PartitionReadFailure(NamedTuple):
@dataclass(frozen=True)
class PartitionReadFailure:
partition: Partition
reason: str # or Any?

Expand Down
4 changes: 2 additions & 2 deletions tests/test_dict_reader.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,5 +39,5 @@ def test_lazy_errors(tmp_path):

lazy_reader = EnumeratedDictReader(lazy_errors=True)
result = read_partitioned_table(f"file://{case1_path}/", Q_TRUE, data_reader=lazy_reader)
assert result[0] == {0: json.loads(data1)}
assert [error_line] == [e.reason for e in result[1]]
assert result.data == {0: json.loads(data1)}
assert [error_line] == [e.reason for e in result.failures]
4 changes: 2 additions & 2 deletions tests/test_pandasreader.py
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,6 @@ def test_lazy_errors(tmp_path):
result = read_partitioned_table(f"file://{case1_path}/", Q_TRUE, data_reader=reader_eager)
reader_lazy = PandasReader(columns=["c3"], lazy_errors=True)
result = read_partitioned_table(f"file://{case1_path}/", Q_TRUE, data_reader=reader_lazy)
assert_frame_equal(df2[["c3"]], result[0])
reasons = [e.reason.split("\n")[0] for e in result[1]]
assert_frame_equal(df2[["c3"]], result.data)
reasons = [e.reason.split("\n")[0] for e in result.failures]
assert reasons == [error_line]

0 comments on commit e165592

Please sign in to comment.