scandeval.exceptions
docs
module
scandeval.exceptions
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183 | """Exceptions to used by other functions."""
class InvalidBenchmark(Exception):
"""The (model, dataset) combination cannot be benchmarked."""
def __init__(
self, message: str = "This model cannot be benchmarked on the given dataset."
):
"""Initialize the exception.
Args:
message:
The message to display.
"""
self.message = message
super().__init__(self.message)
class InvalidModel(Exception):
"""The model cannot be benchmarked on any datasets."""
def __init__(
self, message: str = "The model cannot be benchmarked on any datasets."
):
"""Initialize the exception.
Args:
message:
The message to display.
"""
self.message = message
super().__init__(self.message)
class HuggingFaceHubDown(Exception):
"""The Hugging Face Hub seems to be down."""
def __init__(self, message: str = "The Hugging Face Hub is currently down."):
"""Initialize the exception.
Args:
message:
The message to display.
"""
self.message = message
super().__init__(self.message)
class NoInternetConnection(Exception):
"""There seems to be no internet connection."""
def __init__(self, message: str = "There is currently no internet connection."):
"""Initialize the exception.
Args:
message:
The message to display.
"""
self.message = message
super().__init__(self.message)
class NaNValueInModelOutput(Exception):
"""There is a NaN value in the model output."""
def __init__(self, message: str = "There is a NaN value in the model output."):
"""Initialize the exception.
Args:
message:
The message to display.
"""
self.message = message
super().__init__(self.message)
class FlashAttentionNotInstalled(Exception):
"""The `flash-attn` package has not been installed."""
def __init__(
self,
message: str = (
"The model you are trying to load requires Flash Attention. To use Flash "
"Attention, please install the `flash-attn` package, which can be done by "
"running `pip install -U wheel && FLASH_ATTENTION_SKIP_CUDA_BUILD=TRUE "
"pip install flash-attn --no-build-isolation`."
),
):
"""Initialize the exception.
Args:
message:
The message to display.
"""
self.message = message
super().__init__(self.message)
class NeedsExtraInstalled(InvalidModel):
"""The evaluation requires extra to be installed."""
def __init__(self, extra: str):
"""Initialize the exception.
Args:
extra:
The extra that needs to be installed.
"""
self.extra = extra
self.message = (
f"The model you are trying to load requires the `{extra}` extra to be "
f"installed. To install the `{extra}` extra, please run `pip install "
f"scandeval[{extra}]` or `pip install scandeval[all]`."
)
super().__init__(self.message)
class NeedsManualDependency(InvalidModel):
"""The evaluation requires a dependency to be manually installed."""
def __init__(self, package: str):
"""Initialize the exception.
Args:
package:
The package that needs to be manually installed.
"""
self.package = package
self.message = (
f"The model you are trying to load requires the `{package}` package to be "
f"installed - please run `pip install {package}` and try again."
)
super().__init__(self.message)
class NeedsAdditionalArgument(InvalidModel):
"""The evaluation requires additional arguments to the `scandeval` command."""
def __init__(self, cli_argument: str, script_argument: str, run_with_cli: bool):
"""Initialize the exception.
Args:
cli_argument:
The argument that needs to be passed to the `scandeval` command.
script_argument:
The argument that needs to be passed to the `Benchmarker` class.
run_with_cli:
Whether the benchmark is being run with the CLI.
"""
self.cli_argument = cli_argument
self.script_argument = script_argument
if run_with_cli:
self.message = (
f"The model you are trying to load requires the `{cli_argument}` "
"argument to be passed to the `scandeval` command. Please pass the "
"argument and try again."
)
else:
self.message = (
f"The model you are trying to load requires the `{script_argument}` "
"argument to be passed to the `Benchmarker` class. Please pass the "
"argument and try again."
)
super().__init__(self.message)
class NeedsEnvironmentVariable(InvalidModel):
"""The evaluation requires an environment variable to be set."""
def __init__(self, env_var: str):
"""Initialize the exception.
Args:
env_var:
The environment variable that needs to be set.
"""
self.env_var = env_var
self.message = (
f"The model you are trying to load requires the `{env_var}` environment "
"variable to be set. Please set the environment variable and try again."
)
super().__init__(self.message)
|