Source code for servicex.models
# Copyright (c) 2022, IRIS-HEP
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# * Redistributions of source code must retain the above copyright notice, this
# list of conditions and the following disclaimer.
#
# * Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation
# and/or other materials provided with the distribution.
#
# * Neither the name of the copyright holder nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
import hashlib
from datetime import datetime
from enum import Enum
from pydantic import BaseModel, Field, field_validator
from typing import List, Optional
[docs]
class ResultDestination(str, Enum):
r"""
Direct the output to object store or posix volume
"""
object_store = "object-store"
volume = "volume"
[docs]
class ResultFormat(str, Enum):
r"""
Specify the file format for the generated output
"""
parquet = "parquet"
root_ttree = "root-file"
[docs]
class Status(str, Enum):
r"""
Status of a submitted transform
"""
complete = ("Complete",)
fatal = ("Fatal",)
canceled = ("Canceled",)
submitted = ("Submitted",)
looking = ("Lookup",)
pending = "Pending Lookup"
running = "Running"
[docs]
class TransformRequest(BaseModel):
r"""
Transform request sent to ServiceX
"""
title: Optional[str] = None
did: Optional[str] = None
file_list: Optional[List[str]] = Field(default=None, alias="file-list")
selection: str
image: Optional[str] = None
codegen: str
tree_name: Optional[str] = Field(default=None, alias="tree-name")
result_destination: ResultDestination = Field(
serialization_alias="result-destination"
)
result_format: ResultFormat = Field(serialization_alias="result-format")
model_config = {"populate_by_name": True}
[docs]
def compute_hash(self):
r"""
Compute a hash for this submission. Only include properties that impact the result
so we have maximal ability to reuse transforms
:return: SHA256 hash of request
"""
sha = hashlib.sha256(
str(
[
self.did,
self.selection,
self.tree_name,
self.codegen,
self.image,
self.result_format.name,
self.file_list,
]
).encode("utf-8")
)
return sha.hexdigest()
[docs]
class TransformStatus(BaseModel):
r"""
Status object returned by servicex
"""
request_id: str
did: str
title: Optional[str] = None
selection: str
tree_name: Optional[str] = Field(validation_alias="tree-name")
image: str
result_destination: ResultDestination = Field(validation_alias="result-destination")
result_format: ResultFormat = Field(validation_alias="result-format")
generated_code_cm: str = Field(validation_alias="generated-code-cm")
status: Status
app_version: str = Field(validation_alias="app-version")
files: int
files_completed: int = Field(validation_alias="files-completed")
files_failed: int = Field(validation_alias="files-failed")
files_remaining: Optional[int] = Field(validation_alias="files-remaining", default=0)
submit_time: datetime = Field(validation_alias="submit-time", default=None)
finish_time: Optional[datetime] = Field(validation_alias="finish-time", default=None)
minio_endpoint: Optional[str] = Field(validation_alias="minio-endpoint", default=None)
minio_secured: Optional[bool] = Field(validation_alias="minio-secured", default=None)
minio_access_key: Optional[str] = Field(validation_alias="minio-access-key", default=None)
minio_secret_key: Optional[str] = Field(validation_alias="minio-secret-key", default=None)
log_url: Optional[str] = Field(validation_alias="log-url", default=None)
[docs]
@field_validator("finish_time", mode="before")
@classmethod
def parse_finish_time(cls, v):
if isinstance(v, str) and v == "None":
return None
return v
[docs]
class ResultFile(BaseModel):
r"""
Record reporting the properties of a transformed file result
"""
filename: str
size: int
extension: str
[docs]
class TransformedResults(BaseModel):
r"""
Returned for a submission. Gives you everything you need to know about a completed
transform.
"""
hash: str
title: str
codegen: str
request_id: str
submit_time: datetime
data_dir: str
file_list: List[str]
signed_url_list: List[str]
files: int
result_format: ResultFormat
log_url: Optional[str] = None