Skip to content

Refactor data raw #920

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
darinkrauss merged 1 commit into from
Apr 24, 2026
+688 −183
Merged

Refactor data raw #920

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
105 changes: 69 additions & 36 deletions data/raw/raw.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,28 +15,31 @@ import (
)

const (
FilterCreatedDateFormat = time.DateOnly
FilterDataSetIDsLengthMaximum = 100
DataSizeMaximum = 8 * 1024 * 1024 // Until create directly to S3 is supported
MetadataLengthMaximum = 4 * 1024
MediaTypeDefault = "application/octet-stream"
FilterCreatedDateFormat = time.DateOnly
DataSizeMaximum = 8 * 1024 * 1024 // Until create directly to S3 is supported
MetadataSizeMaximum = 4 * 1024
MediaTypeDefault = "application/octet-stream"
)

type Filter struct {
CreatedDate *string `json:"createdDate,omitempty"`
DataSetIDs *[]string `json:"dataSetIds,omitempty"`
Processed *bool `json:"processed,omitempty"`
CreatedDate *string `json:"createdDate,omitempty"`
DataSetID *string `json:"dataSetId,omitempty"`
Processed *bool `json:"processed,omitempty"`
Archivable *bool `json:"archivable,omitempty"`
Archived *bool `json:"archived,omitempty"`
}

func (f *Filter) Parse(parser structure.ObjectParser) {
f.CreatedDate = parser.String("createdDate")
f.DataSetIDs = parser.StringArray("dataSetIds")
f.DataSetID = parser.String("dataSetId")
f.Processed = parser.Bool("processed")
f.Archivable = parser.Bool("archivable")
f.Archived = parser.Bool("archived")
}

func (f *Filter) Validate(validator structure.Validator) {
validator.String("createdDate", f.CreatedDate).AsTime(FilterCreatedDateFormat).NotZero()
validator.StringArray("dataSetIds", f.DataSetIDs).NotEmpty().LengthLessThanOrEqualTo(FilterDataSetIDsLengthMaximum).EachUsing(data.SetIDValidator).EachUnique()
validator.String("dataSetId", f.DataSetID).Using(data.SetIDValidator)
}

func (f *Filter) CreatedTime() *time.Time {
Expand All @@ -51,9 +54,10 @@ func (f *Filter) CreatedTime() *time.Time {
}

type Create struct {
Metadata map[string]any `json:"metadata,omitempty"`
DigestMD5 *string `json:"digestMD5,omitempty"`
MediaType *string `json:"mediaType,omitempty"`
Metadata map[string]any `json:"metadata,omitempty"`
DigestMD5 *string `json:"digestMD5,omitempty"`
MediaType *string `json:"mediaType,omitempty"`
ArchivableTime *time.Time `json:"archivableTime,omitempty"`
}

func (c *Create) Parse(parser structure.ObjectParser) {
Expand All @@ -62,12 +66,14 @@ func (c *Create) Parse(parser structure.ObjectParser) {
}
c.DigestMD5 = parser.String("digestMD5")
c.MediaType = parser.String("mediaType")
c.ArchivableTime = parser.Time("archivableTime", time.RFC3339Nano)
}

func (c *Create) Validate(validator structure.Validator) {
validator.Object("metadata", &c.Metadata).SizeLessThanOrEqualTo(MetadataLengthMaximum)
validator.Object("metadata", &c.Metadata).SizeLessThanOrEqualTo(MetadataSizeMaximum)
validator.String("digestMD5", c.DigestMD5).Using(net.DigestMD5Validator)
validator.String("mediaType", c.MediaType).Using(net.MediaTypeValidator)
validator.Time("archivableTime", c.ArchivableTime).NotZero()
}

type Content struct {
Expand All @@ -85,31 +91,44 @@ func (c *Content) Validate(validator structure.Validator) {
}

type Update struct {
ProcessedTime time.Time `json:"processedTime"`
ProcessedTime *time.Time `json:"processedTime,omitempty"`
ArchivableTime *time.Time `json:"archivableTime,omitempty"`
ArchivedTime *time.Time `json:"archivedTime,omitempty"`
Metadata *map[string]any `json:"metadata,omitempty"`
}

func (u *Update) Parse(parser structure.ObjectParser) {
if ptr := parser.Time("processedTime", time.RFC3339Nano); ptr != nil {
u.ProcessedTime = *ptr
}
u.ProcessedTime = parser.Time("processedTime", time.RFC3339Nano)
u.ArchivableTime = parser.Time("archivableTime", time.RFC3339Nano)
u.ArchivedTime = parser.Time("archivedTime", time.RFC3339Nano)
u.Metadata = parser.Object("metadata")
}

func (u *Update) Validate(validator structure.Validator) {
validator.Time("processedTime", &u.ProcessedTime).NotZero().BeforeNow(time.Second)
validator.Time("processedTime", u.ProcessedTime).NotZero().BeforeNow(time.Second)
validator.Time("archivableTime", u.ArchivableTime).NotZero()
validator.Time("archivedTime", u.ArchivedTime).NotZero().BeforeNow(time.Second)
validator.Object("metadata", u.Metadata).SizeLessThanOrEqualTo(MetadataSizeMaximum)

if u.ProcessedTime == nil && u.ArchivableTime == nil && u.ArchivedTime == nil && u.Metadata == nil {
validator.ReportError(structureValidator.ErrorValuesNotExistForAny("archivableTime", "archivedTime", "metadata", "processedTime"))
}
}

type Raw struct {
ID string `json:"id"`
UserID string `json:"userId"`
DataSetID string `json:"dataSetId"`
Metadata map[string]any `json:"metadata,omitempty"`
DigestMD5 string `json:"digestMD5"`
MediaType string `json:"mediaType"`
Size int `json:"size"`
ProcessedTime *time.Time `json:"processedTime,omitempty"`
CreatedTime time.Time `json:"createdTime"`
ModifiedTime *time.Time `json:"modifiedTime,omitempty"`
Revision int `json:"revision"`
ID string `json:"id"`
UserID string `json:"userId"`
DataSetID string `json:"dataSetId"`
Metadata map[string]any `json:"metadata,omitempty"`
DigestMD5 string `json:"digestMD5"`
MediaType string `json:"mediaType"`
Size int `json:"size"`
ProcessedTime *time.Time `json:"processedTime,omitempty"`
ArchivableTime *time.Time `json:"archivableTime,omitempty"`
ArchivedTime *time.Time `json:"archivedTime,omitempty"`
CreatedTime time.Time `json:"createdTime"`
ModifiedTime *time.Time `json:"modifiedTime,omitempty"`
Revision int `json:"revision"`
}

func (r *Raw) Parse(parser structure.ObjectParser) {
Expand All @@ -135,6 +154,8 @@ func (r *Raw) Parse(parser structure.ObjectParser) {
r.Size = *ptr
}
r.ProcessedTime = parser.Time("processedTime", time.RFC3339Nano)
r.ArchivableTime = parser.Time("archivableTime", time.RFC3339Nano)
r.ArchivedTime = parser.Time("archivedTime", time.RFC3339Nano)
if ptr := parser.Time("createdTime", time.RFC3339Nano); ptr != nil {
r.CreatedTime = *ptr
}
Expand All @@ -148,20 +169,30 @@ func (r *Raw) Validate(validator structure.Validator) {
validator.String("id", &r.ID).Using(DataRawIDValidator)
validator.String("userId", &r.UserID).Using(user.IDValidator)
validator.String("dataSetId", &r.DataSetID).Using(data.SetIDValidator)
validator.Object("metadata", &r.Metadata).SizeLessThanOrEqualTo(MetadataLengthMaximum)
validator.Object("metadata", &r.Metadata).SizeLessThanOrEqualTo(MetadataSizeMaximum)
validator.String("digestMD5", &r.DigestMD5).Using(net.DigestMD5Validator)
validator.String("mediaType", &r.MediaType).Using(net.MediaTypeValidator)
validator.Int("size", &r.Size).GreaterThanOrEqualTo(0)
validator.Time("processedTime", r.ProcessedTime).NotZero().After(r.CreatedTime).BeforeNow(time.Second)
validator.Time("processedTime", r.ProcessedTime).After(r.CreatedTime).BeforeNow(time.Second)
validator.Time("archivableTime", r.ArchivableTime).After(r.CreatedTime)
validator.Time("archivedTime", r.ArchivedTime).After(r.CreatedTime).BeforeNow(time.Second)
validator.Time("createdTime", &r.CreatedTime).NotZero().BeforeNow(time.Second)
validator.Time("modifiedTime", r.ModifiedTime).NotZero().After(r.CreatedTime).BeforeNow(time.Second)
validator.Time("modifiedTime", r.ModifiedTime).After(r.CreatedTime).BeforeNow(time.Second)
validator.Int("revision", &r.Revision).GreaterThanOrEqualTo(0)
}

func (r *Raw) Processed() bool {
func (r *Raw) IsProcessed() bool {
return r.ProcessedTime != nil
}

func (r *Raw) IsArchivable() bool {
return r.ArchivableTime != nil && r.ArchivableTime.Before(time.Now())
Comment thread
darinkrauss marked this conversation as resolved.
}

func (r *Raw) IsArchived() bool {
return r.ArchivedTime != nil
}

func IsValidDataRawID(value string) bool {
return ValidateDataRawID(value) == nil
}
Expand All @@ -173,7 +204,9 @@ func DataRawIDValidator(value string, errorReporter structure.ErrorReporter) {
func ValidateDataRawID(value string) error {
if value == "" {
return structureValidator.ErrorValueEmpty()
} else if !idExpression.MatchString(value) {
} else if matches := idExpression.FindStringSubmatch(value); len(matches) != 3 {
return ErrorValueStringAsDataRawIDNotValid(value)
} else if _, err := time.Parse(FilterCreatedDateFormat, matches[2]); err != nil {
return ErrorValueStringAsDataRawIDNotValid(value)
}
return nil
Expand All @@ -183,4 +216,4 @@ func ErrorValueStringAsDataRawIDNotValid(value string) error {
return errors.Preparedf(structureValidator.ErrorCodeValueNotValid, "value is not valid", "value %q is not valid as data raw id", value)
}

var idExpression = regexp.MustCompile("^[0-9a-fA-F]{24}:[0-9]{8}$")
var idExpression = regexp.MustCompile("^([0-9a-f]{24}):([0-9]{4}-[0-9]{2}-[0-9]{2})$")
Loading
Loading