Skip to content
  •  
  •  
  •  
8 changes: 7 additions & 1 deletion .castor/qa.php
Original file line number Diff line number Diff line change
Expand Up @@ -442,13 +442,19 @@ function updateExamplesBaselines(): void
{
io()->title('Updating examples baselines');

io()->confirm(
$confirmed = io()->confirm(
"This command will take the results of the tests and write them in their respective baseline files.\n
This will change the expected outputs of the tests.\n
Are you 100% sure the current results of the tests are correct and the baseline is wrong?",
false,
);

if (!$confirmed) {
io()->note('Nothing was written.');

return;
}

updateBaseline(
'schema-org',
SchemaOrgFilesystem::SCHEMA_ORG_EXAMPLES_DIR,
Expand Down
81 changes: 56 additions & 25 deletions .castor/schema-org.php
Original file line number Diff line number Diff line change
Expand Up @@ -51,35 +51,45 @@ function versionedFiles(): array
];
}

#[AsTask(description: 'Download the schema.org types definition file')]
#[AsTask(description: 'Download the schema.org types definition file and the schema.org JSON-LD context')]
function download(
bool $overwrite = false,
): void {
io()->title('Downloading the schema.org types definition file');
$filename = getCurrentSchemaOrgDefinitionFileName();
io()->title('Downloading the schema.org types definition file and JSON-LD context');
$skipped = false;

if (!$overwrite && file_exists($filename)) {
io()->warning("
The definition file already exists. Skipping.\n
Run `castor schema-org:download --overwrite` if you want to overwrite it.
");
foreach (getCurrentReleaseFiles() as $filename => $url) {
if (!$overwrite && file_exists($filename)) {
$skipped = true;

continue;
}

return;
http_download($url, $filename);
}

http_download(getDefinitionFileUrl(SchemaOrg::VERSION), $filename);
if ($skipped) {
io()->warning("
Files already downloaded were skipped.\n
Run `castor schema-org:download --overwrite` if you want to overwrite them.
");
}

io()->success('Schema.org types definition file downloaded successfully');
io()->success('Schema.org release files downloaded successfully');
}

#[AsTask(description: 'Generate classes for JSON-LD validation based on the schema.org types definition file')]
function generate(): void
{
io()->title('Generating schema.org classes');

if (!file_exists(getCurrentSchemaOrgDefinitionFileName())) {
io()->info('The schema.org types definition file is missing, downloading it.');
download();
foreach (array_keys(getCurrentReleaseFiles()) as $filename) {
if (!file_exists($filename)) {
io()->info('A schema.org release file is missing, downloading it.');
download();

break;
}
}

$generator = new Generator();
Expand Down Expand Up @@ -138,18 +148,18 @@ function updateVersion(
return 1;
}

// A version may be announced while its definition file is not published yet: the
// A version may be announced while its data files are not published yet: the
// latest usable release is the latest one we can actually download.
$latest = null;

foreach ($releases as $release) {
if (definitionFileIsPublished($release)) {
if (releaseFilesArePublished($release)) {
$latest = $release;

break;
}

io()->warning(\sprintf('Release %s is listed but its definition file is not published yet, skipping it.', $release));
io()->warning(\sprintf('Release %s is listed but its data files are not published yet, skipping it.', $release));
}

if (null === $latest) {
Expand Down Expand Up @@ -228,13 +238,19 @@ function fetchPublishedReleases(): array
}

/**
* Whether the definition file of a release can actually be downloaded: schema.org
* sometimes lists a release before publishing its data files.
* Whether the definition file and the JSON-LD context of a release can actually be
* downloaded: schema.org sometimes lists a release before publishing its data files.
*/
function definitionFileIsPublished(string $version): bool
function releaseFilesArePublished(string $version): bool
{
try {
return 200 === http_request('HEAD', getDefinitionFileUrl($version))->getStatusCode();
foreach ([getDefinitionFileUrl($version), getContextFileUrl($version)] as $url) {
if (200 !== http_request('HEAD', $url)->getStatusCode()) {
return false;
}
}

return true;
} catch (ExceptionInterface) {
return false;
}
Expand Down Expand Up @@ -292,11 +308,26 @@ function getDefinitionFileUrl(string $version): string
);
}

function getCurrentSchemaOrgDefinitionFileName(): string
function getContextFileUrl(string $version): string
{
return \sprintf(
'%s/schemaorg-%s-https.jsonld',
Filesystem::CACHE_DIR_SCHEMA_ORG,
SchemaOrg::VERSION,
'https://raw.githubusercontent.com/schemaorg/schemaorg/main/data/releases/%s/schemaorgcontext.jsonld',
$version,
);
}

/**
* The files of the configured release: the types definition file the classes are
* generated from, and the JSON-LD context the static context is built from.
*
* @return array<string, string> the local file name, and the URL it is downloaded from
*/
function getCurrentReleaseFiles(): array
{
$filesystem = new Filesystem();

return [
$filesystem->getSchemaOrgTypesDefinitionFilename() => getDefinitionFileUrl(SchemaOrg::VERSION),
$filesystem->getSchemaOrgContextFilename() => getContextFileUrl(SchemaOrg::VERSION),
];
}
38 changes: 38 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,46 @@

## [Unreleased]

### Changed

- Schema.org vocabulary upgraded from 30.0 to 30.1. This release promotes most of the
pending.schema.org terms to the core vocabulary (`jobTitle`, `Legislation`, `DefinedTerm`,
`MemberProgram`...): they are no longer reported as pending vocabulary usage.
- Google validator: `VideoObject` now recommends a `creator` (or `author`), which must have a
`name` or an `alternateName`, and accepts the `LikeAction`, `CommentAction` and
`ShareAction` interaction types, following the Google documentation.
- The FAQ rich result is marked as retired by Google. `FAQPage` is still validated, for
backward compatibility.
- Google validator: merchant listings now cover every property the Google documentation
lists. The product-level recommendations (`category`, `gtin`, `sku`, `size`, `color`,
`hasCertification`...), the offer-level ones (`itemCondition`, `priceValidUntil`, `url`,
`validFrom`, `validThrough`) and the nested shipping, return, unit pricing, size, audience,
certification and 3D model structures are checked. An offer `price` requires a
`priceCurrency`, a finite return window requires `merchantReturnDays`, and return shipping
fees require `returnShippingFeesAmount`.
- Google validator: every structured data type is aligned with the current Google
documentation. A recommended property is reported when missing, unless Google makes it
conditional ("if applicable", "only recommended for..."): such a property is still checked
when present, and enforced by a special rule when the document tells whether it applies.
Notable changes: new recommendations (job posting experience, local business department,
member program and tier URLs, product snippet notes and aggregate offers, recipe and step
videos, vacation rental details), new nested structures (organization return and shipping
policies, video key moments and live badge, beta carousel items, recipe sections), and
requirements relaxed where Google relaxed them (course `provider`, event `description` and
`image`, recipe ingredients and instructions). Most organization, profile page, discussion
forum and Q&A recommendations are conditional and no longer reported when missing.

### Fixed

- The static schema.org context is now generated from the context schema.org publishes with
each release, instead of being rebuilt from the vocabulary definition file. Documents
expand as with `"@context": "https://schema.org"`: `schema:` compact IRIs expand to
`http://schema.org/` (they expanded to `https://schema.org/`), and the definition file's
own prefixes can no longer shadow a schema.org term (such as `geo` since 30.1).
- Google validator: a property accepting either a data type or a structured type (such as
"Text or CategoryCode") now validates the nested properties of an object value. The
`ImageObject` of an article image, the `HowToStep` of recipe instructions and an object
breadcrumb `item` were never checked so far.
- JSON-LD script tags whose `type` attribute has no quotes (valid HTML, and what minifiers
produce) or has whitespace around the `=` are no longer ignored.
- A `data-type="application/ld+json"` attribute no longer makes a script tag count as JSON-LD.
Expand Down
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -422,7 +422,7 @@ vocabulary definitions. These commands refresh them:
Command | Description
---- | -----
`castor schema-org:update-version` | Bump the schema.org version to the latest release published on GitHub
`castor schema-org:download` | Download the schema.org types definition file
`castor schema-org:download` | Download the schema.org types definition file and JSON-LD context
`castor schema-org:generate` | Generate the schema.org validation classes
`castor schema-org:download-examples` | Refresh the schema.org examples used by the tests
`castor google:download` | Crawl the Google structured-data documentation
Expand Down
8 changes: 8 additions & 0 deletions generators/Google/DocumentationCoverageAuditor.php
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,14 @@ public function verifyGoogleDocCoverage(): array
}

[$discovered, $fetchFailures] = $this->crawler->discoverStructuredDataDocs($client, array_values(array_unique($seedUrls)));

// Other pages may still link to a retired type whose page Google removed:
// failing to fetch it is the expected outcome, not a gap.
$fetchFailures = array_filter(
$fetchFailures,
static fn (string $url): bool => 'retired' !== ($manifestUrls[$url]['status'] ?? null),
\ARRAY_FILTER_USE_KEY,
);
$implementedUrls = $this->getImplementedDocumentationUrls();

$missingFromManifest = array_diff_key($discovered, $manifestUrls);
Expand Down
20 changes: 19 additions & 1 deletion generators/Google/DocumentationCrawler.php
Original file line number Diff line number Diff line change
Expand Up @@ -130,13 +130,31 @@ public function discoverStructuredDataDocs(HttpClientInterface $client, array $s
$visited[$currentUrl] = true;

try {
$content = $client->request('GET', $currentUrl)->getContent();
$response = $client->request('GET', $currentUrl);
$content = $response->getContent();
} catch (\Throwable $exception) {
$failures[$currentUrl] = $exception->getMessage();

continue;
}

// Google retires a type by redirecting its page, either to the "What's
// new" log or to the page that now covers it ("logo" to "organization"):
// the original URL is no longer a page of its own. Only the target is
// worth crawling, and only when it is a structured data page itself.
$effectiveUrl = $response->getInfo('url');
$effectiveUrl = \is_string($effectiveUrl) ? strtok($effectiveUrl, '#?') : false;

if (false !== $effectiveUrl && $effectiveUrl !== $currentUrl) {
$redirectTarget = $this->normalizeStructuredDataUrl($effectiveUrl);

if (null !== $redirectTarget && !isset($visited[$redirectTarget])) {
$queue[] = $redirectTarget;
}

continue;
}

$crawler = new Crawler($content);
$title = trim((string) $crawler->filter('title')->first()->text('', true));

Expand Down
50 changes: 39 additions & 11 deletions generators/SchemaOrg/Filesystem.php
Original file line number Diff line number Diff line change
Expand Up @@ -57,9 +57,46 @@ public function saveSchemaOrgExample(string $prefix, string $content): void

public function getSchemaOrgTypesDefinition(): string
{
$filename = $this->getSchemaOrgTypesDefinitionFilename();
return $this->readCachedFile($this->getSchemaOrgTypesDefinitionFilename());
}

public function hasSchemaOrgContextFile(): bool
{
return $this->exists(
$this->getSchemaOrgContextFilename(),
);
}

/**
* The JSON-LD context schema.org publishes along with each release: the one a JSON-LD
* processor gets when it resolves "@context": "https://schema.org".
*/
public function getSchemaOrgContext(): string
{
return $this->readCachedFile($this->getSchemaOrgContextFilename());
}

public function getSchemaOrgTypesDefinitionFilename(): string
{
return \sprintf(
'%s/schemaorg-%s-https.jsonld',
self::CACHE_DIR_SCHEMA_ORG,
SchemaOrg::VERSION,
);
}

if (!$this->hasSchemaOrgTypesDefinitionFile()) {
public function getSchemaOrgContextFilename(): string
{
return \sprintf(
'%s/schemaorg-%s-context.jsonld',
self::CACHE_DIR_SCHEMA_ORG,
SchemaOrg::VERSION,
);
}

private function readCachedFile(string $filename): string
{
if (!$this->exists($filename)) {
throw new IOException(\sprintf('Failed to read file "%s": ', $filename), 0, null, $filename);
}

Expand All @@ -71,13 +108,4 @@ public function getSchemaOrgTypesDefinition(): string

return $content;
}

private function getSchemaOrgTypesDefinitionFilename(): string
{
return \sprintf(
'%s/schemaorg-%s-https.jsonld',
self::CACHE_DIR_SCHEMA_ORG,
SchemaOrg::VERSION,
);
}
}
2 changes: 1 addition & 1 deletion generators/SchemaOrg/SchemaOrg.php
Original file line number Diff line number Diff line change
Expand Up @@ -17,5 +17,5 @@ class SchemaOrg
// Bump this version with care! Sometimes, a version is released but not
// yet available on GitHub.
// Be sure to check https://schema.org/docs/releases.html first.
public const VERSION = '30.0';
public const VERSION = '30.1';
}
44 changes: 12 additions & 32 deletions generators/StaticSchemaOrgContextGenerator.php
Original file line number Diff line number Diff line change
Expand Up @@ -36,45 +36,25 @@ public function __construct(
}

/**
* Uses the Context Processing algorithm on schema.org definitions and pre-build a PHP and a JSON version of it.
* Uses the Context Processing algorithm on the official schema.org context and pre-build a PHP and a JSON version of it.
* This way, we can completely skip the context processing (which is slooow) for schema.org.
*
* The context is the one schema.org publishes with each release, which is what a JSON-LD
* processor gets when it resolves "@context": "https://schema.org". It is used as is
* rather than rebuilt from the definition file: the prefixes the definition file
* declares for its own use can clash with schema.org terms (since 30.1, a "geo" prefix
* for GeoSPARQL shadows the "geo" property) and do not match the published ones (its
* "schema" prefix is https, the published one is http).
*/
public function generate(): void
{
$schemaOrgDefinitions = json_decode(
$this->filesystem->getSchemaOrgTypesDefinition(),
);

$localContext = (object) [
'@vocab' => 'http://schema.org/',
'@version' => 1.1,
'id' => '@id',
'type' => '@type',
];
$officialContext = json_decode($this->filesystem->getSchemaOrgContext());

foreach ((array) $schemaOrgDefinitions->{'@context'} as $term => $mapping) {
if (\is_string($mapping)) {
$localContext->{$term} = $mapping;
}
if (!\is_object($officialContext) || !isset($officialContext->{'@context'}) || !\is_object($officialContext->{'@context'})) {
throw new \RuntimeException('The schema.org context file does not hold a "@context" object.');
}

foreach ((array) $schemaOrgDefinitions->{'@graph'} as $entry) {
if (!\is_object($entry) || !isset($entry->{'@id'}) || !\is_string($entry->{'@id'})) {
continue;
}

if (!str_starts_with($entry->{'@id'}, 'schema:')) {
continue;
}

$term = substr($entry->{'@id'}, 7);

if ('' === $term || isset($localContext->{$term})) {
continue;
}

$localContext->{$term} = 'http://schema.org/' . $term;
}
$localContext = $officialContext->{'@context'};

$processer = new ContextProcessor();
$remoteContexts = [];
Expand Down
Loading
Loading