val session = ScentContexts.createSession()
val portalUrl = HNormUrl.parse("https://www.amazon.com/Best-Sellers/zgbs", conf.toVolatileConfig())
val options = portalUrl.hOptions.apply {
outLinkSelector = "a[href~=/dp/]"
topLinks = 40
refresh = true
}
val page = session.load(portalUrl.urlString, options)
println(page.baseURI)
// val pages = session.loadOutPages(portalUrl, options)
val urls = session.parse(page)
.selectAnchors(options.outLinkSelector)
.mapNotNull { AsinUrlNormalizer()(it.url) }
.distinct()
.take(options.topLinks)
session.submitAll(urls)
session.context.await()
15:44:19.639 [DefaultDispatcher-worker-8] INFO ai.platon.pulsar.loop.impl.StreamingTaskLoop -- Main loop is created | #1 | SpringStreamingTaskLoop@1353388305
15:44:19.640 [DefaultDispatcher-worker-8] WARN ai.platon.pulsar.loop.impl.StreamingTaskLoop -- SWARM session does not exist, falling back to default
15:44:19.639 [DefaultDispatcher-worker-8] INFO ai.platon.pulsar.loop.impl.StreamingTaskLoop -- Main loop is created | #1 | SpringStreamingTaskLoop@1353388305
15:44:19.640 [DefaultDispatcher-worker-8] WARN ai.platon.pulsar.loop.impl.StreamingTaskLoop -- SWARM session does not exist, falling back to default