6#include "opdsprovider_p.h"
9#include <KLocalizedString>
15#include <syndication/atom/atom.h>
16#include <syndication/documentsource.h>
18#include <knewstuffcore_debug.h>
20#include "searchpreset.h"
21#include "searchpreset_p.h"
22#include "tagsfilterchecker.h"
26static const QLatin1String OPDS_REL_ACQUISITION{
"http://opds-spec.org/acquisition"};
27static const QLatin1String OPDS_REL_AC_OPEN_ACCESS{
"http://opds-spec.org/acquisition/open-access"};
28static const QLatin1String OPDS_REL_AC_BORROW{
"http://opds-spec.org/acquisition/borrow"};
29static const QLatin1String OPDS_REL_AC_BUY{
"http://opds-spec.org/acquisition/buy"};
30static const QLatin1String OPDS_REL_AC_SUBSCRIBE{
"http://opds-spec.org/acquisition/subscribe"};
32static const QLatin1String OPDS_REL_IMAGE{
"http://opds-spec.org/image"};
33static const QLatin1String OPDS_REL_THUMBNAIL{
"http://opds-spec.org/image/thumbnail"};
34static const QLatin1String OPDS_REL_CRAWL{
"http://opds-spec.org/crawlable"};
36static const QLatin1String OPDS_REL_SHELF{
"http://opds-spec.org/shelf"};
37static const QLatin1String OPDS_REL_SORT_NEW{
"http://opds-spec.org/sort/new"};
38static const QLatin1String OPDS_REL_SORT_POPULAR{
"http://opds-spec.org/sort/popular"};
39static const QLatin1String OPDS_REL_FEATURED{
"http://opds-spec.org/featured"};
40static const QLatin1String OPDS_REL_RECOMMENDED{
"http://opds-spec.org/recommended"};
41static const QLatin1String OPDS_REL_SUBSCRIPTIONS{
"http://opds-spec.org/subscriptions"};
42static const QLatin1String OPDS_EL_PRICE{
"opds:price"};
47static const QLatin1String OPDS_ATOM_MT{
"application/atom+xml"};
48static const QLatin1String OPDS_PROFILE{
"profile=opds-catalog"};
49static const QLatin1String OPDS_TYPE_ENTRY{
"type=entry"};
53static const QLatin1String REL_START{
"start"};
63static const QLatin1String REL_UP{
"up"};
64static const QLatin1String REL_SELF{
"self"};
65static const QLatin1String REL_ALTERNATE{
"alternate"};
66static const QLatin1String ATTR_CURRENCY_CODE{
"currencycode"};
70static const QLatin1String OPENSEARCH_NS{
"http://a9.com/-/spec/opensearch/1.1/"};
71static const QLatin1String OPENSEARCH_MT{
"application/opensearchdescription+xml"};
72static const QLatin1String REL_SEARCH{
"search"};
74static const QLatin1String OPENSEARCH_EL_URL{
"Url"};
75static const QLatin1String OPENSEARCH_ATTR_TYPE{
"type"};
76static const QLatin1String OPENSEARCH_ATTR_TEMPLATE{
"template"};
77static const QLatin1String OPENSEARCH_SEARCH_TERMS{
"searchTerms"};
78static const QLatin1String OPENSEARCH_COUNT{
"count"};
79static const QLatin1String OPENSEARCH_START_INDEX{
"startIndex"};
80static const QLatin1String OPENSEARCH_START_PAGE{
"startPage"};
82static const QLatin1String HTML_MT{
"text/html"};
84static const QLatin1String KEY_MIME_TYPE{
"data##mimetype="};
85static const QLatin1String KEY_URL{
"data##url="};
86static const QLatin1String KEY_LANGUAGE{
"data##language="};
88class OPDSProviderPrivate
91 OPDSProviderPrivate(OPDSProvider *qq)
94 , loadingExtraDetails(false)
111 QDateTime currentTime;
112 bool loadingExtraDetails;
114 XmlLoader *xmlLoader;
116 Entry::List cachedEntries;
117 SearchRequest currentRequest;
119 QUrl openSearchDocumentURL;
120 QString openSearchTemplate;
123 QUrl openSearchStringForRequest(
const KNSCore::SearchRequest &request)
126 QUrl searchUrl = QUrl(openSearchTemplate);
128 QUrlQuery templateQuery(searchUrl);
131 for (QPair<QString, QString> key : templateQuery.queryItems()) {
132 if (key.second.contains(OPENSEARCH_SEARCH_TERMS)) {
133 query.addQueryItem(key.first, request.searchTerm());
134 }
else if (key.second.contains(OPENSEARCH_COUNT)) {
136 }
else if (key.second.contains(OPENSEARCH_START_PAGE)) {
138 }
else if (key.second.contains(OPENSEARCH_START_INDEX)) {
148 QUrl fixRelativeUrl(QString urlPart)
150 QUrl
query = QUrl(urlPart);
151 if (
query.isRelative()) {
152 if (selfUrl.isEmpty() || QUrl(selfUrl).isRelative()) {
153 return currentUrl.resolved(query);
155 return QUrl(selfUrl).resolved(query);
161 Entry::List installedEntries()
const {{Entry::List entries;
162 for (
const Entry &entry : std::as_const(cachedEntries)) {
163 if (entry.status() == KNSCore::Entry::Installed || entry.status() == KNSCore::Entry::Updateable) {
164 entries.append(entry);
171void slotLoadingFailed()
173 qCWarning(KNEWSTUFFCORE) <<
"OPDS Loading failed for" << currentUrl;
174 Q_EMIT q->loadingFailed(currentRequest);
179void parseOpenSearchDocument(
const QDomDocument &doc){{openSearchTemplate = QString();
181 qCWarning(KNEWSTUFFCORE) <<
"Opensearch link does not point at document with opensearch namespace" << openSearchDocumentURL;
187 if (openSearchTemplate.isEmpty() || el.
attribute(OPENSEARCH_ATTR_TYPE).
contains(OPDS_PROFILE)) {
188 openSearchTemplate = el.
attribute(OPENSEARCH_ATTR_TEMPLATE);
204void parseFeedData(
const QDomDocument &doc)
206 Syndication::DocumentSource source(doc.
toByteArray(), currentUrl.toString());
207 Syndication::Atom::Parser parser;
208 Syndication::Atom::FeedDocumentPtr feedDoc = parser.
parse(source).staticCast<Syndication::Atom::FeedDocument>();
210 QString fullEntryMimeType = QStringList({OPDS_ATOM_MT, OPDS_TYPE_ENTRY, OPDS_PROFILE}).join(QStringLiteral(
";"));
212 if (!feedDoc->isValid()) {
213 qCWarning(KNEWSTUFFCORE) <<
"OPDS Feed at" << currentUrl <<
"not valid";
214 Q_EMIT q->loadingFailed(currentRequest);
217 if (!feedDoc->title().isEmpty()) {
218 providerName = feedDoc->title();
220 if (!feedDoc->icon().isEmpty()) {
221 iconUrl = QUrl(fixRelativeUrl(feedDoc->icon()));
225 QList<SearchPreset> presets;
228 SearchRequest request(SortMode::Downloads, Filter::None, providerId);
229 SearchPreset preset(
new SearchPresetPrivate{
234 .providerId = providerId,
241 for (
auto link : feedDoc->links()) {
242 if (
link.rel().contains(REL_SELF)) {
243 selfUrl =
link.href();
247 for (
auto link : feedDoc->links()) {
249 if (
link.rel() == REL_SEARCH &&
link.type() == OPENSEARCH_MT) {
250 std::function<void(Syndication::Atom::Link)> osdUrlLoader;
251 osdUrlLoader = [
this, &osdUrlLoader](Syndication::Atom::Link theLink) {
252 openSearchDocumentURL = fixRelativeUrl(theLink.href());
253 xmlLoader =
new XmlLoader(q);
255 QObject::connect(xmlLoader, &XmlLoader::signalLoaded, q, [
this](
const QDomDocument &doc) {
256 q->d->parseOpenSearchDocument(doc);
259 qCWarning(KNEWSTUFFCORE) <<
"OpenSearch XML Document Loading failed" << openSearchDocumentURL;
263 &XmlLoader::signalHttpError,
265 [
this, &osdUrlLoader, theLink](
int status, QList<QNetworkReply::RawHeaderPair> rawHeaders) {
267 QDateTime retryAfter;
268 static const QByteArray retryAfterKey{
"Retry-After"};
270 if (headerPair.first == retryAfterKey) {
275 QNetworkRequest dummyRequest;
276 dummyRequest.
setRawHeader(QByteArray{
"Last-Modified"}, headerPair.second);
283 osdUrlLoader(theLink);
288 static const KFormat formatter;
289 Q_EMIT q->signalErrorCode(
290 KNSCore::ErrorCode::TryAgainLaterError,
291 i18n(
"The service is currently undergoing maintenance and is expected to be back in %1.",
298 xmlLoader->load(openSearchDocumentURL);
300 }
else if (
link.type().contains(OPDS_PROFILE) &&
link.rel() != REL_SELF) {
301 SearchRequest request(SortMode::Downloads, Filter::None, fixRelativeUrl(
link.href()).toString());
302 SearchPreset preset(
new SearchPresetPrivate{
304 .displayName =
link.title().isEmpty() ?
link.rel() :
link.title(),
308 if (
link.rel() == REL_START) {
311 if (
link.rel() == OPDS_REL_FEATURED) {
314 if (
link.rel() == OPDS_REL_SHELF) {
317 if (
link.rel() == OPDS_REL_SORT_NEW) {
320 if (
link.rel() == OPDS_REL_SORT_POPULAR) {
323 if (
link.rel() == REL_UP) {
326 if (
link.rel() == OPDS_REL_CRAWL) {
329 if (
link.rel() == OPDS_REL_RECOMMENDED) {
332 if (
link.rel() == OPDS_REL_SUBSCRIPTIONS) {
335 return SearchPreset::Type::NoPresetType;
337 .providerId = providerId,
342 TagsFilterChecker downloadTagChecker(q->downloadTagFilter());
343 TagsFilterChecker entryTagChecker(q->tagFilter());
345 for (
int i = 0; i < feedDoc->entries().size(); i++) {
346 Syndication::Atom::Entry feedEntry = feedDoc->entries().at(i);
349 entry.setName(feedEntry.
title());
350 entry.setProviderId(providerId);
351 entry.setUniqueId(feedEntry.
id());
353 entry.setStatus(KNSCore::Entry::Invalid);
354 for (
const Entry &cachedEntry : std::as_const(cachedEntries)) {
355 if (entry.uniqueId() == cachedEntry.uniqueId()) {
363 QStringList entryTags;
364 for (
int j = 0; j < feedEntry.
categories().size(); j++) {
365 QString tag = feedEntry.
categories().at(j).label();
371 if (entryTagChecker.filterAccepts(entryTags)) {
372 entry.setTags(entryTags);
377 for (
int j = 0; j < feedEntry.
authors().size(); j++) {
379 Syndication::Atom::Person person = feedEntry.
authors().
at(j);
380 author.setId(person.
uri());
381 author.setName(person.
name());
382 author.setEmail(person.
email());
383 author.setHomepage(person.
uri());
384 entry.setAuthor(author);
386 entry.setLicense(feedEntry.
rights());
392 entry.setShortSummary(feedEntry.
summary());
394 int counterThumbnails = 0;
395 int counterImages = 0;
396 QString groupEntryUrl;
397 for (
int j = 0; j < feedEntry.
links().size(); j++) {
398 Syndication::Atom::Link
link = feedEntry.
links().at(j);
400 KNSCore::Entry::DownloadLinkInformation download;
401 download.id = entry.downloadLinkCount() + 1;
403 QStringList linkRelation =
link.rel().split(QStringLiteral(
" "));
407 if (!
link.hrefLanguage().isEmpty()) {
408 tags.
append(KEY_LANGUAGE +
link.hrefLanguage());
410 QString linkUrl = fixRelativeUrl(
link.href()).toString();
411 tags.
append(KEY_URL + linkUrl);
412 download.name =
link.title();
413 download.size =
link.length() / 1000;
414 download.tags = tags;
415 download.isDownloadtypeLink =
false;
417 if (
link.rel().startsWith(OPDS_REL_ACQUISITION)) {
418 if (
link.title().isEmpty()) {
421 l.
append(QStringLiteral(
"(") +
link.rel().split(QStringLiteral(
"/")).last() + QStringLiteral(
")"));
422 download.name = l.
join(QStringLiteral(
" "));
425 if (!downloadTagChecker.filterAccepts(download.tags)) {
429 if (linkRelation.
contains(OPDS_REL_AC_BORROW) || linkRelation.
contains(OPDS_REL_AC_SUBSCRIBE) || linkRelation.
contains(OPDS_REL_AC_BUY)) {
433 }
else if (linkRelation.
contains(OPDS_REL_ACQUISITION) || linkRelation.
contains(OPDS_REL_AC_OPEN_ACCESS)) {
434 download.isDownloadtypeLink =
true;
436 if (entry.status() != KNSCore::Entry::Installed && entry.status() != KNSCore::Entry::Updateable) {
437 entry.setStatus(KNSCore::Entry::Downloadable);
449 entry.appendDownloadLinkInformation(download);
451 }
else if (
link.rel().startsWith(OPDS_REL_IMAGE)) {
452 if (
link.rel() == OPDS_REL_THUMBNAIL) {
453 entry.setPreviewUrl(linkUrl, KNSCore::Entry::PreviewType(counterThumbnails));
454 counterThumbnails += 1;
456 entry.setPreviewUrl(linkUrl, KNSCore::Entry::PreviewType(counterImages + 3));
464 if (
link.type().startsWith(OPDS_ATOM_MT)) {
465 if (
link.type() == fullEntryMimeType) {
466 entry.appendDownloadLinkInformation(download);
468 groupEntryUrl = linkUrl;
471 }
else if (
link.type() == HTML_MT && linkRelation.
contains(REL_ALTERNATE)) {
472 entry.setHomepage(QUrl(linkUrl));
474 }
else if (downloadTagChecker.filterAccepts(download.tags)) {
475 entry.appendDownloadLinkInformation(download);
488 if (entry.releaseDate().isNull()) {
489 entry.setReleaseDate(date.
date());
492 if (entry.status() != KNSCore::Entry::Invalid) {
493 entry.setPayload(QString());
497 if (date.
secsTo(currentTime) > 360) {
498 if (entry.releaseDate() < date.
date()) {
499 entry.setUpdateReleaseDate(date.
date());
500 if (entry.status() == KNSCore::Entry::Installed) {
501 entry.setStatus(KNSCore::Entry::Updateable);
506 if (counterThumbnails == 0) {
508 if (!feedDoc->icon().isEmpty()) {
509 entry.setPreviewUrl(fixRelativeUrl(feedDoc->icon()).toString());
513 if (entry.downloadLinkCount() == 0) {
518 entry.setPayload(groupEntryUrl);
522 entries.append(entry);
525 if (loadingExtraDetails) {
526 Q_EMIT q->entryDetailsLoaded(entries.first());
527 loadingExtraDetails =
false;
529 Q_EMIT q->entriesLoaded(currentRequest, entries);
531 Q_EMIT q->searchPresetsLoaded(presets);
536OPDSProvider::OPDSProvider()
537 : d(new OPDSProviderPrivate(this))
541OPDSProvider::~OPDSProvider() =
default;
543QString OPDSProvider::id()
const
545 return d->providerId;
548QString OPDSProvider::name()
const
550 return d->providerName;
553QUrl OPDSProvider::icon()
const
560 d->currentRequest = request;
562 if (request.filter() == Filter::Installed) {
563 Q_EMIT entriesLoaded(request, d->installedEntries());
564 Q_EMIT loadingDone(request);
566 }
else if (request.filter() == Filter::ExactEntryId) {
567 for (Entry entry : d->cachedEntries) {
568 if (entry.uniqueId() == request.searchTerm()) {
569 loadEntryDetails(entry);
574 d->currentUrl =
QUrl(request.searchTerm());
575 }
else if (!d->openSearchTemplate.isEmpty() && !request.searchTerm().
isEmpty()) {
578 d->currentUrl = d->openSearchStringForRequest(request);
583 QUrl url = d->currentUrl;
585 qCDebug(KNEWSTUFFCORE) <<
"requesting url" << url;
586 d->xmlLoader =
new XmlLoader(
this);
588 d->loadingExtraDetails =
false;
590 d->parseFeedData(doc);
592 connect(d->xmlLoader, &XmlLoader::signalFailed,
this, [
this]() {
593 d->slotLoadingFailed();
595 d->xmlLoader->load(url);
597 Q_EMIT loadingFailed(request);
602void OPDSProvider::loadEntryDetails(
const Entry &entry)
605 QString entryMimeType =
QStringList({OPDS_ATOM_MT, OPDS_TYPE_ENTRY, OPDS_PROFILE}).join(QStringLiteral(
";"));
606 for (
auto link : entry.downloadLinkInformationList()) {
607 if (
link.tags.contains(KEY_MIME_TYPE + entryMimeType)) {
609 if (
string.startsWith(KEY_URL)) {
610 url =
QUrl(
string.split(QStringLiteral(
"=")).last());
616 d->xmlLoader =
new XmlLoader(
this);
618 d->loadingExtraDetails =
true;
620 d->parseFeedData(doc);
622 connect(d->xmlLoader, &XmlLoader::signalFailed,
this, [
this]() {
623 d->slotLoadingFailed();
625 d->xmlLoader->load(url);
629void OPDSProvider::loadPayloadLink(
const KNSCore::Entry &entry,
int linkNumber)
633 if (downloadInfo.id == linkNumber) {
634 for (
QString string : downloadInfo.tags) {
635 if (
string.startsWith(KEY_URL)) {
636 copy.setPayload(
string.split(QStringLiteral(
"=")).last());
641 Q_EMIT payloadLinkLoaded(copy);
644bool OPDSProvider::setProviderXML(
const QDomElement &xmldata)
649 d->providerId = xmldata.
attribute(QStringLiteral(
"downloadurl"));
652 if (!iconurl.isValid()) {
655 d->iconUrl = iconurl;
665 d->currentUrl =
QUrl(d->providerId);
667 d->initialized =
true;
668 Q_EMIT providerInitialized(
this);
673bool OPDSProvider::isInitialized()
const
675 return d->initialized;
678void OPDSProvider::setCachedEntries(
const KNSCore::Entry::List &cachedEntries)
680 d->cachedEntries = cachedEntries;
683[[nodiscard]]
QString OPDSProvider::version()
688[[nodiscard]]
QUrl OPDSProvider::website()
693[[nodiscard]]
QUrl OPDSProvider::host()
698[[nodiscard]]
QString OPDSProvider::contactEmail()
703[[nodiscard]]
bool OPDSProvider::supportsSsl()
709#include "moc_opdsprovider_p.cpp"
KNewStuff data entry container.
QList< DownloadLinkInformation > downloadLinkInformationList() const
A list of downloadable data for this entry.
@ GroupEntry
these are entries whose payload is another feed. Currently only used by the OPDS provider.
@ CatalogEntry
These are the main entries that KNewStuff can get the details about and download links for.
@ FolderUp
preset indicating going up in the search result hierarchy.
@ New
preset indicating new items.
@ Featured
preset for featured items.
@ Shelf
preset indicating previously acquired items.
@ Popular
preset indicating popular items.
@ Recommended
preset for recommended. This may be customized by the server per user.
@ AllEntries
preset indicating all possible entries, such as a crawlable list. Might be intense to load.
@ Subscription
preset indicating items that the user is subscribed to.
@ Start
preset indicating the first entry.
@ Root
preset indicating a root directory.
bool isEscapedHTML() const
QList< Person > authors() const
QList< Category > categories() const
QList< Link > links() const
Syndication::SpecificDocumentPtr parse(const Syndication::DocumentSource &source) const override
QString childNodesAsXML() const
QList< QDomElement > elementsByTagName(const QString &tagName) const
Q_SCRIPTABLE CaptureState status()
QString i18n(const char *text, const TYPE &arg...)
KSERVICE_EXPORT KService::List query(FilterFunc filterFunc)
KIOCORE_EXPORT CopyJob * link(const QList< QUrl > &src, const QUrl &destDir, JobFlags flags=DefaultFlags)
QAction * copy(const QObject *recvr, const char *slot, QObject *parent)
QDateTime currentDateTime()
qint64 currentMSecsSinceEpoch()
qint64 currentSecsSinceEpoch()
QDateTime fromSecsSinceEpoch(qint64 secs)
qint64 secsTo(const QDateTime &other) const const
qint64 toMSecsSinceEpoch() const const
qint64 toSecsSinceEpoch() const const
QDomElement documentElement() const const
QByteArray toByteArray(int indent) const const
QString attribute(const QString &name, const QString &defValue) const const
QString tagName() const const
QString text() const const
QDomNode firstChild() const const
QDomElement firstChildElement(const QString &tagName, const QString &namespaceURI) const const
bool isNull() const const
QDomNode nextSibling() const const
QDomElement nextSiblingElement(const QString &tagName, const QString &namespaceURI) const const
QDomElement toElement() const const
void append(QList< T > &&value)
const_reference at(qsizetype i) const const
QString toCurrencyString(double value, const QString &symbol, int precision) const const
QMetaObject::Connection connect(const QObject *sender, PointerToMemberFunction signal, Functor functor)
bool contains(QChar ch, Qt::CaseSensitivity cs) const const
bool isEmpty() const const
QString number(double n, char format, int precision)
bool startsWith(QChar c, Qt::CaseSensitivity cs) const const
float toFloat(bool *ok) const const
QString trimmed() const const
bool contains(QLatin1StringView str, Qt::CaseSensitivity cs) const const
QString join(QChar separator) const const
QFuture< ArgsType< Signal > > connect(Sender *sender, Signal signal)
QUrl fromLocalFile(const QString &localFile)
bool isEmpty() const const
QString scheme() const const
void setQuery(const QString &query, ParsingMode mode)
QDateTime toDateTime() const const