Kgapi

fileabstractresumablejob.cpp
1/*
2 * This file is part of LibKGAPI library
3 *
4 * SPDX-FileCopyrightText: 2020 David Barchiesi <david@barchie.si>
5 *
6 * SPDX-License-Identifier: LGPL-2.1-only OR LGPL-3.0-only OR LicenseRef-KDE-Accepted-LGPL
7 */
8
9#include "fileabstractresumablejob.h"
10#include "debug.h"
11#include "utils.h"
12
13#include <QMimeDatabase>
14#include <QNetworkReply>
15#include <QNetworkRequest>
16#include <QUrlQuery>
17
18using namespace KGAPI2;
19using namespace KGAPI2::Drive;
20
21namespace
22{
23static const int ChunkSize = 262144;
24}
25
26class Q_DECL_HIDDEN FileAbstractResumableJob::Private
27{
28public:
29 Private(FileAbstractResumableJob *parent);
30 void startUploadSession();
31 void uploadChunk(bool lastChunk);
32 void processNext();
33 void readFromDevice();
34 bool isTotalSizeKnown() const;
35
36 void _k_uploadProgress(qint64 bytesSent, qint64 totalBytes);
37
38 FilePtr metaData;
39 QIODevice *device = nullptr;
40
41 QString sessionPath;
42 QList<QByteArray> chunks;
43 int uploadedSize = 0;
44 int totalUploadSize = 0;
45
46 enum SessionState { ReadyStart, Started, ClientEnough, Completed };
47
48 SessionState sessionState = ReadyStart;
49
50private:
52};
53
54FileAbstractResumableJob::Private::Private(FileAbstractResumableJob *parent)
55 : q(parent)
56{
57}
58
59void FileAbstractResumableJob::Private::startUploadSession()
60{
61 qCDebug(KGAPIDebug) << "Opening resumable upload session";
62
63 // Setup job url and generic params
64 QUrl url = q->createUrl();
65 q->updateUrl(url);
66
67 QUrlQuery query(url);
68 query.removeQueryItem(QStringLiteral("uploadType"));
69 query.addQueryItem(QStringLiteral("uploadType"), QStringLiteral("resumable"));
70 url.setQuery(query);
71
72 QNetworkRequest request(url);
73 QByteArray rawData;
74 if (!metaData.isNull()) {
75 if (metaData->mimeType().isEmpty() && !chunks.isEmpty()) {
76 // No mimeType set, determine from title and first chunk
77 const QMimeDatabase db;
78 const QMimeType mime = db.mimeTypeForFileNameAndData(metaData->title(), chunks.first());
79 const QString contentType = mime.name();
80 metaData->setMimeType(contentType);
81 qCDebug(KGAPIDebug) << "Metadata mimeType was missing, determined" << contentType;
82 }
83 qCDebug(KGAPIDebug) << "Metadata has mimeType" << metaData->mimeType();
84
85 rawData = File::toJSON(metaData);
86 }
87
88 QString contentType = QStringLiteral("application/json");
89
90 request.setHeader(QNetworkRequest::ContentLengthHeader, rawData.length());
91 request.setHeader(QNetworkRequest::ContentTypeHeader, contentType);
92
93 q->enqueueRequest(request, rawData, contentType);
94}
95
96void FileAbstractResumableJob::Private::uploadChunk(bool lastChunk)
97{
98 QString rangeHeader;
99 QByteArray partData;
100 if (chunks.isEmpty()) {
101 // We have consumed everything but must send one last request with total file size
102 qCDebug(KGAPIDebug) << "Chunks is empty, sending only final size" << uploadedSize;
103 rangeHeader = QStringLiteral("bytes */%1").arg(uploadedSize);
104 } else {
105 partData = chunks.takeFirst();
106 // Build range header from saved upload size and new
107 QString tempRangeHeader = QStringLiteral("bytes %1-%2/%3").arg(uploadedSize).arg(uploadedSize + partData.size() - 1);
108 if (lastChunk) {
109 // Need to send last chunk, therefore final file size is known now
110 tempRangeHeader = tempRangeHeader.arg(uploadedSize + partData.size());
111 } else {
112 // Use star in the case that total upload size in unknown
113 QString totalSymbol = isTotalSizeKnown() ? QString::number(totalUploadSize) : QStringLiteral("*");
114 tempRangeHeader = tempRangeHeader.arg(totalSymbol);
115 }
116 rangeHeader = tempRangeHeader;
117 }
118
119 qCDebug(KGAPIDebug) << "Sending chunk of" << partData.size() << "bytes with Content-Range header" << rangeHeader;
120
121 QUrl url = QUrl(sessionPath);
122 QNetworkRequest request(url);
123 request.setRawHeader(QByteArray("Content-Range"), rangeHeader.toUtf8());
124 request.setHeader(QNetworkRequest::ContentLengthHeader, partData.length());
125 q->enqueueRequest(request, partData);
126 uploadedSize += partData.size();
127}
128
129void FileAbstractResumableJob::Private::processNext()
130{
131 qCDebug(KGAPIDebug) << "Processing next";
132
133 switch (sessionState) {
134 case ReadyStart:
135 startUploadSession();
136 return;
137 case Started: {
138 if (chunks.isEmpty() || chunks.first().size() < ChunkSize) {
139 qCDebug(KGAPIDebug) << "Chunks empty or not big enough to process, asking for more";
140
141 if (device) {
142 readFromDevice();
143 } else {
144 // Warning: an endless loop could be started here if the signal receiver isn't using
145 // a direct connection.
146 q->emitReadyWrite();
147 }
148 processNext();
149 return;
150 }
151 uploadChunk(false);
152 return;
153 }
154 case ClientEnough: {
155 uploadChunk(true);
156 sessionState = Completed;
157 return;
158 }
159 case Completed:
160 qCDebug(KGAPIDebug) << "Nothing left to process, done";
161 q->emitFinished();
162 return;
163 }
164}
165
166void KGAPI2::Drive::FileAbstractResumableJob::Private::readFromDevice()
167{
168 char buf[ChunkSize];
169 int read = device->read(buf, ChunkSize);
170 if (read == -1) {
171 qCWarning(KGAPIDebug) << "Failed reading from device" << device->errorString();
172 return;
173 }
174 qCDebug(KGAPIDebug) << "Read from device bytes" << read;
175 q->write(QByteArray(buf, read));
176}
177
178bool FileAbstractResumableJob::Private::isTotalSizeKnown() const
179{
180 return totalUploadSize != 0;
181}
182
183void FileAbstractResumableJob::Private::_k_uploadProgress(qint64 bytesSent, qint64 totalBytes)
184{
185 // uploadedSize corresponds to total bytes enqueued (including current chunk upload)
186 qint64 totalUploaded = uploadedSize - totalBytes + bytesSent;
187 q->emitProgress(totalUploaded, totalUploadSize);
188}
189
191 : FileAbstractDataJob(account, parent)
192 , d(new Private(this))
193{
194}
195
197 : FileAbstractDataJob(account, parent)
198 , d(new Private(this))
199{
200 d->metaData = metadata;
201}
202
204 : FileAbstractDataJob(account, parent)
205 , d(new Private(this))
206{
207 d->device = device;
208}
209
211 : FileAbstractDataJob(account, parent)
212 , d(new Private(this))
213{
214 d->device = device;
215 d->metaData = metadata;
216}
217
219
221{
222 return d->metaData;
223}
224
226{
227 if (isRunning()) {
228 qCWarning(KGAPIDebug) << "Can't set upload size when the job is already running";
229 return;
230 }
231
232 d->totalUploadSize = size;
233}
234
236{
237 qCDebug(KGAPIDebug) << "Received" << data.size() << "bytes to upload";
238
239 if (data.isEmpty()) {
240 qCDebug(KGAPIDebug) << "Data empty, won't receive any more data from client";
241 d->sessionState = Private::ClientEnough;
242 return;
243 }
244
245 int pos = 0;
246 // Might need to add to last chunk
247 if (!d->chunks.isEmpty() && d->chunks.last().size() < ChunkSize) {
248 QByteArray lastChunk = d->chunks.takeLast();
249 int missing = ChunkSize - lastChunk.size();
250 qCDebug(KGAPIDebug) << "Previous last chunk was" << lastChunk.size() << "bytes and could use" << missing << "bytes more, adding to it";
251 lastChunk.append(data.mid(0, missing));
252 pos = missing;
253 d->chunks << lastChunk;
254 }
255
256 int dataSize = data.size();
257 QList<QByteArray> chunks;
258 while (pos < dataSize) {
259 QByteArray chunk = data.mid(pos, ChunkSize);
260 chunks << chunk;
261 pos += chunk.size();
262 }
263
264 qCDebug(KGAPIDebug) << "Added" << chunks.size() << "new chunks";
265 d->chunks << chunks;
266}
267
269{
270 if (d->device) {
271 d->readFromDevice();
272 }
273 // Ask for more chunks right away in case
274 // write() wasn't called before starting
275 if (d->chunks.isEmpty()) {
277 }
278 d->processNext();
279}
280
282 const QNetworkRequest &request,
283 const QByteArray &data,
284 const QString &contentType)
285{
286 Q_UNUSED(contentType)
287
288 QNetworkReply *reply;
289 if (d->sessionState == Private::ReadyStart) {
290 reply = accessManager->post(request, data);
291 } else {
292 reply = accessManager->put(request, data);
293 }
294
295 if (d->isTotalSizeKnown()) {
296 connect(reply, &QNetworkReply::uploadProgress, this, [this](qint64 bytesSent, qint64 totalBytes) {
297 d->_k_uploadProgress(bytesSent, totalBytes);
298 });
299 }
300}
301
303{
304 Q_UNUSED(rawData)
305
307
308 switch (d->sessionState) {
309 case Private::ReadyStart: {
310 if (replyCode != KGAPI2::OK) {
311 qCWarning(KGAPIDebug) << "Failed opening upload session" << replyCode;
313 setErrorString(tr("Failed opening upload session"));
314 emitFinished();
315 return;
316 }
317
318 const QString uploadLocation = reply->header(QNetworkRequest::LocationHeader).toString();
319 qCDebug(KGAPIDebug) << "Got upload session location" << uploadLocation;
320 d->sessionPath = uploadLocation;
321 d->sessionState = Private::Started;
322 break;
323 }
324 case Private::Started: {
325 // If during upload total size is declared via Content-Range header, Google will
326 // respond with 200 on the last chunk upload. The job is complete in that case.
327 if (d->isTotalSizeKnown() && replyCode == KGAPI2::OK) {
328 d->sessionState = Private::Completed;
329 const QString contentType = reply->header(QNetworkRequest::ContentTypeHeader).toString();
330 ContentType ct = Utils::stringToContentType(contentType);
331 if (ct == KGAPI2::JSON) {
332 d->metaData = File::fromJSON(rawData);
333 }
334 return;
335 }
336
337 // Google will continue answering ResumeIncomplete until the total upload size is declared
338 // in the Content-Range header or until last upload range not total upload size.
339 if (replyCode != KGAPI2::ResumeIncomplete) {
340 qCWarning(KGAPIDebug) << "Failed uploading chunk" << replyCode;
342 setErrorString(tr("Failed uploading chunk"));
343 emitFinished();
344 return;
345 }
346
347 // Server could send us a new upload session location any time, use it if present
348 const QString newUploadLocation = reply->header(QNetworkRequest::LocationHeader).toString();
349 if (!newUploadLocation.isEmpty()) {
350 qCDebug(KGAPIDebug) << "Got new location" << newUploadLocation;
351 d->sessionPath = newUploadLocation;
352 }
353
354 const QString readRange = QString::fromUtf8(reply->rawHeader(QStringLiteral("Range").toUtf8()));
355 qCDebug(KGAPIDebug) << "Server confirms range" << readRange;
356 break;
357 }
358 case Private::ClientEnough:
359 case Private::Completed:
360 if (replyCode != KGAPI2::OK) {
361 qCWarning(KGAPIDebug) << "Failed completing upload session" << replyCode;
363 setErrorString(tr("Failed completing upload session"));
364 emitFinished();
365 return;
366 }
367 const QString contentType = reply->header(QNetworkRequest::ContentTypeHeader).toString();
368 ContentType ct = Utils::stringToContentType(contentType);
369 if (ct == KGAPI2::JSON) {
370 d->metaData = File::fromJSON(rawData);
371 }
372 break;
373 }
374
375 d->processNext();
376}
377
382
383#include "moc_fileabstractresumablejob.cpp"
Abstract superclass for KGAPI2::Drive::File create or modify jobs that use chunked uploading of the f...
void write(const QByteArray &data)
This function writes all the bytes in data to the upload session.
void readyWrite(KGAPI2::Drive::FileAbstractResumableJob *job)
Emitted when job requires more data to proceed.
void handleReply(const QNetworkReply *reply, const QByteArray &rawData) override
KGAPI2::Job::handleReply implementation.
~FileAbstractResumableJob() override
Destructor.
FilePtr metadata() const
Returns metadata supplied at Job creation or retrieved on Job completion.
virtual QUrl createUrl()=0
Generates url that will be used during upload session start.
FileAbstractResumableJob(const AccountPtr &account, QObject *parent=nullptr)
Constructs a job that will upload an Untitled file in the users root folder.
void dispatchRequest(QNetworkAccessManager *accessManager, const QNetworkRequest &request, const QByteArray &data, const QString &contentType) override
KGAPI2::Job::dispatchRequest implementation.
void setUploadSize(int size)
Sets the total upload size and is required for progress reporting via the Job::progress() signal.
void start() override
KGAPI2::Job::start implementation.
void setErrorString(const QString &errorString)
Set job error description to errorString.
Definition job.cpp:401
AccountPtr account() const
Returns account used to authenticate requests.
Definition job.cpp:436
virtual void emitFinished()
Emits Job::finished() signal.
Definition job.cpp:493
void setError(KGAPI2::Error error)
Set job error to error.
Definition job.cpp:386
virtual void enqueueRequest(const QNetworkRequest &request, const QByteArray &data=QByteArray(), const QString &contentType=QString())
Enqueues request in dispatcher queue.
Definition job.cpp:513
bool isRunning
Whether the job is running.
Definition job.h:67
std::optional< QSqlQuery > query(const QString &queryStatement)
A job to fetch a single map tile described by a StaticMapUrl.
Definition blog.h:16
@ ResumeIncomplete
Drive Api returns 308 when accepting a partial file upload.
Definition types.h:193
@ UnknownError
LibKGAPI error - a general unidentified error.
Definition types.h:179
@ OK
Request successfully executed.
Definition types.h:190
ContentType
Definition types.h:210
QVariant read(const QByteArray &data, int versionOverride=0)
QByteArray & append(QByteArrayView data)
bool isEmpty() const const
qsizetype length() const const
QByteArray mid(qsizetype pos, qsizetype len) const const
qsizetype size() const const
T & first()
bool isEmpty() const const
qsizetype size() const const
QMimeType mimeTypeForFileNameAndData(const QString &fileName, QIODevice *device) const const
QNetworkReply * post(const QNetworkRequest &request, QHttpMultiPart *multiPart)
QNetworkReply * put(const QNetworkRequest &request, QHttpMultiPart *multiPart)
QVariant attribute(QNetworkRequest::Attribute code) const const
QVariant header(QNetworkRequest::KnownHeaders header) const const
QByteArray rawHeader(const QByteArray &headerName) const const
void uploadProgress(qint64 bytesSent, qint64 bytesTotal)
Q_EMITQ_EMIT
QMetaObject::Connection connect(const QObject *sender, PointerToMemberFunction signal, Functor functor)
QObject * parent() const const
QString tr(const char *sourceText, const char *disambiguation, int n)
bool isNull() const const
QString arg(Args &&... args) const const
QString fromUtf8(QByteArrayView str)
bool isEmpty() const const
QString number(double n, char format, int precision)
QByteArray toUtf8() const const
void setQuery(const QString &query, ParsingMode mode)
int toInt(bool *ok) const const
QString toString() const const
This file is part of the KDE documentation.
Documentation copyright © 1996-2024 The KDE developers.
Generated on Sat Dec 21 2024 17:00:12 by doxygen 1.12.0 written by Dimitri van Heesch, © 1997-2006

KDE's Doxygen guidelines are available online.