Line data Source code
1 : import 'dart:async';
2 : import 'dart:convert';
3 : import 'dart:io';
4 : import 'dart:isolate';
5 :
6 : import 'package:at_client/at_client.dart';
7 : import 'package:at_client/src/manager/sync_isolate_manager.dart';
8 : import 'package:at_client/src/response/json_utils.dart';
9 : import 'package:at_client/src/service/sync_service.dart';
10 : import 'package:at_client/src/util/sync_util.dart';
11 : import 'package:at_commons/at_builders.dart';
12 : import 'package:at_commons/at_commons.dart';
13 : import 'package:at_lookup/at_lookup.dart';
14 : import 'package:at_persistence_secondary_server/at_persistence_secondary_server.dart';
15 : import 'package:at_utils/at_logger.dart';
16 : import 'package:cron/cron.dart';
17 :
18 : /// [Deprecate] Use [SyncService]
19 : @Deprecated("Use SyncService.Sync")
20 : class SyncManager {
21 : var logger = AtSignLogger('SyncManager');
22 :
23 : LocalSecondary? _localSecondary;
24 :
25 : RemoteSecondary? _remoteSecondary;
26 :
27 : AtClientPreference? _preference;
28 :
29 : String? _atSign;
30 :
31 : bool isSyncInProgress = false;
32 :
33 : bool pendingSyncExists = false;
34 :
35 : var _isScheduled = false;
36 :
37 0 : SyncManager(this._atSign);
38 :
39 0 : void init(String atSign, AtClientPreference preference,
40 : RemoteSecondary? _remoteSecondary, LocalSecondary? _localSecondary) {
41 0 : _atSign = atSign;
42 0 : _preference = preference;
43 0 : this._localSecondary = _localSecondary;
44 0 : this._remoteSecondary = RemoteSecondary(atSign, _preference!,
45 0 : privateKey: _preference!.privateKey);
46 0 : if (preference.syncStrategy == SyncStrategy.scheduled && !_isScheduled) {
47 0 : _scheduleSyncTask();
48 : }
49 : }
50 :
51 : @Deprecated("Use SyncService.isInSync")
52 0 : Future<bool> isInSync() async {
53 0 : var serverCommitId = await SyncUtil.getLatestServerCommitId(
54 0 : _remoteSecondary!, _preference!.syncRegex);
55 0 : var lastSyncedEntry = await SyncUtil.getLastSyncedEntry(
56 0 : _preference!.syncRegex,
57 0 : atSign: _atSign!);
58 0 : var lastSyncedCommitId = lastSyncedEntry?.commitId;
59 0 : var lastSyncedLocalSeq = lastSyncedEntry != null ? lastSyncedEntry.key : -1;
60 0 : var unCommittedEntries = await SyncUtil.getChangesSinceLastCommit(
61 0 : lastSyncedLocalSeq, _preference!.syncRegex,
62 0 : atSign: _atSign!);
63 0 : return SyncUtil.isInSync(
64 : unCommittedEntries, serverCommitId, lastSyncedCommitId);
65 : }
66 :
67 : /// Cloud Secondary server throws [BufferOverFlowException] is sync data is large than the buffer size.
68 : /// Optionally isStream when set to true, initiates the sync process via streams which facilitates in
69 : /// syncing large data without [BufferOverFlowException].
70 : /// [Deprecated] Use [SyncService]
71 : @Deprecated("Use SyncService.sync")
72 0 : Future<void> sync({bool appInit = false, String? regex}) async {
73 : //initially isSyncInProgress and pendingSyncExists are false.
74 : //If a new sync triggered while previous sync isInprogress,then pendingSyncExists set to true and returns.
75 0 : if (isSyncInProgress) {
76 0 : pendingSyncExists = true;
77 : return;
78 : }
79 0 : regex ??= _preference!.syncRegex;
80 0 : await _sync(appInit: appInit, regex: regex);
81 : //once the sync done, we will check for any new sync requests(pendingSyncExists == true)
82 : //If pendingSyncExists is true,then sync triggers again.
83 0 : if (pendingSyncExists) {
84 0 : pendingSyncExists = false;
85 0 : return sync(appInit: appInit, regex: regex);
86 : }
87 : return;
88 : }
89 :
90 0 : Future<void> _sync({bool appInit = false, String? regex}) async {
91 : try {
92 0 : regex ??= _preference!.syncRegex;
93 : //isSyncProgress set to true during the sync is in progress.
94 : //once sync process done, it will again set to false.
95 0 : isSyncInProgress = true;
96 : var lastSyncedEntry =
97 0 : await SyncUtil.getLastSyncedEntry(regex, atSign: _atSign!);
98 0 : var lastSyncedCommitId = lastSyncedEntry?.commitId;
99 : var serverCommitId =
100 0 : await SyncUtil.getLatestServerCommitId(_remoteSecondary!, regex);
101 : var lastSyncedLocalSeq =
102 0 : lastSyncedEntry != null ? lastSyncedEntry.key : -1;
103 0 : if (appInit && lastSyncedLocalSeq > 0) {
104 0 : serverCommitId ??= -1;
105 0 : if (lastSyncedLocalSeq > serverCommitId) {
106 : lastSyncedLocalSeq = serverCommitId;
107 : }
108 0 : logger.finer('app init: lastSyncedLocalSeq: $lastSyncedLocalSeq');
109 : }
110 0 : var unCommittedEntries = await SyncUtil.getChangesSinceLastCommit(
111 : lastSyncedLocalSeq, regex,
112 0 : atSign: _atSign!);
113 : // cloud and local are in sync if there is no synced changes in local and commitIDs are equals
114 0 : if (SyncUtil.isInSync(
115 : unCommittedEntries, serverCommitId, lastSyncedCommitId)) {
116 0 : logger.info('server and local is in sync');
117 : return;
118 : }
119 0 : lastSyncedCommitId ??= -1;
120 0 : serverCommitId ??= -1;
121 : // cloud is ahead if server commit id is > last synced commit id in local
122 0 : if (serverCommitId > lastSyncedCommitId) {
123 : // Iterates until serverCommitId is greater than localCommitId are equal.
124 0 : while (serverCommitId > lastSyncedCommitId!) {
125 0 : var syncBuilder = SyncVerbBuilder()
126 0 : ..commitId = lastSyncedCommitId
127 0 : ..regex = regex;
128 0 : var syncResponse = await _remoteSecondary!.executeVerb(syncBuilder);
129 0 : if (syncResponse.isNotEmpty && syncResponse != 'data:null') {
130 0 : syncResponse = syncResponse.replaceFirst('data:', '');
131 0 : var syncResponseJson = JsonUtils.decodeJson(syncResponse);
132 : // Iterates over each commit
133 0 : await Future.forEach(syncResponseJson,
134 0 : (dynamic serverCommitEntry) => _syncLocal(serverCommitEntry));
135 : }
136 : // assigning the lastSynced local commit id.
137 : var lastSyncedEntry =
138 0 : await SyncUtil.getLastSyncedEntry(regex, atSign: _atSign!);
139 0 : lastSyncedCommitId = lastSyncedEntry?.commitId;
140 : }
141 : return;
142 : }
143 :
144 : // local is ahead. push the changes to secondary server
145 0 : var uncommittedEntryBatch = _getUnCommittedEntryBatch(unCommittedEntries);
146 0 : for (var unCommittedEntryList in uncommittedEntryBatch) {
147 : try {
148 0 : var batchRequests = await _getBatchRequests(unCommittedEntryList);
149 0 : var batchResponse = await _sendBatch(batchRequests);
150 0 : for (var entry in batchResponse) {
151 : try {
152 0 : var batchId = entry['id'];
153 0 : var serverResponse = entry['response'];
154 0 : var responseObject = Response.fromJson(serverResponse);
155 0 : var commitId = -1;
156 0 : if (responseObject.data != null) {
157 0 : commitId = int.parse(responseObject.data!);
158 : }
159 0 : var commitEntry = unCommittedEntryList.elementAt(batchId - 1);
160 0 : if (commitId == -1) {
161 0 : logger.severe(
162 0 : 'update/delete for key ${commitEntry.atKey} failed. Error code ${responseObject.errorCode} error message ${responseObject.errorMessage}');
163 : }
164 :
165 0 : logger.finer('***batchId:$batchId key: ${commitEntry.atKey}');
166 0 : await SyncUtil.updateCommitEntry(commitEntry, commitId, _atSign!);
167 0 : } on Exception catch (e) {
168 0 : logger.severe(
169 0 : 'exception while updating commit entry for entry:$entry ${e.toString()}');
170 : }
171 : }
172 0 : } on Exception catch (e) {
173 0 : logger.severe(
174 0 : 'exception while syncing batch: ${e.toString()} batch commit entries: $unCommittedEntryList');
175 : }
176 : }
177 0 : } on AtLookUpException catch (e) {
178 0 : if (e.errorCode == 'AT0021') {
179 0 : logger.info('skipping sync since secondary is not reachable');
180 : }
181 : } finally {
182 0 : isSyncInProgress = false;
183 : }
184 : }
185 :
186 0 : Future<void> syncWithIsolate() async {
187 0 : var lastSyncedEntry = await SyncUtil.getLastSyncedEntry(
188 0 : _preference!.syncRegex,
189 0 : atSign: _atSign!);
190 0 : var lastSyncedCommitId = lastSyncedEntry?.commitId;
191 0 : var commitIdReceivePort = ReceivePort();
192 0 : var privateKey = await _localSecondary!.keyStore!.get(AT_PKAM_PRIVATE_KEY);
193 0 : var isolate = await Isolate.spawn(
194 : SyncIsolateManager.executeRemoteCommandIsolate,
195 0 : commitIdReceivePort.sendPort);
196 : var syncDone = false;
197 : dynamic syncSendPort;
198 : dynamic pushedCount;
199 0 : commitIdReceivePort.listen((message) async {
200 0 : if (syncSendPort == null && message is SendPort) {
201 : //1. Request to isolate to get latest server commit id from server
202 : syncSendPort = message;
203 0 : logger.info('sending:');
204 0 : var isolateInput = <String, dynamic>{};
205 0 : isolateInput['operation'] = 'get_commit_id';
206 0 : isolateInput['atsign'] = _atSign;
207 0 : isolateInput['preference'] = _preference;
208 0 : isolateInput['private_key'] = privateKey?.data;
209 0 : syncSendPort.send(isolateInput);
210 : } else {
211 0 : logger.info('received server commit id from isolate: $message');
212 0 : var operation = message['operation'];
213 : switch (operation) {
214 0 : case 'get_commit_id_result':
215 : // 1.1 commit id response from isolate
216 0 : var serverCommitId = message['commit_id'];
217 : var lastSyncedLocalSeq =
218 0 : lastSyncedEntry != null ? lastSyncedEntry.key : -1;
219 0 : var unCommittedEntries = await SyncUtil.getChangesSinceLastCommit(
220 0 : lastSyncedLocalSeq, _preference!.syncRegex!,
221 0 : atSign: _atSign!);
222 0 : if (SyncUtil.isInSync(
223 : unCommittedEntries, serverCommitId, lastSyncedCommitId)) {
224 0 : logger.info('server and local is in sync');
225 : syncDone = true;
226 : }
227 0 : lastSyncedCommitId ??= -1;
228 0 : serverCommitId ??= -1;
229 0 : var isolateInput = <String, dynamic>{};
230 0 : isolateInput['atsign'] = _atSign;
231 0 : isolateInput['preference'] = _preference;
232 0 : if (serverCommitId > lastSyncedCommitId) {
233 : //2. server is ahead
234 : //2.1 Send last synced id to isolate to get latest changes from server
235 0 : isolateInput['operation'] = 'get_server_commits';
236 0 : isolateInput['last_synced_commit_id'] = lastSyncedCommitId;
237 0 : syncSendPort.send(isolateInput);
238 : } else {
239 : //3. local is ahead
240 : //3.1 For each uncommitted entry send request to isolate to send update/delete to server
241 0 : pushedCount = unCommittedEntries.length;
242 0 : for (var entry in unCommittedEntries) {
243 0 : var command = await _getCommand(entry);
244 0 : logger.info('command:$command');
245 : dynamic builder;
246 0 : switch (entry.operation) {
247 0 : case CommitOp.UPDATE:
248 0 : builder = UpdateVerbBuilder.getBuilder(command);
249 : break;
250 0 : case CommitOp.DELETE:
251 0 : builder = DeleteVerbBuilder.getBuilder(command);
252 : break;
253 0 : case CommitOp.UPDATE_META:
254 0 : builder = UpdateVerbBuilder.getBuilder(command);
255 : break;
256 0 : case CommitOp.UPDATE_ALL:
257 0 : builder = UpdateVerbBuilder.getBuilder(command);
258 : break;
259 : default:
260 : break;
261 : }
262 0 : isolateInput['operation'] = 'push_to_remote';
263 0 : isolateInput['builder'] = builder;
264 0 : isolateInput['entry_key'] = entry.key;
265 0 : syncSendPort.send(isolateInput);
266 0 : sleep(Duration(
267 : seconds:
268 : 1)); // workaround for receiving out of order response from message listener
269 : }
270 : }
271 : break;
272 0 : case 'get_server_commits_result':
273 : //2.2 Sync verb response from isolate. For each entry sync to local storage and update commit id.
274 0 : var syncResponse = message['sync_response'];
275 0 : var syncResponseJson = jsonDecode(syncResponse);
276 0 : await Future.forEach(syncResponseJson,
277 0 : (dynamic serverCommitEntry) => _syncLocal(serverCommitEntry));
278 : syncDone = true;
279 : break;
280 0 : case 'push_to_remote_result':
281 : // 3.2 Update/delete verb commit id response from server. Update server commit id in local commit log.
282 0 : var serverCommitId = message['operation_commit_id'];
283 0 : dynamic entryKey = message['entry_key'];
284 0 : var entry = SyncUtil.getEntry(entryKey, _atSign!);
285 0 : logger.info(
286 0 : 'received remote push result: $entryKey $entry $entryKey');
287 0 : await SyncUtil.updateCommitEntry(
288 0 : entry, int.parse(serverCommitId), _atSign!);
289 0 : pushedCount--;
290 0 : print('pushedCount:$pushedCount');
291 0 : if (pushedCount == 0) syncDone = true;
292 : break;
293 : }
294 : if (syncDone) {
295 : // 2.3 server ahead sync done
296 : // 3.3 local ahead sync done
297 0 : isolate.kill(priority: Isolate.immediate);
298 0 : logger.info('isolate sync complete');
299 : return;
300 : }
301 : }
302 : });
303 : }
304 :
305 0 : dynamic _sendBatch(List<BatchRequest> requests) async {
306 : var command = 'batch:';
307 0 : command += jsonEncode(requests);
308 0 : command += '\n';
309 : var verbResult =
310 0 : await _remoteSecondary!.executeCommand(command, auth: true);
311 0 : logger.finer('batch result:$verbResult');
312 : if (verbResult != null) {
313 0 : verbResult = verbResult.replaceFirst('data:', '');
314 : }
315 0 : return jsonDecode(verbResult!);
316 : }
317 :
318 0 : Future<void> _syncLocal(serverCommitEntry) async {
319 0 : switch (serverCommitEntry['operation']) {
320 0 : case '+':
321 0 : case '#':
322 0 : case '*':
323 0 : var builder = UpdateVerbBuilder()
324 0 : ..atKey = serverCommitEntry['atKey']
325 0 : ..value = serverCommitEntry['value'];
326 0 : builder.operation = UPDATE_ALL;
327 0 : _setMetaData(builder, serverCommitEntry);
328 0 : await _pullToLocal(builder, serverCommitEntry, CommitOp.UPDATE_ALL);
329 : break;
330 0 : case '-':
331 0 : var builder = DeleteVerbBuilder()..atKey = serverCommitEntry['atKey'];
332 0 : await _pullToLocal(builder, serverCommitEntry, CommitOp.DELETE);
333 : break;
334 : }
335 : }
336 :
337 0 : void _setMetaData(builder, serverCommitEntry) {
338 0 : var metaData = serverCommitEntry['metadata'];
339 0 : if (metaData != null && metaData.isNotEmpty) {
340 0 : if (metaData[AT_TTL] != null) builder.ttl = int.parse(metaData[AT_TTL]);
341 0 : if (metaData[AT_TTB] != null) builder.ttb = int.parse(metaData[AT_TTB]);
342 0 : if (metaData[AT_TTR] != null) builder.ttr = int.parse(metaData[AT_TTR]);
343 0 : if (metaData[CCD] != null) {
344 0 : (metaData[CCD].toLowerCase() == 'true')
345 0 : ? builder.ccd = true
346 0 : : builder.ccd = false;
347 : }
348 0 : if (metaData[PUBLIC_DATA_SIGNATURE] != null) {
349 0 : builder.dataSignature = metaData[PUBLIC_DATA_SIGNATURE];
350 : }
351 0 : if (metaData[IS_BINARY] != null) {
352 0 : (metaData[IS_BINARY].toLowerCase() == 'true')
353 0 : ? builder.isBinary = true
354 0 : : builder.isBinary = false;
355 : }
356 0 : if (metaData[IS_ENCRYPTED] != null) {
357 0 : (metaData[IS_ENCRYPTED].toLowerCase() == 'true')
358 0 : ? builder.isEncrypted = true
359 0 : : builder.isEncrypted = false;
360 : }
361 : }
362 : }
363 :
364 0 : Future<void> _pullToLocal(
365 : VerbBuilder builder, serverCommitEntry, CommitOp operation) async {
366 0 : var verbResult = await _localSecondary!.executeVerb(builder, sync: false);
367 : if (verbResult == null) {
368 : return;
369 : }
370 0 : var sequenceNumber = int.parse(verbResult.split(':')[1]);
371 0 : var commitEntry = await (SyncUtil.getCommitEntry(sequenceNumber, _atSign!));
372 : if (commitEntry == null) {
373 : return;
374 : }
375 0 : commitEntry.operation = operation;
376 0 : await SyncUtil.updateCommitEntry(
377 0 : commitEntry, serverCommitEntry['commitId'], _atSign!);
378 : }
379 :
380 0 : Future<void> syncImmediate(
381 : String localSequence, VerbBuilder builder, CommitOp? operation) async {
382 : try {
383 0 : var verbResult = await _remoteSecondary!.executeVerb(builder);
384 0 : var serverCommitId = verbResult.split(':')[1];
385 : var localCommitEntry =
386 0 : await (SyncUtil.getCommitEntry(int.parse(localSequence), _atSign!));
387 : if (localCommitEntry == null) {
388 : return;
389 : }
390 0 : localCommitEntry.operation = operation;
391 0 : await SyncUtil.updateCommitEntry(
392 0 : localCommitEntry, int.parse(serverCommitId), _atSign!);
393 0 : } on SecondaryConnectException {
394 0 : logger.severe('Unable to connect to secondary');
395 : }
396 : }
397 :
398 0 : Future<String> _getCommand(CommitEntry entry) async {
399 : late String command;
400 : // ignore: missing_enum_constant_in_switch
401 0 : switch (entry.operation) {
402 0 : case CommitOp.UPDATE:
403 0 : var key = entry.atKey;
404 0 : var value = await _localSecondary!.keyStore!.get(key);
405 0 : command = 'update:$key ${value?.data}';
406 : break;
407 0 : case CommitOp.DELETE:
408 0 : var key = entry.atKey;
409 0 : command = 'delete:$key';
410 : break;
411 0 : case CommitOp.UPDATE_META:
412 0 : var key = entry.atKey;
413 0 : var metaData = await _localSecondary!.keyStore!.getMeta(key);
414 : if (metaData != null) {
415 0 : key = '$key$_metadataToString(metaData)';
416 : }
417 0 : command = 'update:meta:$key';
418 : break;
419 0 : case CommitOp.UPDATE_ALL:
420 0 : var key = entry.atKey;
421 0 : var value = await _localSecondary!.keyStore!.get(key);
422 0 : var metaData = await _localSecondary!.keyStore!.getMeta(key);
423 : var keyGen = '';
424 : if (metaData != null) {
425 0 : keyGen = _metadataToString(metaData);
426 : }
427 0 : keyGen += ':$key';
428 0 : value?.metaData = metaData;
429 0 : command = 'update$keyGen ${value?.data}';
430 : break;
431 : }
432 0 : return command;
433 : }
434 :
435 0 : String _metadataToString(dynamic metadata) {
436 : var metadataStr = '';
437 0 : if (metadata.ttl != null) metadataStr += ':ttl:${metadata.ttl}';
438 0 : if (metadata.ttb != null) metadataStr += ':ttb:${metadata.ttb}';
439 0 : if (metadata.ttr != null) metadataStr += ':ttr:${metadata.ttr}';
440 0 : if (metadata.isCascade != null) {
441 0 : metadataStr += ':ccd:${metadata.isCascade}';
442 : }
443 0 : if (metadata.dataSignature != null) {
444 0 : metadataStr += ':dataSignature:${metadata.dataSignature}';
445 : }
446 0 : if (metadata.isBinary != null) {
447 0 : metadataStr += ':isBinary:${metadata.isBinary}';
448 : }
449 0 : if (metadata.isEncrypted != null) {
450 0 : metadataStr += ':isEncrypted:${metadata.isEncrypted}';
451 : }
452 : return metadataStr;
453 : }
454 :
455 0 : List<dynamic> _getUnCommittedEntryBatch(
456 : List<CommitEntry?> uncommittedEntries) {
457 0 : var unCommittedEntryBatch = [];
458 0 : var batchSize = _preference!.syncBatchSize, i = 0;
459 0 : var totalEntries = uncommittedEntries.length;
460 0 : var totalBatch = (totalEntries % batchSize == 0)
461 0 : ? totalEntries / batchSize
462 0 : : (totalEntries / batchSize).floor() + 1;
463 : var startIndex = i;
464 0 : while (i < totalBatch) {
465 0 : var endIndex = startIndex + batchSize < totalEntries
466 0 : ? startIndex + batchSize
467 : : totalEntries;
468 0 : var currentBatch = uncommittedEntries.sublist(startIndex, endIndex);
469 0 : unCommittedEntryBatch.add(currentBatch);
470 0 : startIndex += batchSize;
471 0 : i++;
472 : }
473 : return unCommittedEntryBatch;
474 : }
475 :
476 0 : Future<List<BatchRequest>> _getBatchRequests(
477 : List<CommitEntry> uncommittedEntries) async {
478 0 : var batchRequests = <BatchRequest>[];
479 : var batchId = 1;
480 0 : for (var entry in uncommittedEntries) {
481 0 : var command = await _getCommand(entry);
482 0 : command = command.replaceAll('cached:', '');
483 0 : command = VerbUtil.replaceNewline(command);
484 0 : var batchRequest = BatchRequest(batchId, command);
485 0 : logger.finer('batchId:$batchId key:${entry.atKey}');
486 0 : batchRequests.add(batchRequest);
487 0 : batchId++;
488 : }
489 : return batchRequests;
490 : }
491 :
492 0 : void _scheduleSyncTask() {
493 : try {
494 0 : var cron = Cron();
495 0 : cron.schedule(
496 0 : Schedule.parse('*/${_preference!.syncIntervalMins} * * * *'),
497 0 : () async {
498 0 : await syncWithIsolate();
499 : });
500 0 : _isScheduled = true;
501 0 : } on Exception catch (e) {
502 0 : print('Exception during scheduleSyncTask ${e.toString()}');
503 : }
504 : }
505 : }
|