Compare commits

...

3142 Commits

Author SHA1 Message Date
Ice
6f628be053 Update pom.xml - Deps
Corrections for NTP slipage at start up
2025-07-20 03:18:52 -04:00
Ice
eb07c45955 Merge pull request #255 from IceBurst/master
* Abstraction of AltCoinJ 
* Abstraction of CIYAM
* Update to BouncyCastle
2025-07-13 14:08:15 -04:00
Ice
8bea11bc52 Merge branch 'master' into master 2025-07-13 14:06:11 -04:00
Qortal-Auto-Update
415f594b25 Bump version to 5.0.2 2025-07-12 15:46:54 -07:00
crowetic
1e593cdf13 Merge pull request #263 from crowetic/master
updated minPeerVersion to 5.0.0 and removed duplicate entry in pom
2025-07-12 15:43:45 -07:00
71d2fbe0b6 updated minPeerVersion to 5.0.0 and removed duplicate entry in pom 2025-07-12 15:42:26 -07:00
crowetic
5a760db37d Merge pull request #262 from kennycud/master
Full Send 

Tested and ready
2025-07-12 15:30:11 -07:00
kennycud
05d629e717 removed logging spam 2025-07-12 14:03:35 -07:00
kennycud
cea63e7ec7 PeerSendManagement support for sending all messages through a queue 2025-07-12 14:02:19 -07:00
Qortal-Auto-Update
5fabc7792c Bump version to 5.0.1 2025-07-10 13:56:06 -07:00
crowetic
09d0af9b78 Merge pull request #260 from kennycud/master
Promising QDN Improvements
2025-07-10 13:51:35 -07:00
crowetic
698e616bc9 Merge pull request #261 from crowetic/master
added new auto-update scripts
2025-07-10 13:50:21 -07:00
6c0a9b3539 added new auto-update scripts 2025-07-10 13:47:10 -07:00
kennycud
60811f9f65 log spam reduction 2025-07-10 13:38:02 -07:00
kennycud
d91a777ffd Delete qortal.log 2025-07-10 13:32:30 -07:00
kennycud
c19cad020e Merge pull request #14 from Philreact/master-11
PeerSendManager
2025-07-10 13:18:14 -07:00
52519e3662 PeerSendManagement loose-ends 2025-07-10 23:16:42 +03:00
fd62e6156c increase request timeout 2025-07-10 17:38:32 +03:00
e5890b3b6f added cooling period in case of re-connections 2025-07-10 17:38:25 +03:00
256baeb1f4 reduce interval cleanup 2025-07-10 17:37:46 +03:00
05b83ade47 remove unused code 2025-07-10 17:37:39 +03:00
f7cb4ce264 PeerSendManger added 2025-07-10 17:37:25 +03:00
086ed6574f Merge remote-tracking branch 'kenny/master' into master-10 2025-07-09 22:38:00 +03:00
kennycud
4b56690118 qdn relay optimizations 2025-07-09 12:34:47 -07:00
kennycud
44d26b513a waiting and retrying clogged write channels 2025-07-08 13:42:49 -07:00
kennycud
dbd900f74a peer fetcher executor shutdown for inactivity, thanks to philreact research, peer fetcher thread naming added 2025-07-08 05:43:30 -07:00
kennycud
38463f6b1a follower compile error fix 2025-07-07 14:51:24 -07:00
kennycud
16e48aba04 follower initial implementation 2025-07-07 14:34:55 -07:00
kennycud
56d97457a1 Merge remote-tracking branch 'origin/master' 2025-07-07 14:32:25 -07:00
kennycud
2167d2f8fe reduced logging spam 2025-07-07 14:30:45 -07:00
kennycud
8425d62673 Merge pull request #13 from Philreact/bugfix/data-renderer-name-spaces
replace name spaces with encoded space
2025-07-05 05:06:15 -07:00
4995bee3e3 replace name spaces with encoded space 2025-07-05 07:03:55 +03:00
Qortal-Auto-Update
87897d7db8 Bump version to 5.0.0 2025-07-03 16:17:36 -07:00
crowetic
49e9a53c6a Merge pull request #257 from kennycud/master
Additional Settings and Timeout threshold update
2025-07-03 15:52:11 -07:00
kennycud
b5c4599005 Merge branch 'Qortal:master' into master 2025-07-03 15:44:22 -07:00
kennycud
3aabedda92 increased additional thresholds for auto update release 2025-07-03 15:37:01 -07:00
crowetic
dd88decc40 Added FeatureTrigger block heights
Feature trigger for multipleNamesPerAccountHeight and mintedBlocksAdjustmentRemovalHeight SET. Estimated Activation time: Friday, July 4th, 2025
2025-07-03 15:26:00 -07:00
crowetic
4f3b4e4a58 Merge pull request #256 from kennycud/master
Foreign Fees Manager, Multiple Names, QDN Oprtimaizations - Tested for minimum 1 week, most longer.
2025-07-03 15:07:39 -07:00
kennycud
b2c72c3927 null pointer solution by using an empty list instead of a null value 2025-06-29 11:08:42 -07:00
kennycud
65c014b215 removed redundant data collecting, reintroduced relay timeout threshold 2025-06-27 14:13:14 -07:00
kennycud
b2579a457c reverting the GET_ARBITRARY_DATA_FILE thread limit, because it puts too much pressure on the peers with the previously lower limit, planning on updating this to a higher number right before the next release when all nodes are ready for it 2025-06-27 14:09:08 -07:00
kennycud
170668ef78 reduced logging levels on numerous messages 2025-06-27 14:04:39 -07:00
kennycud
b48b6b9d42 added test cases for single file websites 2025-06-27 14:01:51 -07:00
kennycud
22dc3e55df Merge pull request #12 from Philreact/bugfix/allow-blob-connect
Bugfix/allow blob connect
2025-06-23 12:50:32 -07:00
kennycud
66bfed93ee Merge pull request #11 from IceBurst/patch-1
Logging for Failed Respository Connections on Optional Runs
2025-06-23 12:49:40 -07:00
b8e1712881 add blob: to connect-src directive 2025-06-23 13:48:35 +03:00
6a5013d378 Merge remote-tracking branch 'kenny/master' into master-kenny3 2025-06-20 02:18:23 +03:00
kennycud
3687455c62 increasing arbitrary data message thread limits, because the algorithms can handle it 2025-06-18 17:57:18 -07:00
kennycud
60b3bacd15 reduced arbitrary data storage addition and deletion thresholds from 98% and 90% to 90% and 80% 2025-06-18 17:55:30 -07:00
kennycud
7a7f0e53ac reduced index caching errors to warnings, because it is only an error if it continually happens 2025-06-17 15:56:04 -07:00
kennycud
940c641759 removed stack trace from streaming error warnings 2025-06-17 15:10:37 -07:00
kennycud
a3bb6638bf added support for single file websites 2025-06-17 15:09:11 -07:00
kennycud
5b402e0bca validate name buyer's balance relative to the amount of the name purchase in addition to the fee 2025-06-17 15:08:20 -07:00
kennycud
89236d6504 no longer repackaging missing data exceptions as io exceptions when loading json data for indices 2025-06-14 13:11:19 -07:00
kennycud
47e313067f fixed a flaw in the blocks minted adjustment removal feature, instead of increasing or decreasing the level we need to reset the level when it is incorrect 2025-06-13 12:13:52 -07:00
Ice
92077f2912 Logging for Failed Respository Connections on Optional Runs 2025-06-11 15:45:08 -04:00
Ice
95e12395ae Merge pull request #1 from IceBurst/Abstract-and-Update-Deps
Abstract and update deps
2025-06-11 03:15:34 -04:00
Ice
47e5c473b3 Merge branch 'master' into Abstract-and-Update-Deps 2025-06-11 03:15:22 -04:00
kennycud
15f793ccb4 Merge remote-tracking branch 'origin/master' 2025-06-09 18:26:01 -07:00
kennycud
ccb59559d6 the bootstrapper was resetting the database configuration that the db cache was dependent on, so that dependency was changed 2025-06-09 18:25:43 -07:00
MergeMerc
30c5136c44 Add Logging for failing to get a Repository Connection for Non-Required/Non-Blocking Tasks 2025-06-09 13:34:05 -04:00
kennycud
91a58c50e1 Merge pull request #10 from Philreact/master-kenny3
add cleanup of leftover chunks at startup
2025-06-06 19:54:36 -07:00
f8daf50ccb Merge remote-tracking branch 'kenny/master' into master-kenny3 2025-06-07 05:43:47 +03:00
kennycud
8e0e455d41 blocks minted adjustments removal is a new feature trigger
primary names are now used throughout the chat repository

numerous message handlers have been optimized, many message handlers are now getting added to a list and scheduled for processing and when they get processed, the database gets queried significantly less, because the message requests and responses are getting batched together for database access rather than querying the database one by one, the thread limits for these message types have been significantly increased, because each individual thread coming in does very little, all it does is add the message to a list to be scheduled at a later time
2025-06-06 19:01:09 -07:00
6145db5357 add cleanup of chunks at startup 2025-06-03 03:33:35 +03:00
kennycud
7ccd06e5c3 Merge pull request #9 from Philreact/master-kenny3
fix in digest, was putting whole file in memory.
2025-06-01 10:43:35 -07:00
517f7b92d5 in memory to stream 2025-06-01 20:31:36 +03:00
kennycud
fa8b9f2cee Merge pull request #8 from Philreact/fix/load-data
fix issue of not breaking when file is complete
2025-05-28 17:21:55 -07:00
d66616f375 fix issue of not breaking when file is complete 2025-05-28 16:29:52 +03:00
kennycud
02e10e9de9 invalidated name buys and sales that violate primary names 2025-05-27 08:15:50 -07:00
kennycud
61c010754e Merge branch 'Qortal:master' into master 2025-05-25 12:20:58 -07:00
kennycud
5013c68b61 Merge pull request #7 from Philreact/feature/allow-for-unlimited-size-publishes
Feature/allow for unlimited size publishes
2025-05-25 11:45:27 -07:00
140d86e209 added comments 2025-05-24 22:29:33 +03:00
9e4925c8dd added back comments 2025-05-24 19:15:36 +03:00
kennycud
88fe3b0af6 primary names implementation 2025-05-23 17:49:26 -07:00
Ice
e6f032a2a9 Merge pull request #253 from IceBurst/IceBurst-Unit-Tests-Updates
Unit Test Updates
2025-05-19 15:34:27 -04:00
ca88cb1f88 allow downloads 2025-05-19 16:55:12 +03:00
58ab02c4f0 fix to temp dir 2025-05-18 23:21:49 +03:00
e1ea8d65f8 fix blank filename issue 2025-05-16 23:39:32 +03:00
1c52c18d32 added endpoints 2025-05-16 15:49:47 +03:00
2cd5f9e4cd change limit 2025-05-16 01:18:02 +03:00
f2b5802d9c change to streaming 2025-05-16 01:17:01 +03:00
bc4e0716db fix streaming for base64 2025-05-15 16:56:53 +03:00
994761a87e added missing requires 2025-05-15 01:20:40 +03:00
5780a6de7d remove zip best speed 2025-05-14 20:21:13 +03:00
8c811ef1ef initial 2025-05-14 20:00:04 +03:00
kennycud
f5a4a0a16c Merge remote-tracking branch 'origin/master' 2025-05-13 11:14:08 -07:00
kennycud
93dab1a3e3 detailed test case for the invite orphan vulnerability patch that was committed in 2/1/25 2025-05-13 11:13:55 -07:00
Ice
7d14d381bc Merge pull request #235 from infinitydaemon/patch-2
Update SellNameTransaction.java
2025-05-12 16:32:09 -04:00
kennycud
6511086d18 Merge pull request #6 from Philreact/master-kenny2
pass ui language to qapps
2025-05-10 12:36:57 -07:00
70ae122f5c pass ui lang to qapps 2025-05-10 22:21:13 +03:00
Ice
33475ace00 Merge pull request #236 from infinitydaemon/patch-3
Update CancelSellNameTransaction.java
2025-05-10 04:23:09 -04:00
kennycud
88d009c979 multiple registered names for single accounts API call now returns ordered by time of registration, earliest to latest 2025-05-06 15:26:24 -07:00
kennycud
26a345a909 introducing feature trigger that enables multiple registered names for single accounts 2025-05-04 11:52:09 -07:00
Ice
618945620d Abstract CIYAM.AT out of Repo 2025-04-29 07:13:34 -04:00
Ice
b6d3e407c8 Updates to Dependencies - Test Improvements 2025-04-28 07:25:58 -04:00
kennycud
4b74bb37dc unsigned fee event handling now provides address 2025-04-27 15:02:28 -07:00
kennycud
17b2bf3848 added logging and added positive boolean to the fee waiting and unsigned fee events 2025-04-26 17:53:41 -07:00
kennycud
1f6ee72fc5 the message types were corrected 2025-04-26 09:58:13 -07:00
kennycud
83bc84909a Merge branch 'master' of https://github.com/kennycud/qortal 2025-04-25 17:55:02 -07:00
kennycud
144d6cc5c7 foreign fees manager implementation, feeCeiling -> feeRequired name change, thread-safety measures for fee values, fee backup file implementation, unsigned fees socket implementation 2025-04-25 17:51:01 -07:00
crowetic
eff2e6d150 Merge pull request #249 from IceBurst/hsqldb-2.7.4-build-update
Hsqldb 2.7.4 build update
2025-04-24 15:17:40 -07:00
crowetic
c1041d2ad3 Merge pull request #192 from karl-dv/master
Some small corrections for "NL" translations
2025-04-24 14:22:04 -07:00
crowetic
699d8815c4 Merge branch 'master' into master 2025-04-24 14:21:54 -07:00
Ice
2a97fba108 Merge remote-tracking branch 'origin/IceBurst-Unit-Tests-Updates' into Abstract-and-Update-Deps 2025-04-24 03:45:38 -04:00
Ice
f1a0472c57 Corrections for Unit Tests - Lots of Corrections 2025-04-24 03:27:28 -04:00
Ice
c4d8a17355 Merge branch 'hsqldb-2.7.4-build-update' into IceBurst-Unit-Tests-Updates 2025-04-17 06:34:26 -04:00
Ice
9c1cb9da77 Update test-chain-v2-reward-levels.json
Add Missing Feature
2025-04-17 06:23:47 -04:00
Ice
7dae60d35f Update test-settings-v2-block-archive.json
Performance Improvement of 00% for block archive tests
2025-04-16 16:10:23 -04:00
Ice
8421336016 Update pr-testing.yml
-- Process 'Install' to load Deps Before testing
2025-04-16 15:11:04 -04:00
Ice
2e7cd93716 Delete .github/workflows/pr-testomg 2025-04-16 15:07:52 -04:00
Ice
2cf0aeac22 Update pr-testing.yml 2025-04-16 14:30:10 -04:00
Ice
cc4056047e Create pr-testomg 2025-04-15 15:45:00 -04:00
Ice
421e241729 Update test-chain-v2-founder-rewards.json
Correction for Test - testFounderrewards
2025-04-15 14:51:25 -04:00
Ice
c977660c47 Update Service.java
Add qortal as valid extension for QCHAT_ATTACHMENT, needed when fetching a previous TX
2025-04-15 10:40:10 -04:00
Ice
867d0e29e0 Merge branch 'Qortal:master' into IceBurst-Unit-Tests-Updates 2025-04-15 08:16:40 -04:00
Ice
57d12b4afe block-archive test performance improvement
Added parameter: "archivingPause": 5
Default Value is: 3000
2025-04-15 08:06:39 -04:00
aeab1acbbc Bump version to 4.7.1 2025-04-08 20:13:43 -07:00
crowetic
0b37666d2b Merge pull request #250 from kennycud/master
Merging current 'test release' from kennycud repo after extensive testing by community.
2025-04-08 08:34:07 -07:00
kennycud
bcf3538d18 dd cache enabled to true by default 2025-04-07 12:19:56 -07:00
kennycud
b2d9d0539e removed cache orphaning, crowetic and I agree it should have never been added to begin with 2025-04-05 11:42:21 -07:00
kennycud
1bd6076e33 forgot IndexCache.java in the last commit
replaced index service attribute with a category attribute and reduced index attribute names to single characters to reduce memory footprint, t is for term, n is for name, c is for category, l if for link

changed default indexing frequency from 1 minute to 10 minutes to reduce memory use

added arbitrary resource endpoint for index search by issuer name and index prefix

added some additional error handling concerning unrecognized properties in the indices
2025-04-03 10:23:58 -07:00
kennycud
a6309e925b replaced index service attribute with a category attribute and reduced index attribute names to single characters to reduce memory footprint, t is for term, n is for name, c is for category, l if for link
changed default indexing frequency from 1 minute to 10 minutes to reduce memory use

added arbitrary resource endpoint for index search by issuer name and index prefix

added some additional error handling concerning unrecognized properties in the indices
2025-04-03 10:18:45 -07:00
kennycud
23de8a98bc removed logging 2025-03-21 18:59:32 -07:00
kennycud
d0a85d4717 QDN bug resolution 2025-03-21 18:44:41 -07:00
kennycud
a893888a2e reduced logging level for invalid formatting 2025-03-21 18:43:06 -07:00
kennycud
bd4472c2c0 Merge pull request #5 from Philreact/feature/search-keywords
added keywords to qortalRequest
2025-03-19 17:37:14 -07:00
kennycud
10dda255e2 added rebuild arbitrary rebuild resource timer task 2025-03-16 18:49:28 -07:00
kennycud
934c23402a added logging, so we can better understand the exception thrown 2025-03-16 18:47:59 -07:00
kennycud
4188f18a9a added error handling 2025-03-16 18:46:38 -07:00
kennycud
e48fd96c1e nullified impossible time constraints 2025-03-14 14:15:18 -07:00
kennycud
e76694e214 implemented before and after filtering 2025-03-13 13:46:17 -07:00
kennycud
dbf49309ec added some critical exception handling for arbitrary data indexing support 2025-03-12 14:24:23 -07:00
kennycud
ab4730cef0 initial implementation of arbitrary data indexing support 2025-03-12 11:21:57 -07:00
kennycud
7f3c1d553f removed name based arbitrary resource storage capacity limits and added arbitrary resource cache rebuild logging verbosity 2025-03-10 15:17:04 -07:00
ab0ef85458 added keywords to SEARCH_QDN_RESOURCES 2025-03-08 20:43:34 +02:00
b64674783a Merge remote-tracking branch 'kenny/master' into feature/search-keywords 2025-03-08 20:19:56 +02:00
kennycud
92fb52220a Merge pull request #4 from Philreact/feature/search-keywords
Feature/search keywords
2025-03-06 07:10:26 -08:00
kennycud
2d0bdca8dc Merge pull request #3 from Philreact/bugfix/get-qdn-resource-metadata
fix var bug for GET_QDN_RESOURCE_PROPERTIES
2025-03-06 07:06:43 -08:00
2e9f358d0b changed to list and added to cache 2025-03-06 16:10:30 +02:00
6a6380e9e7 Merge remote-tracking branch 'kenny/master' into feature/search-keywords 2025-03-06 14:02:36 +02:00
kennycud
11c2d1bd14 a solution for the metadata and status members getting nullified in the cache 2025-03-05 18:47:01 -08:00
1d79df078e Merge remote-tracking branch 'kenny/master' into feature/search-keywords 2025-03-05 21:14:47 +02:00
kennycud
4baafd1305 more arbitrary data optimizations, including the arbitrary resources cache rebuild and a setting to support it, added and removed notifications, added method to the arbitrary repository, also removed an unnecessary setting that was added in the last commit 2025-03-03 10:37:39 -08:00
Ice
0fae20a3c3 Update README.md
Added IntelliJ Information
2025-03-02 10:14:35 -05:00
f8cee2e0b7 Merge remote-tracking branch 'kenny/master' into feature/search-keywords 2025-02-27 18:36:00 +02:00
kennycud
676885ea2d optimized arbitrary metadata fetching, added arbitrary data cache manager notifications, removed redundant notifications, added method to arbitrary repository and a setting to support the optimization 2025-02-24 16:36:13 -08:00
Ice
a90f217212 Update pom.xml
Changes for hsqldb to use local 2.7.4 version with modified manifest
2025-02-24 14:03:39 -05:00
Ice
e40a77542b New hsqldb-2.7.4 with modified manifest 2025-02-24 13:58:32 -05:00
Ice
80b24b185f Create Notes.txt 2025-02-24 13:55:10 -05:00
kennycud
1f4ca6263f data monitor initial implementation 2025-02-19 17:18:05 -08:00
kennycud
df37372180 trade ledger export implementation, completed trades bug fix 2025-02-11 18:45:57 -08:00
086b0809d6 remove log 2025-02-08 22:20:03 +02:00
33650cc432 when the path is render/hash do not save path for nav history 2025-02-05 15:11:48 +02:00
c22abc440b change label 2025-02-04 18:02:56 +02:00
258eb3a0f9 added keywords query for arbitrary research search 2025-02-04 15:42:25 +02:00
kennycud
91ceafe0e3 supporting multiple minting groups instead of supporting one and only one minting group 2025-02-03 18:19:56 -08:00
kennycud
9017db725e Merge remote-tracking branch 'origin/master' 2025-02-01 18:44:17 -08:00
kennycud
a42f214358 invite orphan vulnerability patch, detailed test case coming in a commit soon 2025-02-01 18:43:48 -08:00
ecd4233dd0 fix fetch block qortalRequest 2025-01-26 00:13:58 +02:00
e5b6e893cd GET_AT missing a slash 2025-01-24 21:30:55 +02:00
9e45d640bc fix var bug 2025-01-23 23:52:39 +02:00
crowetic
faee7c8f6a Merge pull request #247 from crowetic/master
push featureTrigger blocks back a bit to give more time to prepare+sign auto-update
2025-01-21 19:26:47 -08:00
ca238c995e push featureTrigger blocks back a bit to give more time for auto-update. 2025-01-21 19:12:17 -08:00
e434a28d00 Merge remote-tracking branch 'origin/master' 2025-01-21 19:11:19 -08:00
996d5e0e86 push featureTrigger blocks back a bit to give more time for auto-update. 2025-01-21 19:10:06 -08:00
8b797b5bd5 push featureTrigger blocks back a bit to give more time for auto-update. 2025-01-21 19:05:57 -08:00
crowetic
999cfafe00 Merge pull request #246 from crowetic/master
updates/fixes to publish-auto-update.pl
2025-01-21 18:24:13 -08:00
4991618f19 updates/fixes to publish-auto-update.pl 2025-01-21 18:22:25 -08:00
crowetic
4c35239bb1 Merge pull request #245 from crowetic/master
bump version to 4.7.0 and set featureTrigger block heights
2025-01-21 18:10:13 -08:00
d6cf45b311 bump version to 4.7.0 and set featureTrigger block heights 2025-01-21 18:07:25 -08:00
crowetic
ea9a24dca2 Merge pull request #244 from kennycud/master
Balance Recorder & Hard Forks
2025-01-21 17:35:26 -08:00
kennycud
72f0194487 get admin query fix and hardfork 2025-01-17 19:31:13 -08:00
kennycud
b2dbcbb603 made adjustments to support the ignore level feature trigger and removed the fail-safe feature trigger since the ignore level feature trigger now satisfies it implicitly 2025-01-13 13:52:17 -08:00
kennycud
69cba78d94 exclude blocked implementation completion 2025-01-11 19:01:13 -08:00
kennycud
70f4ff4fb3 ignore level for reward share feature hard fork 2025-01-11 18:20:28 -08:00
kennycud
a8a8904ebf removed the NULL account from the dev admin reward distribution and added some fail safes in case the admin groups are empty 2025-01-08 16:19:38 -08:00
kennycud
2805bb8364 corrected an arithmetic error 2025-01-07 13:20:39 -08:00
kennycud
d9a7648d36 access to decoded online accounts by block 2025-01-05 15:59:09 -08:00
kennycud
2392b7b155 system info and database connection status access 2025-01-05 13:49:31 -08:00
kennycud
f5d338435a Since the Groups table is now named Groups with back ticks, it is now case-sensitive. Since it is now case-sensitive it needs to be in all caps, so when other SQL statements call on this table using the Groups without backticks it will be compatible. When Groups is used in a statement without back ticks or quotes it automatically gets converted into capital letters. 2025-01-02 18:10:25 -08:00
kennycud
8f6b55a98b rollback the Groups table back quotes, because this only works with my testing environment and causes problems in production 2024-12-31 13:57:39 -08:00
kennycud
278243f01c rollback the negation of founder effective minting level, because I made it under the assumption that it was used for reward distributions when it is used for block signatures only 2024-12-31 13:54:20 -08:00
kennycud
756f3a243d negate founder effective minting level for admins replace founders hardfork 2024-12-30 18:36:44 -08:00
kennycud
950c4a5b35 Merge remote-tracking branch 'origin/master' 2024-12-30 16:06:24 -08:00
kennycud
ebc58c5c5c qualified Groups table name, so it will be compatible with HSQLDB updated release which uses Groups for as a reserved word 2024-12-30 16:01:53 -08:00
kennycud
8bbb994876 Merge pull request #2 from Philreact/master
added seller/buyer to filter completed trades
2024-12-30 12:19:01 -08:00
kennycud
c2ba9d142c crowetic's logging suggestions for the new reward distribution update 2024-12-30 12:15:27 -08:00
kennycud
a300ac2393 added capabilities for groups with null ownership including banning and kicking members and member ban cancellations; enforcing group approval thresholds to invites and invite cancellations; the established add and remove admin capabilities were used as guidance for this implementation; this was added as a hardfork to preserve group transactions from previous blocks 2024-12-29 18:08:04 -08:00
kennycud
bdbbd0152f updated the hard fork heights for the test chain 2024-12-28 14:01:01 -08:00
kennycud
45d88c1bac Admin share typo fix and new test case submission. 2024-12-26 14:40:44 -08:00
kennycud
3952705edd Admin replace founders hardfork and online validation fail-safe hardfork. 2024-12-26 13:53:00 -08:00
kennycud
4f0aabfb36 For Balance Recorder, reward recordings only, that is the default. 2024-12-25 13:24:24 -08:00
5ac0027b5a fix css for qdn resource loading 2024-12-25 09:16:35 +02:00
e9b75b051b added seller/buyer to filter completed trades 2024-12-24 14:39:31 +02:00
kennycud
c71f5fa8bf added another logging line to troubleshoot QDN problem 2024-12-13 15:21:51 -08:00
kennycud
5e145de52b Balance Recorder initial implementation. 2024-12-12 13:46:18 -08:00
kennycud
543d0a7d22 Merge remote-tracking branch 'origin/master' 2024-12-10 14:07:32 -08:00
kennycud
5346c97922 added logging to help solve the updated field problem, the problem is the updated field is not getting updated 2024-12-10 14:07:11 -08:00
crowetic
c2bfa26376 Merge pull request #242 from crowetic/master
Bump version to 4.6.6 and other changes
2024-12-06 10:48:37 -08:00
crowetic
386387fa16 Added modifications to current Windows Installer build in preparation for 4.6.6 release
modified AdvancedInstaller settings, created new installer visual settings and included logo utilized for that. Modified Readme file to include additional instructions.
2024-12-05 20:18:52 -08:00
071325cf6d Bump version to 4.6.6 to prepare for update, modified auto-update repos settings to plan for removal of reliance upon GitHub, increased maxPeerConnectionTime to 6 hours instead of 4, and set default minPeerversion to 4.6.5. 2024-12-05 20:13:32 -08:00
a23eb02182 Revert "modified autoUpdateRepos further to plan ahead."
This reverts commit 04203e7c31.
2024-12-05 20:08:24 -08:00
04203e7c31 modified autoUpdateRepos further to plan ahead. 2024-12-05 20:00:31 -08:00
crowetic
749143e98e Merge pull request #241 from crowetic/master
Selective acceptance of recent PRs to Qortal master branch, and updated start.sh script. See description for details.
2024-12-04 14:12:04 -08:00
9b20192b30 Changes need to be reverted prior to the PR from crowetic repo being merged. All of these changes aside from those in the 'network' folder, will be re-applied with crowetic's PR.
Revert "Various changes"

This reverts commit adbba0f947.
2024-12-04 14:08:25 -08:00
8d6830135c Changes need to be reverted prior to new PR from crowetic repo.
Revert "Update dependencies"

This reverts commit e3a85786e7.
2024-12-04 14:07:43 -08:00
448b536238 Modified start script to work with optimized Garbage Collection made available in version 4.6.6 and beyond. Suggestion to machines with 6GB of RAM or less, increase the percentage from 50 to 75. Qortal Core will only utilize the RAM it needs, up to the percentage set maximum. 2024-12-03 09:09:42 -08:00
2e989aaa57 A merge of just alpha's validation changes, phil and quick's commits, and kenny's changes to test. 2024-12-03 08:29:53 -08:00
crowetic
8bd293ccd5 Merge pull request #217 from QuickMythril/4.6.2-unit-test-fix
Fix for more unit tests fails
2024-12-02 14:44:16 -08:00
crowetic
a8d73926b3 Merge pull request #238 from AlphaX-Qortal/master
Added real address to API results - Currently the address shown in the API results when querying blocks, shows an address formed by the 'reward share public key', this address is not useful for viewing, as it is not the address utilized for QORT. This change makes it so the 'real' Qortal address is displayed instead of this useless address. Thanks @AlphaX-Qortal

Added group member check to validations - validation fixes.

Network changes - Moved unnecessary 'we already have connection' messages from info logging to debug. Updated minPeerVersion default to current release version. (4.6.5). Updated default peer list. Updated syntax. Updated formatting.

Updated dependencies

Thanks @AlphaX-Qortal
2024-12-02 14:42:34 -08:00
crowetic
bd214a0fd3 Merge pull request #220 from Qortal/master2
adjust timeouts for qortalrequests
2024-12-02 14:28:53 -08:00
crowetic
2347118e59 Merge pull request #239 from kennycud/master
Restructuring database connections for better garbage collection - resolves long-standing memory leak in multiple places that was discovered more specifically after the thread crashes were made to restart if crashed. Thanks so much to @kennycud  for this improvement!
2024-12-02 14:26:43 -08:00
crowetic
7fb093e98a Merge pull request #237 from Philreact/active-chat-haschatreference
add haschatreference query to activechats endpoint
2024-12-02 14:24:45 -08:00
AlphaX-Qortal
e3a85786e7 Update dependencies 2024-12-02 15:06:46 +01:00
AlphaX-Qortal
adbba0f947 Various changes
- Added real address to API results
- Added group member check to validations
- Network changes
2024-12-02 14:22:05 +01:00
61dec0e4b7 add haschatreference query to activechats endpoint 2024-12-01 12:38:38 +02:00
cwd.systems | 0KN
15105306d1 Update CancelSellNameTransaction.java 2024-11-30 19:20:27 +06:00
cwd.systems | 0KN
3ddef1e13f Update SellNameTransaction.java 2024-11-30 19:19:25 +06:00
kennycud
08a2284ce4 deleting file that interferes with building the last commit 2024-11-27 18:06:32 -08:00
kennycud
2e3f97b51f Merge remote-tracking branch 'origin/master' 2024-11-27 17:43:51 -08:00
kennycud
84b973773a restructuring database connections for better garbage collection, adding in the initial implementation of the balance recorder 2024-11-27 17:43:18 -08:00
AlphaX
8ffb0625a1 Bump version to 4.6.5 2024-11-26 23:27:35 +01:00
AlphaX
2ce02faa07 Bump version to 4.6.4 2024-11-26 19:42:13 +01:00
AlphaX
89999e6b33 Set feature trigger 2024-11-26 19:41:15 +01:00
AlphaX
4d28ba692d Update minimum peer version 2024-11-26 19:34:45 +01:00
AlphaX
cd6d7a3a98 Merge pull request #223 from AlphaX-Qortal/master
Set peer connect to a dedicated thread pool for non-blocking I/O (Thanks to RAZ)
2024-11-26 12:34:06 +01:00
AlphaX-Qortal
0a44928e93 Set peer connect to a dedicated thread pool for non-blocking I/O (Thanks to RAZ) 2024-11-26 11:05:46 +01:00
AlphaX
4b037ad13f Merge pull request #222 from AlphaX-Qortal/master
Fix batch reward
2024-11-26 07:51:06 +01:00
crowetic
1f9a2edca4 Merge pull request #221 from kennycud/master
Minter Group Check Optimizations - Have been tested by 50+ nodes for multiple days. The only thing we have to verify prior to merging the upcoming changes from Alpha, is validate the additional boolean passed in to canMint on line 1521 in current block.java (isMinterValid)
2024-11-25 18:01:13 -08:00
AlphaX-Qortal
c010ab47db Fix batch reward 2024-11-26 00:03:04 +01:00
7803d6c8f5 adjust timeouts for qortalrequests 2024-11-25 09:36:11 +02:00
kennycud
b0d43a1890 minter group check optimizations 2024-11-20 19:12:21 -08:00
kennycud
f277611d31 Merge branch 'master' of https://github.com/kennycud/qortal
 Conflicts:
	src/main/java/org/qortal/account/Account.java
2024-11-20 15:40:11 -08:00
AlphaX
d89f7ad41d Bump version to 4.6.3 2024-11-20 19:50:14 +01:00
AlphaX
39cc56c4d8 Update minimum peer version 2024-11-20 19:49:17 +01:00
AlphaX
fccd5a7c97 Merge pull request #219 from AlphaX-Qortal/master
Update canMint and HSQLDB
2024-11-20 19:45:18 +01:00
AlphaX-Qortal
46395bf4dc Updare canMint and HSQLDB 2024-11-20 19:35:47 +01:00
AlphaX
0eb551acc1 Merge pull request #214 from Philreact/master2
add connect-src to csp
2024-11-20 01:22:00 +01:00
kennycud
f55efe38c5 Removed logging statements to demonstrate order of operations to others. Added optimizations for the canMint() method. This is a quick fix and a more comprehensive fix will be done in the future. 2024-11-18 15:09:43 -08:00
kennycud
130bb6cf50 Added logging statements to demonstrate order of operations. This will be removed ASAP and should not be included in a PR. 2024-11-17 17:17:00 -08:00
QuickMythril
652c902607 Add missing feature triggers to unit tests 2024-11-17 16:45:39 -05:00
QuickMythril
915bb1ded3 Merge pull request #74 from QuickMythril/4.6.1-unit-test-fix
4.6.1 unit test fix
2024-11-17 13:50:06 -05:00
AlphaX
8319193453 Bump version to 4.6.2 2024-11-17 18:48:32 +01:00
AlphaX
831ed72e56 Update minimum peer version 2024-11-17 18:47:06 +01:00
AlphaX
885133195e Set timestamps 2024-11-17 18:44:01 +01:00
crowetic
c45d59b389 Merge pull request #216 from AlphaX-Qortal/master
Removed name check and decreased difficulty for online signature calculation
2024-11-17 09:40:35 -08:00
AlphaX-Qortal
30a289baab Update dependencies 2024-11-16 21:22:00 +01:00
AlphaX-Qortal
d79d64f6b0 Removed name check and decreased difficulty for online signature 2024-11-16 21:14:42 +01:00
QuickMythril
3d83a79014 Fix whitespace only 2024-11-13 06:28:23 -05:00
QuickMythril
82d5d25c59 Add logging to block archive unit tests 2024-11-13 06:16:39 -05:00
QuickMythril
1676098abe Add missing feature triggers to unit tests 2024-11-13 03:06:58 -05:00
0a47ca1462 add font-src csp 2024-11-11 16:07:51 +02:00
0cf9b23142 remove log 2024-11-10 18:57:45 +02:00
0850654519 add connect-src to csp 2024-11-10 18:55:32 +02:00
AlphaX
6648c4be22 Bump version to 4.6.1 2024-11-08 18:30:40 +01:00
kennycud
07474ab841 Merge pull request #212 from kennycud/master
Reduced connection error to a warning. Removed unnecessary return val…
2024-11-08 09:13:43 -08:00
kennycud
76f5d17f81 Reduced connection error to a warning. Removed unnecessary return value. Another file needed to be added to the last commit; here it is. 2024-11-08 09:03:36 -08:00
crowetic
50d6e388f0 Merge pull request #211 from crowetic/master
PR for combined changes from kenny+alpha+crowetic for release candidate
2024-11-08 08:56:26 -08:00
crowetic
93c8f78cd3 Merge branch 'master' into master 2024-11-08 08:56:05 -08:00
crowetic
d42acb788b Merge pull request #209 from QuickMythril/fix-unit-tests
Fix for most unit tests fails
2024-11-08 08:55:11 -08:00
2ee5bc5b35 added README changes to build a single PR for new release candidate 2024-11-08 08:40:55 -08:00
5e1ad82738 Merged alpha default restart settings and kenny logging changes, made a couple small default settings changes for thread priority. 2024-11-08 08:30:16 -08:00
7fa1180622 Merge remote-tracking branch 'kenny/master' 2024-11-08 08:30:04 -08:00
65c637740a Merge remote-tracking branch 'alphax/master' 2024-11-08 08:27:53 -08:00
AlphaX-Reloaded
a0b4853518 Moved auto restart node as feature in settings 2024-11-08 09:59:18 +01:00
kennycud
bb40dcde65 Reduced connection error to a warning. Removed unnecessary return value. 2024-11-07 11:51:56 -08:00
crowetic
e74a06e031 Update README.md
added project overview, and additional information to readme.
2024-11-07 11:03:40 -08:00
crowetic
ea10759bd3 Merge pull request #210 from crowetic/master
Added kenny+alpha+phil changes to create what should be release candidate. Merging upon verification from alpha+kenny.
2024-11-07 10:46:40 -08:00
2c017fc1b0 added alpha's changes together with kenny's to make final BlockArchiver.java 2024-11-06 09:07:37 -08:00
497259f652 Merge remote-tracking branch 'alphax/master' 2024-11-06 08:38:41 -08:00
38fd0c55a0 pulled alpha's recent changes for hopefully final release candidate 2024-11-06 08:31:19 -08:00
AlphaX-Reloaded
3f2fc5c6a2 Give block archiver more time after started 2024-11-06 16:04:36 +01:00
AlphaX-Reloaded
56db39e190 Fixed restart
Checks if for some reason blockchain lock file still exist after shutdown.
2024-11-06 15:42:17 +01:00
AlphaX-Reloaded
bcf68ea63d Adjusted network settings and added reboot 2024-11-05 16:57:53 +01:00
fd5ba48611 Merge kenny's changes to ExecuteProduceConsume making it threadsafe and changing the methods utilized. 2024-11-04 13:11:40 -08:00
kennycud
5e315de213 Made networking engine thread-safe and removed redundant locking. 2024-11-04 12:59:27 -08:00
QuickMythril
aa6c38e631 Fix for most unit tests fails 2024-11-03 15:20:29 -05:00
a0c7e3d94e made modifications to prevent double-setting the same priorities. Kept any/all hard-coded changes from Alpha for anything that didn't already have a setting created by Kenny. Essentially this is a merger of the two repos from Alpha and Kenny, keeping the changes that are relevant from each, while also keeping the thread priority settings in settings.json.
This will give us the best of both worlds (in theory), and dramatically improved overall network performance.
2024-11-01 14:49:54 -07:00
209920b2f8 Merge remote-tracking branch 'kenny/master' 2024-11-01 13:49:18 -07:00
AlphaX-Reloaded
941847dd98 Reduce thread pool size for less resource use 2024-11-01 12:23:15 +01:00
AlphaX-Reloaded
881d9c9635 Changes to thread priority and executor service 2024-11-01 10:10:47 +01:00
kennycud
ab78f22b5e The last update introduced an issue that blocked out the synchronizer and that has been fixed. Also, removed some logging spam and removed some false alarms in the logging. 2024-10-29 16:05:13 -07:00
kennycud
c2d7dfe42e allowing user to set thread priorities for numerous threads, added a pause to the block archive writing thread that occurs between each block written to allow other threads to execute, the length of the pause is also available in the settings for modification by the user 2024-10-27 16:32:21 -07:00
kennycud
2629e5f2e8 broader exception handling and added verbosity to the logging to 4 more threads 2024-10-23 17:39:48 -07:00
kennycud
fa1e86a46b added more descriptive labelling to the logging 2024-10-23 17:37:49 -07:00
kennycud
85910573a3 broader exception handling and added verbosity to the logging 2024-10-22 11:30:39 -07:00
kennycud
3043d1c2cb replaced String array declarations with Object array declarations to satisfy compiler warnings 2024-10-20 06:20:24 -07:00
kennycud
e6d13536e0 Merge branch 'Qortal:master' into master 2024-10-18 15:38:49 -07:00
AlphaX
ec8ddf2b1c Bump version to 4.6.0 2024-10-18 18:25:58 +02:00
AlphaX
e44382a72e Update minimum peer version 2024-10-18 14:44:24 +02:00
AlphaX
1e90d8757b Set feature triggers 2024-10-18 14:41:10 +02:00
kennycud
bd942dd8e3 Merge branch 'Qortal:master' into master 2024-10-17 16:27:32 -07:00
crowetic
524ed2b301 Merge pull request #207 from AlphaX-Reloaded/master
Merging - adding features for 'emergency update', modifications to testnet config and scripts, and reindex functionality for sync-from-genesis.
2024-10-17 13:09:11 -07:00
AlphaX
5c7ffce887 Merge pull request #1 from Qortal/master
Merge Api calls
2024-10-15 19:59:18 +02:00
kennycud
27cc9e458e added support for limit set to zero or negative as no limit 2024-10-14 03:56:22 -07:00
kennycud
187a360467 Implemented the HSQLDB Cache to enhance /arbitrary/resource/search. This is disabled by default and must be set to true in the settings.json file. 2024-10-13 12:52:05 -07:00
kennycud
0a5ac37815 Merge remote-tracking branch 'origin/master' 2024-10-13 12:35:39 -07:00
kennycud
85cf740839 fixed a copy/paste error in toString by regenerating it 2024-10-13 12:35:24 -07:00
crowetic
900e2a9e61 Merge pull request #204 from QuickMythril/add-bootstrap-server
Added bootstrap4.qortal.org to settings
2024-10-12 08:37:44 -07:00
crowetic
252bb84f24 Merge pull request #203 from kennycud/master
New endpoints for arbitrary resources (simple search), sponsorship analytics and address/level pairings
2024-10-11 10:59:57 -07:00
crowetic
036d9b67ed Merge pull request #205 from QuickMythril/pubkey-conversion
removed API restriction from pubkey conversion
2024-10-11 10:57:47 -07:00
kennycud
7c9d82b780 fixed the incorrect sell amounts by replacing the complex buy sell query with 2 queries, one for buys and one for sells 2024-10-05 12:47:40 -07:00
kennycud
d9ad0bd663 sponsorship endpoints now return a different report type than the mintership endpoint 2024-10-03 12:42:44 -07:00
QuickMythril
f105af696d removed API restriction from pubkey conversion 2024-09-30 09:25:07 -04:00
QuickMythril
3162a83dc1 Added bootstrap4.qortal.org to settings 2024-09-30 09:08:28 -04:00
kennycud
39da7edf5a handled null pointer case caused by the last update 2024-09-29 19:02:06 -07:00
kennycud
2b83c4bbf3 fixed the query to get registered names 2024-09-29 08:37:16 -07:00
kennycud
68a2e65fc7 added MintershipReport endpoint, restructured the sponsorship report logic, so the MintershipReport could take advantage of the common logic, fixed some of the annotations in the sponsorship report endpoints 2024-09-29 05:50:30 -07:00
kennycud
639e1df531 in sponsorship reports, exclude the recipients that get real reward shares 2024-09-28 12:52:55 -07:00
kennycud
7c4b0bd7f2 Merge remote-tracking branch 'origin/master' 2024-09-27 19:26:17 -07:00
kennycud
bbf2787ba4 removed import that was put in by accident on a merge 2024-09-27 19:25:46 -07:00
kennycud
d42148c3a3 Merge pull request #1 from Crowetic-Hardware-Development/master
added new '/addresses/levels' API call that will pull an array of acc…
2024-09-27 18:06:26 -07:00
aba4c6000f added new '/addresses/levels' API call that will pull an array of accounts with their levels, based on an input number. Accounts and levels at or above the input number will be pulled and displayed along with their level. Thanks to @kennycud! 2024-09-27 18:00:32 -07:00
kennycud
5a691762ed bug fix, no rows should return zero counts instead of throwing an exception 2024-09-23 17:07:04 -07:00
kennycud
a530b64ae7 added transfer privs inquiries to sponsorship analytics 2024-09-23 16:24:24 -07:00
kennycud
f82533370d Merge remote-tracking branch 'origin/master' 2024-09-23 08:05:35 -07:00
kennycud
d976904a8e added 2 endpoints providing sponsorship analytics 2024-09-23 08:04:44 -07:00
AlphaX-Reloaded
d0139c24dc Added member of group check 2024-09-21 21:29:02 +02:00
AlphaX
c58d2b5813 Added get namecheck height 2024-09-19 14:22:21 +02:00
AlphaX
cf591bea4c Update testchain.json 2024-09-19 14:16:25 +02:00
AlphaX
acd30eddf0 Added namecheck to feature triggers 2024-09-19 14:11:15 +02:00
AlphaX
b6272c1b0f Fix namecheck 2024-09-19 14:00:09 +02:00
AlphaX-Reloaded
885aa10395 Added Rewardshare disabled 2024-09-16 15:27:00 +02:00
kennycud
3fda19c8b9 Merge branch 'Qortal:master' into master 2024-09-15 16:11:59 -07:00
kennycud
f14bc86b39 implemented arbitrary resources simple search 2024-09-15 16:10:30 -07:00
AlphaX
51664f0561 Fix testnet start script 2024-09-12 17:00:33 +02:00
AlphaX
b10dfd28c1 Fix Testnet 2024-09-12 16:59:22 +02:00
AlphaX
9600185365 Testnet new genesis 2024-09-12 10:22:18 +02:00
AlphaX
bbb346a98e Update dependencies 2024-09-11 22:17:09 +02:00
AlphaX
f59e6580ab Disable batch reward on testnet 2024-09-11 21:45:51 +02:00
AlphaX
0266248ca6 Merge branch 'Qortal:master' into master 2024-09-11 20:47:00 +02:00
crowetic
cbb171f859 Merge pull request #202 from kennycud/master
Foreign Wallet Balance Enhancements
2024-09-05 17:57:58 -07:00
kennycud
a64e9052dd consolidated the cache limits into an attribute in Settings.java 2024-09-04 16:24:08 -07:00
kennycud
acc37cef0e storing blockchain data in a cache to reduce redundant RPCs to the ElectrumX servers 2024-09-03 18:42:27 -07:00
kennycud
7f5692eac0 Merge remote-tracking branch 'origin/master' 2024-09-03 18:40:02 -07:00
kennycud
454c471dfe Changed gapLimit from 24 to 3 since we have mitigated the gap problem. 2024-09-03 18:39:41 -07:00
crowetic
1a42c52f65 Merge pull request #201 from kennycud/master
Support for responding to multiple crosschain sell offers
2024-09-03 16:21:40 -07:00
kennycud
145191075a Responding to multiple trade offers on Pirate Chain now throws an invalid criteria error. The Pirate Chain API we are using does not support multiple spends. 2024-08-30 04:53:38 -07:00
kennycud
da889f2905 Including unconfirmed transactions for wallet balances while spending foreign coin. This is for sends and for responding to trade sell orders. This is not for any other stages in the trading process after the initial response. 2024-07-27 12:50:22 -07:00
kennycud
211fc0d5a4 Protocol version error handling improvements. 2024-07-26 13:53:05 -07:00
kennycud
51feb96824 Delegated buildOfferMessage functionality to CrossChainUtils and removed copies of it from each individual class. Delegated trade bot state enumerations to the TradeStates class. 2024-07-26 13:50:43 -07:00
kennycud
8c1251d716 Delegated buildOfferMessage functionality to CrossChainUtils and removed copies of it from each individual class. Delegated trade bot state enumerations to the TradeStates class. 2024-07-26 10:40:27 -07:00
kennycud
4f05b61a8e Support for responding to multiple crosschain sell offers. 2024-07-25 15:38:56 -07:00
kennycud
a07052161a Support for responding to multiple crosschain sell offers. 2024-07-25 15:36:22 -07:00
kennycud
780bfe6249 Support for responding to multiple crosschain sell offers. 2024-07-25 15:30:08 -07:00
kennycud
d4b0d47c90 Support for responding to multiple crosschain sell offers. 2024-07-25 15:18:11 -07:00
kennycud
da1ea9fe2c Adding for later use. 2024-07-25 13:11:12 -07:00
kennycud
4cf157ba64 Adding for later use. 2024-07-25 13:07:04 -07:00
kennycud
661827f92a Adding for later use. 2024-07-25 13:04:04 -07:00
kennycud
1cd5dccbd6 Adding support for BitcoinyTBD, version parsing and crosschain offer message building. 2024-07-25 12:56:03 -07:00
kennycud
f5cd664dde I'm adding this in now for later use. These are the parameters needed for future foreign blockchain support. 2024-07-25 12:52:25 -07:00
AlphaX
456a2aebca Add start stop scripts for testnet 2024-07-19 17:03:03 +02:00
AlphaX
6377b67274 Update settings-test.json 2024-07-19 17:01:01 +02:00
AlphaX
449aab0a11 Update testchain.json 2024-07-19 16:59:33 +02:00
AlphaX
02346bd9f0 Update testchain.json 2024-07-17 17:06:07 +02:00
AlphaX-Reloaded
21257065ff Revert enable heckpoints, raise private mail size and add testnet 2024-07-17 16:17:30 +02:00
AlphaX-Reloaded
708a978de0 Activate checkpoints 2024-07-07 12:12:16 +02:00
AlphaX-Reloaded
e4134a769b Added sync from genesis and reindex function 2024-07-05 14:25:27 +02:00
AlphaX-Projects
a02d1cec75 Bump version to 4.5.2 2024-06-30 18:41:04 +02:00
AlphaX-Projects
2d9f1d6d81 Update minimum peer version 2024-06-30 18:38:34 +02:00
crowetic
c1da495dd1 Merge pull request #197 from AlphaX-Projects/master
Bug fixes and additions
2024-06-30 08:48:11 -07:00
AlphaX-Projects
533df9f2b8 Updated dependencies 2024-06-29 15:01:36 +02:00
AlphaX-Projects
44b4b08117 Added block minter thread check 2024-06-29 13:17:07 +02:00
AlphaX-Projects
a140805c36 Updated group transactions (owner check) 2024-06-29 12:58:58 +02:00
AlphaX-Projects
ea1d4dd962 Updated electrumx with non null objects 2024-06-29 12:48:06 +02:00
AlphaX-Projects
a6cbdaafd8 Updated to bouncycastle 1.70 (AT and Core) 2024-06-29 12:23:57 +02:00
AlphaX-Projects
5db808e300 Replaced reference method with qualifier (websockets) 2024-06-29 12:18:52 +02:00
AlphaX-Projects
cc95106019 Removed unnecessary continue (last statement in a loop) 2024-06-29 12:15:55 +02:00
AlphaX-Projects
64537ad705 Fixed synchronization on a non-final field 2024-06-29 12:13:12 +02:00
AlphaX-Projects
806dc6d056 Replaced string.equals() with string.isEmpty() 2024-06-29 12:11:38 +02:00
AlphaX-Projects
d58fbab1b5 Removed pointless boolean Expression 2024-06-29 11:58:25 +02:00
AlphaX-Projects
61ede811cd Replaced size() == / > 0 with isEmpty() 2024-06-29 11:54:41 +02:00
AlphaX-Projects
95d42db773 Removed unnecessary semicolons 2024-06-29 11:52:22 +02:00
AlphaX-Projects
ef07a444d6 Removed unnecessary imports 2024-06-29 11:49:19 +02:00
crowetic
5910ceff80 Merge pull request #194 from kennycud/master
Adding in foreign blockchain server configuration add and remove capa…
2024-05-31 17:44:50 -07:00
kennycud
f916d3581b Corrected incorrect language in the comments. This was copied and pasted 6 times. 2024-05-27 10:10:45 -07:00
kennycud
498f409346 Adding in foreign blockchain server configuration add and remove capabilities. Also adding in connection recording and connection setting capabilities. 2024-05-26 11:08:07 -07:00
karl-dv
991636ccad Some small corrections for "NL" translations 2024-05-15 07:34:44 +02:00
crowetic
d54f840265 Merge pull request #185 from kennycud/master
Server Info Current Server & Ordering
2024-03-25 09:15:41 -07:00
crowetic
cc740cceec Merge pull request #187 from AlphaX-Projects/master
Update QDN file management
2024-03-25 09:13:02 -07:00
AlphaX-Projects
df39819de0 Update QDN file management
- Allow files to move around the network more quickly
- Ensuring that metadata isn't saved for blocked names
- More selective disk usage
- Minor fixes
2024-03-25 07:34:26 +01:00
kennycud
e83b2263f0 some simple error logging 2024-03-23 09:42:33 -07:00
kennycud
5c90c4bc61 Merge remote-tracking branch 'origin/master' 2024-03-23 06:28:02 -07:00
kennycud
f7793443f3 call current blockchain height to ensure the current server is set to the blockchain provider and sort the servers to ensure the current server is listed first 2024-03-23 06:25:31 -07:00
crowetic
f6b91df7b6 Merge pull request #181 from kennycud/master
Foreign Coin Trade Fees & Summaries
2024-02-27 12:02:24 -08:00
kennycud
92d589a1ca Merge branch 'Qortal:master' into master 2024-02-19 04:56:33 -08:00
AlphaX-Projects
c4a7fb3b92 Bump version to 4.5.1 2024-02-18 18:14:33 +01:00
AlphaX-Projects
2d27901f9f Adding new algos 2024-02-18 18:12:43 +01:00
AlphaX-Projects
ce8fb002cc Merge pull request #179 from lgedgar/log-invalid-timestamps
Add debug logging when invalid timestamp is encountered
2024-02-14 15:41:09 +01:00
kennycud
3f29116b47 Foreign coin trade transaction summaries 2024-02-12 06:31:17 -08:00
Lance Edgar
d05359dfa9 Add debug logging when invalid timestamp is encountered 2024-02-07 16:52:12 -06:00
kennycud
587b063e6a Merge branch 'Qortal:master' into master 2024-02-06 15:22:51 -08:00
AlphaX-Projects
9e001dfc16 Remove penalty fix 2024-02-03 19:51:51 +01:00
AlphaX-Projects
55f941467f Remove fix in order for the chain history to remain valid 2024-02-03 14:05:11 +01:00
AlphaX-Projects
4f9a4a2091 Disable fix in order for the chain history to remain valid 2024-02-02 16:40:00 +01:00
crowetic
d579606d2d Merge pull request #174 from QuickMythril/fix-unit-tests
Update ElectrumX servers & fix unit tests
2024-01-31 19:32:20 -08:00
crowetic
dcedcf8685 Merge pull request #176 from QuickMythril/votes-api-fix
Fixed vote weight results API call
2024-01-31 19:27:01 -08:00
QuickMythril
f78764880c Fixed vote weight results API call
blocksMintedPenalty is a negative value, so it should be added to blocksMinted, not subtracted.
2024-01-31 21:13:19 -05:00
AlphaX-Projects
070f14b3dd Bump version to 4.5.0 2024-01-30 17:38:34 +01:00
AlphaX-Projects
2120490f4b Fix wrong penaties
This will set incorrectly penalized accounts from previous algo run back to previous state.
2024-01-30 17:36:17 +01:00
QuickMythril
6051b85e52 Remove out of service 2024-01-21 08:42:05 -05:00
QuickMythril
9c62740f44 Merge branch 'master' into fix-unit-tests 2024-01-21 08:36:41 -05:00
AlphaX-Projects
0ed27228b1 Merge pull request #173 from AlphaX-Projects/master
Out of Service
2024-01-21 14:24:40 +01:00
AlphaX-Projects
b75c2029ac Out of Service 2024-01-21 14:23:46 +01:00
kennycud
21c45535be Enabled fee updates for foreign coin trade transactions. 2024-01-17 18:19:09 -08:00
AlphaX-Projects
867fe764ca Bump version to 4.4.2 2024-01-17 17:50:22 +01:00
AlphaX-Projects
140f14f2f4 Set exclude reward share transactions blockheight 2024-01-17 17:49:05 +01:00
AlphaX-Projects
9dd61f0e7a Merge pull request #170 from AlphaX-Projects/master
Update dependencies
2024-01-17 17:45:42 +01:00
AlphaX-Projects
747b1a4f9d Update dependencies 2024-01-17 17:28:27 +01:00
QuickMythril
15ae32efd9 Fix crosschain tests 2024-01-17 06:56:06 -05:00
QuickMythril
03ba36729b Update ElectrumX servers 2024-01-17 06:08:09 -05:00
AlphaX-Projects
425152657a Merge pull request #166 from AlphaX-Projects/master
Exclude reward share transactions from the online accounts blocks
2024-01-14 20:36:46 +01:00
AlphaX-Projects
41645ac7b4 Exclude reward share transactions from the online accounts blocks 2024-01-14 18:40:40 +01:00
AlphaX-Projects
83e324c4ad Fix file ending 2024-01-07 14:34:20 +01:00
AlphaX-Projects
d7f44376be Default minPeerVersion set to 4.4.1 2024-01-07 14:32:12 +01:00
AlphaX-Projects
1400e7ae80 Bump version to 4.4.1 2024-01-06 12:25:24 +01:00
AlphaX-Projects
3c116ca4f4 Merge pull request #162 from QuickMythril/testing-20240104
Remove code for unused "Open UI" function
2024-01-06 11:43:16 +01:00
QuickMythril
8caf5bf8be Remove code for unused "Open UI" function 2024-01-04 13:08:37 -05:00
AlphaX-Projects
687667c8fe Update Settings.java 2024-01-04 14:00:33 +01:00
AlphaX-Projects
feb5564666 Merge pull request #161 from crowetic/master
modifications to settings defaults, and start script defaults.
2024-01-04 13:55:52 +01:00
8062cace30 Improve default JVM memory arguments for mac/linux. 2024-01-03 17:23:23 -08:00
bac0f01007 Modified default settings for optimal QDN performance. 2024-01-03 17:06:04 -08:00
AlphaX-Projects
995ed6ab2a Merge pull request #160 from AlphaX-Projects/master
Rework restart and bootstrap node
2024-01-02 15:04:01 +01:00
AlphaX-Projects
cc02810f0c Update dependencies 2024-01-02 14:26:08 +01:00
AlphaX-Projects
84974775b4 Reduce log spam 2024-01-02 14:17:21 +01:00
AlphaX-Projects
677fd7a64f Rework restart node and bootstrap node 2024-01-01 19:31:22 +01:00
AlphaX-Projects
2d070f343b Update translations 2024-01-01 18:56:09 +01:00
AlphaX-Projects
903fa0346a Merge pull request #159 from kennycud/master
Added Core API endpoint to repair LTC wallets
2023-12-30 19:31:04 +01:00
kennycud
be3c6d0b25 Merge remote-tracking branch 'origin/master' 2023-12-28 14:16:22 -08:00
kennycud
21796341f2 Added Core API endpoint to repair LTC wallets generated before 2024 by moving all the address balances to the first address. 2023-12-28 14:15:57 -08:00
AlphaX-Projects
97b4db4095 Update dependencies 2023-12-28 14:54:20 +01:00
AlphaX-Projects
8977dc7933 Merge pull request #158 from AlphaX-Projects/master
Remove unusedaddress ( no more functional )
2023-12-28 14:38:07 +01:00
AlphaX-Projects
7dbf532616 Remove unusedaddress ( no more functional ) 2023-12-28 14:22:01 +01:00
AlphaX-Projects
d6dec118a9 Merge pull request #157 from kennycud/master
Unused Receive Address Rewrite
2023-12-28 14:03:12 +01:00
AlphaX-Projects
813f16df11 Merge pull request #145 from QuickMythril/api-bootstrap
Added API call for bootstrapping node
2023-12-28 14:02:45 +01:00
kennycud
aaba6bf4cf Merge remote-tracking branch 'origin/master' 2023-12-22 13:24:16 -08:00
kennycud
423a1f7bed Practically rewrote the get unused receive address functionality. The prior implementation was skipping addresses. 2023-12-22 13:22:26 -08:00
AlphaX-Projects
e118f4a410 Merge pull request #155 from kennycud/master
Foreign Blockchain Server Configuration Information Offered in the Core API
2023-12-21 18:01:53 +01:00
kennycud
8607c30cb6 Merge branch 'Qortal:master' into master 2023-12-20 04:30:41 -08:00
kennycud
46a9075faf Extracted Server classes into ChainableServer in order offer foreign blockchain server information to the Core API. They were not extracted into the ServerConfigurationInfo class as stated in a recent commit. 2023-12-20 04:24:51 -08:00
AlphaX-Projects
6b775cc639 Merge pull request #154 from QuickMythril/testing-20231219
Add vote weights to API call
2023-12-20 09:17:13 +01:00
kennycud
7e509f27fb Merge remote-tracking branch 'origin/master' 2023-12-19 18:34:13 -08:00
kennycud
de9c3e551d Extracted Server classes into ServerConfigurationInfo in order offer foreign blockchain server information to the Core API 2023-12-19 18:33:36 -08:00
QuickMythril
72aa7b7a77 Add vote weights to API call 2023-12-19 20:38:41 -05:00
AlphaX-Projects
7bd570ae71 Merge pull request #150 from AlphaX-Projects/master
Default minPeerVersion set to 4.4.0
2023-12-12 17:20:59 +01:00
AlphaX-Projects
5afa4e5b1a Default minPeerVersion set to 4.4.0 2023-12-12 17:05:08 +01:00
AlphaX-Projects
3f0999e59d Merge pull request #149 from AlphaX-Projects/master
Update dependencies
2023-12-12 16:43:24 +01:00
AlphaX-Projects
87b3b037bd Update dependencies 2023-12-12 16:20:30 +01:00
AlphaX-Projects
3bf54dbd0a Merge pull request #148 from AlphaX-Projects/master
Update trustless manager and AT 1.4.1
2023-12-12 16:16:34 +01:00
AlphaX-Projects
bf8005aa5a Update AT 2023-12-12 10:26:30 +01:00
AlphaX-Projects
79b674d2f2 Update AT metadata 2023-12-12 10:25:18 +01:00
AlphaX-Projects
c989e3c413 Rework trustless manager 2023-12-12 10:05:24 +01:00
AlphaX-Projects
f6e398ec0f Merge pull request #147 from catbref/AT-ref-fix
Chain AT-transaction references to avoid duplicates
2023-12-11 09:03:23 +01:00
catbref
5054773761 Chain AT-transaction references to avoid duplicates 2023-12-10 14:03:47 +00:00
AlphaX-Projects
a2ccf0e7da Merge pull request #146 from QuickMythril/translate-hebrew
Added Hebrew translations
2023-12-09 09:47:47 +01:00
QuickMythril
b266d205b1 Added Hebrew translations 2023-12-08 18:11:02 -05:00
AlphaX-Projects
6cd722e735 Update testchain.json 2023-12-06 16:17:34 +01:00
AlphaX-Projects
1e4d4e178d Merge pull request #144 from kennycud/master
Added Spendable Attribute to Address Info
2023-12-06 08:55:04 +01:00
kennycud
deaef4fc4a checking in and pushing the isSpendable and public key generation for testing again 2023-12-03 12:45:31 -08:00
kennycud
9671c2da61 using public key for address infos 2023-12-03 11:05:16 -08:00
QuickMythril
14279e58bc Added API call for bootstrapping node 2023-12-02 23:06:36 -05:00
kennycud
87a7b9df08 Revert "Ignoring test cases intended for Bitcoiny coins"
This reverts commit e287fa0ebe.
2023-11-29 17:51:07 -08:00
kennycud
3c307f15b3 Merge remote-tracking branch 'origin/master' 2023-11-29 17:30:26 -08:00
kennycud
e287fa0ebe Ignoring test cases intended for Bitcoiny coins 2023-11-29 17:29:48 -08:00
QuickMythril
a94ef17883 Removed API key from admin/summary 2023-11-18 16:44:41 -05:00
QuickMythril
94cfcd66cd Bump version to 4.4.0 2023-11-15 21:57:41 -05:00
QuickMythril
dbbe78263d Merge pull request #143 from AlphaX-Projects/merge-batch-payout
Merge batch payout
2023-11-15 21:48:44 -05:00
AlphaX-Projects
9e8e0df5d6 Set batch payout blockheight 2023-11-15 17:53:25 +01:00
AlphaX-Projects
95948d35ca Merge pull request #14 from Qortal/master
Merge pull request #142 from AlphaX-Projects/master
2023-11-15 14:06:20 +01:00
QuickMythril
9ca2289ed4 Merge pull request #142 from AlphaX-Projects/master
Added 2 Pirate Chain api endpoints
2023-11-10 13:19:13 -05:00
AlphaX-Projects
e86dab066d Merge pull request #8 from AlphaX-Projects/batch-rewards
Batch rewards
2023-11-09 11:43:14 +01:00
AlphaX-Projects
debb67a676 Merge branch 'merge-batch-payout' into batch-rewards 2023-11-09 11:42:45 +01:00
AlphaX-Projects
6c4ed2496d Added 2 PirateChain api endpoints 2023-11-08 15:29:19 +01:00
AlphaX-Projects
61d6bd9bca Merge pull request #7 from Qortal/master
Merge master
2023-11-08 12:19:01 +01:00
QuickMythril
4e53e6f0ad Merge pull request #141 from QuickMythril/altcoin-address-infos
Add address infos support for altcoins
2023-11-06 16:17:41 -05:00
QuickMythril
183dac5ac4 Add address infos for BTC, DOGE, DGB, RVN 2023-11-06 15:11:55 -05:00
QuickMythril
3a723ed116 Revert temporary change 2023-11-06 14:29:35 -05:00
QuickMythril
9f2b44484e Merge pull request #49 from kennycud/master
Add LTC address infos
2023-11-06 14:24:48 -05:00
QuickMythril
b05c0158b1 Resolve temporary conflict 2023-11-06 14:23:49 -05:00
QuickMythril
d915dc10f3 Default minPeerVersion set to 4.3.2 2023-11-06 13:03:21 -05:00
QuickMythril
b0f21c2eee Merge pull request #140 from AlphaX-Projects/master
Merge arbitary resource cache
2023-11-05 17:07:24 -05:00
AlphaX-Projects
975b0a5f3d Bump version 4.3.2 2023-11-05 19:15:16 +01:00
kennycud
0d64e809c7 Ignoring test cases intended for Bitcoiny coins 2023-11-04 15:26:42 -07:00
kennycud
d4ef175c4d Added Address info support for Litecoin 2023-11-04 14:49:15 -07:00
AlphaX-Projects
3190fed3b9 Optimize imports 2023-11-04 17:11:07 +01:00
AlphaX-Projects
dfcc7bb370 Update UPnP 2023-11-04 16:44:18 +01:00
AlphaX-Projects
8bd9b41a13 Update dependencies 2023-11-04 16:36:59 +01:00
AlphaX-Projects
85300178c7 Update Advance Installer 2023-11-04 16:32:24 +01:00
AlphaX-Projects
e885a69688 Fix status 2023-11-04 16:30:28 +01:00
AlphaX-Projects
050886a496 Merge pull request #1 from AlphaX-Projects/arbitrary-resources-cache
Arbitrary resources cache
2023-11-04 16:21:24 +01:00
AlphaX-Projects
1aab7d8d8f Merge branch 'master' into arbitrary-resources-cache 2023-11-04 16:20:39 +01:00
QuickMythril
5febfaf3ee Merge pull request #135 from karl-dv/master
Manual review of "nl" translations
2023-10-30 22:03:59 -04:00
QuickMythril
9b64f12b7a Default minPeerVersion set to 4.3.1 2023-10-30 22:01:48 -04:00
karl-dv
6525cac66c Manual review of "nl" translations 2023-10-28 09:11:08 +02:00
QuickMythril
033b6adb72 Bump version to 4.3.1 2023-10-19 13:51:53 -04:00
QuickMythril
53c3f7899a Reward share limit activation timestamp set to 1698508800000 (Sat Oct 28 2023 16:00:00 UTC) 2023-10-19 13:48:33 -04:00
QuickMythril
f7f2d70b77 Merge pull request #134 from AlphaX-Projects/master
Final dependencies updates and fixes
2023-10-19 12:57:44 -04:00
AlphaX-Projects
5cbd4490cc Back to Bouncycastle 1.69 2023-10-18 17:29:10 +02:00
AlphaX-Projects
7103f41e36 Reorg pom update dependencies 2023-10-18 13:49:53 +02:00
AlphaX-Projects
2d599ec3c5 Summary of activity past 24 hours 2023-10-17 18:39:17 +02:00
AlphaX-Projects
e40d884c81 Merge pull request #10 from Qortal/master
Merge thread limit
2023-10-17 18:23:59 +02:00
QuickMythril
7cbeb00c1a Merge pull request #132 from Qortal/thread-limits
Added support for thread limits.
2023-10-17 12:10:35 -04:00
AlphaX-Projects
2af8199d9c Update dependencies 2023-10-16 17:27:14 +02:00
AlphaX-Projects
404c5d0300 Filter failed trade after 1 attempt 2023-10-15 17:12:02 +02:00
AlphaX-Projects
db7b17e52e Revert HSQLDB update 2023-10-13 15:57:43 +02:00
AlphaX-Projects
6d202b2b48 Create settings.json 2023-10-12 11:23:36 +02:00
AlphaX-Projects
499e2ac3f4 hide-failed-trade 2023-10-10 23:24:57 +02:00
CalDescent
15eefa4177 Improved batch reward logging and moved it to debug level 2023-10-06 11:03:06 +01:00
AlphaX-Projects
bf270a63ff Update bouncycastle 2023-09-24 15:19:07 +02:00
CalDescent
9454031b48 Added support for thread limits.
Default thread limits per message type can be specified in Settings.setAdditionalDefaults(), e.g

maxThreadsPerMessageType.add(new ThreadLimit("GET_ARBITRARY_DATA_FILE", 5));

These can also be overridden on a per-node basis in settings.json, e.g

"maxThreadsPerMessageType": [
    { "messageType": "GET_ARBITRARY_DATA_FILE", "limit": 3 },
    { "messageType": "GET_ARBITRARY_DATA_FILE_LIST", "limit": 3 }
]

settings.json values take priority, but any message types that aren't specified in settings.json will still be included from the Settings.java defaults. This allows single message types to be overridden in settings.json without removing the limits for all of the other message types.

Any messages that arrive are discarded if the node is already at the thread limit for that message type.

Warnings are now shown in the logs if the total number of active threads reaches 90% of the allocated thread pool size. Additionally, it can warn per message type by specifying a per-message-type warning threshold in settings.json, e.g

"threadCountPerMessageTypeWarningThreshold": 20

The above setting would warn in the logs if a single message type was consuming more than 20 threads at once, therefore making it a candidate to be limited in maxThreadsPerMessageType.

Initial values of maxThreadsPerMessageType are guesses and may need modifying based on real world results. Limiting threads may impact functionality, so this should be carefully tested.

Also be aware that the thread tracking may reduce network performance slightly, so be sure to test thoroughly on slower hardware.
2023-09-22 13:06:03 +01:00
AlphaX-Projects
cae3fdcb06 Update ElectrumX Servers 2023-09-22 11:12:50 +02:00
CalDescent
8b69b65712 Initial implementation of batch block reward distributions.
There are 3 values that need to be specified for this feature trigger:

- blockRewardBatchSize: the number of blocks per batch (1000 recommended).

- blockRewardBatchStartHeight: the block height at which the system switches to batch reward distributions. Must be a multiple of blockRewardBatchSize. Ideally this would be set to a block height that will occur at least 1-2 weeks after the release / auto update goes live.

- blockRewardBatchAccountsBlockCount: the number of blocks to include online accounts in the lead up to the batch reward distribution block (25 recommended).

Once active, rewards are no longer distributed every block and instead are distributed every 1000th block. This includes all fees from the distribution block and the previous 999 blocks.

The online accounts used for the distribution are taken from one of the previous 25 blocks (e.g. blocks xxxx975-xxxx999). The validation ensures that it is always the block with the highest number of online accounts in this range. If this number of online accounts is shared by multiple blocks, it will pick the one with the lowest height.

The idea behind 25 blocks is that it's low enough to reduce load in the other 975 blocks, but high enough for it to be extremely difficult for block signers to influence reward payouts.

Batch distribution blocks contain a copy of the online accounts from one of these 25 preceding blocks. However, online account signatures and online accounts timestamp are excluded to save space. The core will validate that the copy of online accounts is exactly matching those of the earlier validated block, and are therefore valid.

Fairly comprehensive unit tests have been written but this needs a lot more testing on testnets before it can be considered stable.

Future note: Once released, the online accounts mempow can optionally be scaled back so that it is no longer running 24/7, and instead is only running for 2-3 hours each day in the lead up to the batch distribution block. In each 1000 block cycle, the mempow would ideally start at least an hour before block xxxx975 is due to be minted, and continue for an hour after block xxxx000 (the reward distribution block). None of the ideas mentioned in this last paragraph are coded yet.
2023-09-17 15:04:37 +01:00
AlphaX-Projects
1fbb1659a3 Update dependencies 2023-09-13 10:43:05 +02:00
AlphaX-Projects
9959985a13 Update grpc 2023-09-09 10:56:17 +02:00
AlphaX-Projects
15c073edfe Merge branch 'Qortal:master' into master 2023-09-08 15:20:44 +02:00
QuickMythril
d453e80c6b Update fee in qdn script 2023-09-08 03:30:06 -04:00
QuickMythril
4fb799ba38 Merge pull request #125 from crowetic/master
ADD: (Tools) New features for qdn script
2023-09-08 03:26:34 -04:00
QuickMythril
29e56158ae Merge pull request #98 from Nuc1eoN/german-translation
ADD: (Language) German updates
2023-09-08 02:37:45 -04:00
QuickMythril
f74c9672f6 Merge branch 'test-german' into german-translation 2023-09-08 02:29:07 -04:00
AlphaX-Projects
a7ca306d1b Update json dependency 2023-09-07 16:54:09 +02:00
AlphaX-Projects
1a9087984a Swagger updates 2023-09-06 17:22:13 +02:00
AlphaX-Projects
94e9f86245 Version 3.8.0 get faster the NTP offset 2023-09-06 09:11:41 +02:00
AlphaX-Projects
bd05578035 Reverse need latest 1.7.x 2023-09-05 18:31:32 +02:00
AlphaX-Projects
c0ed4022a5 Update log4j2.properties 2023-09-05 15:27:33 +02:00
AlphaX-Projects
12dbff79c9 Update logging properties 2023-09-05 15:20:21 +02:00
AlphaX-Projects
6be3897fdb Update logging 2023-09-05 15:06:46 +02:00
AlphaX-Projects
43921e6ab8 Update jetty server 2023-09-05 13:44:54 +02:00
AlphaX-Projects
b92c7cc866 Update dependencies and ntp servers 2023-09-05 11:35:53 +02:00
AlphaX-Projects
053d56d01d Update hsqldb to 2.7.2 , ciyam at to 1.4.1 2023-09-05 09:40:01 +02:00
CalDescent
eb6a834fd9 Default minPeerVersion set to 4.3.0 2023-09-01 10:47:45 +01:00
CalDescent
a08f10ece3 Bump version to 4.3.0 2023-08-25 16:13:23 +01:00
CalDescent
ad51073f25 mempowTransactionUpdatesTimestamp set to Fri Sep 01 2023 09:00:00 UTC 2023-08-25 16:11:43 +01:00
CalDescent
5983e6ccc9 Merge branch 'add-create-bytes-endpoint' 2023-08-25 16:05:35 +01:00
CalDescent
760788e82b Updated tools. 2023-08-25 12:12:32 +01:00
CalDescent
d39131ffa9 Merge pull request #128 from kennycud/master
consolidated shared functionality into ACCTTests.java
2023-08-25 11:17:52 +01:00
CalDescent
3fbcc50503 Fixed deserialization issues with CreationRequest, added validation, and made small code tweaks for consistency with other endpoints. 2023-08-25 11:01:48 +01:00
kennycud
26ac7e5be5 consolidated shared functionality into ACCTTests.java 2023-08-24 15:47:45 -07:00
b051f9be89 fix strings in request 2023-08-23 14:41:50 -05:00
24ff3ab581 base64 to byte array 2023-08-22 22:46:58 -05:00
34382b6e69 add endpoint 2023-08-22 19:58:24 -05:00
CalDescent
086a9afa0e Testnet mempowTransactionUpdatesTimestamp set to 1692554400000 2023-08-20 16:52:49 +01:00
CalDescent
9428f9688f Merge pull request #126 from jschulthess/master
Fix website sub-folder rendering 404
2023-08-20 14:22:58 +01:00
CalDescent
9f4a0b7957 Merge pull request #127 from kennycud/master
consolidated shared functionality into BitcoinyTests.java
2023-08-20 14:22:18 +01:00
CalDescent
3c8574a466 Trade bot improvements
- Add async responder thread from @catbref which was previously only in place for LTC
- Log when computing mempow nonces
- Skip transaction import if signature is invalid
- Added checks to a message test to mimic trade bot transaction lookups
2023-08-20 12:42:49 +01:00
Jürg Schulthess
c428d7ce2e Merge branch 'Qortal:master' into master 2023-08-20 09:21:48 +02:00
CalDescent
4feb8f46c8 Updated testchain.json to use new unitFees structure. 2023-08-19 20:42:15 +01:00
CalDescent
1f7a60dfd8 Fixed long term issue preventing trade bot statuses from being logged correctly. 2023-08-19 20:27:05 +01:00
CalDescent
379b850bbd Increase default maxTransactionsPerBlock to 50 now that the process is faster. Can be increased much further in the future. 2023-08-19 14:12:12 +01:00
CalDescent
7bb61ec564 Update various transaction types at a future unknown timestamp.
- PUBLICIZE transactions are no longer possible.
- ARBITRARY transactions are now only possible using a fee.
- MESSAGE transactions only confirm when they are being sent to an AT. Messages to regular addresses (or no recipient) will expire after 24 hours.
- Difficulty for confirmed MESSAGE transactions increases from 14 to 16.
- Difficulty for unconfirmed MESSAGE transactions decreases from 14 to 12.
2023-08-19 13:57:26 +01:00
CalDescent
b0224651c2 Always use rate limiter for metadata requests, and sleep for a random amount of time between fetching metadata items. 2023-08-18 20:32:44 +01:00
CalDescent
6bf2b99913 Wait until unconfirmed transactions are considered to be valid before broadcasting them. 2023-08-18 15:37:24 +01:00
Jürg Schulthess
fd9d0c4e51 Merge branch 'Qortal:master' into master 2023-08-13 19:17:11 +02:00
kennycud
c2756a5872 Merge branch 'Qortal:master' into master 2023-08-13 05:32:47 -07:00
CalDescent
ecfb9a7d6d Bump version to 4.2.4 2023-08-12 19:34:59 +01:00
CalDescent
dd9d3fdb22 Add to the unconfirmed transactions cache when importing a transaction. 2023-08-12 19:27:19 +01:00
CalDescent
eea2884112 Bump version to 4.2.3 2023-08-12 18:59:00 +01:00
CalDescent
640a472876 Unit fee increase set to 1692118800000 2023-08-12 18:55:27 +01:00
CalDescent
e244a5f882 Removed legacy code that is no longer needed. 2023-08-12 16:09:41 +01:00
CalDescent
278dca75e8 Increase minimum fee at a future undecided timestamp. 2023-08-12 15:18:29 +01:00
CalDescent
897c44ffe8 Optimized transaction importing by using a temporary cache of unconfirmed transactions. 2023-08-12 11:14:21 +01:00
CalDescent
d9147b3af3 Cache the online accounts validation result, to speed up block minting. 2023-08-12 10:24:55 +01:00
kennycud
fe840bbf02 consolidated shared functionality into BitcoinyTests.java 2023-08-08 12:17:29 -07:00
CalDescent
133848ef50 Speed up status rebuilding by excluding transactions that aren't hosted locally by the node. 2023-08-05 19:42:30 +01:00
CalDescent
e44b38819e Specify table name in query, to avoid potential ambiguity with indexes. 2023-08-05 13:17:30 +01:00
CalDescent
eecd37d6bc Speed up finding arbitrary transactions. 2023-08-05 13:10:03 +01:00
CalDescent
a3ab5238d3 Merge branch 'master' into arbitrary-resources-cache 2023-08-05 13:01:00 +01:00
Jürg Schulthess
e7901a391f Merge branch 'Qortal:master' into master 2023-08-03 21:32:06 +02:00
CalDescent
9574100a08 Bump version to 4.2.2 2023-08-02 21:36:57 +01:00
CalDescent
528583fe38 Added logging relating to unconfirmed transactions. 2023-08-02 21:32:57 +01:00
CalDescent
33cfd02c49 Fixed issues in commit f5c8dfe 2023-08-02 21:13:33 +01:00
Jürg Schulthess
18e880158f Merge branch 'Qortal:master' into master 2023-08-01 11:09:06 +02:00
CalDescent
94d3664cb0 Bump version to 4.2.1 2023-07-31 19:30:45 +01:00
CalDescent
f5c8dfe766 Added maxTransactionsPerBlock setting (default 25) to reduce minting load on slower machines.
This is a short term limit, is well above current usage levels, and can be increased substantially in future once the block minter code has been improved.
2023-07-31 19:25:26 +01:00
Jürg Schulthess
a3526d84bc Merge branch 'Qortal:master' into master 2023-07-30 13:39:44 +02:00
CalDescent
f7e1f2fca8 Increased timeout for SEARCH_QDN_RESOURCES from 10 to 30 seconds. 2023-07-28 21:47:29 +01:00
CalDescent
811b647c88 Catch UnsupportedAddressTypeException and fall back to IPv4 binding. 2023-07-28 18:58:47 +01:00
CalDescent
3215bb638d More online accounts improvements 2023-07-22 10:44:41 +01:00
CalDescent
8ae7a1d65b Removed (Get)OnlineAccountsV1 and V2, as these are no longer used. 2023-07-21 14:28:47 +01:00
CalDescent
29dcd53002 Revert "Improved filtering of online accounts data."
This reverts commit c14fca5660.
2023-07-16 20:04:45 +01:00
CalDescent
62908f867a Bump version to 4.2.0 2023-07-16 19:09:08 +01:00
Jürg Schulthess
ff7a87ab18 Merge branch 'Qortal:master' into master 2023-07-10 20:53:24 +02:00
CalDescent
5f86ecafd9 Refactored developer proxy, and modified IPv6 fallback so that it only occurs on a connection failure. 2023-07-09 12:35:46 +01:00
CalDescent
9694094bbf Sanitize inputs used for the working path when building arbitrary data, and throw/handle an exception if it still doesn't work.
Should fix issue on Windows systems due to reserved characters in certain resource names.
2023-07-08 14:27:24 +01:00
CalDescent
d8237abde5 Don't update statuses when processing arbitrary transactions, to improve success rate and speed it up. 2023-07-08 14:16:50 +01:00
CalDescent
fe999a11f4 Include CANCEL_SELL_NAME transactions when performing a complete rebuild of names. 2023-07-08 11:06:33 +01:00
CalDescent
c14fca5660 Improved filtering of online accounts data. 2023-07-08 11:05:14 +01:00
Jürg Schulthess
cc8cdcd93a Fix website sub-folder rendering 404 2023-07-03 09:52:07 +02:00
CalDescent
537779b152 Use a separate repository instance when updating caches. 2023-07-02 17:49:49 +01:00
crowetic
ac433b1527 Add files via upload 2023-06-27 13:29:10 -07:00
CalDescent
fd8d720946 Added support for group encryption in service validation. 2023-06-23 13:32:23 +01:00
CalDescent
c0eeef546a Added support for group encryption in service validation. 2023-06-23 13:30:10 +01:00
CalDescent
badd6ad2b0 Added optional minLevel filter to GET /arbitrary/resources/search and the SEARCH_QDN_RESOURCES action. 2023-06-23 11:55:49 +01:00
CalDescent
b4794ada72 Merge branch 'master' into arbitrary-resources-cache 2023-06-17 14:17:49 +01:00
CalDescent
d628b3ab2a Renamed the "usePrefix" parameter to "includeResourceIdInPrefix", and slightly modified its functionality. 2023-06-17 13:04:46 +01:00
CalDescent
5928b54a33 Added developer QDN proxy.
This allows Q-Apps and websites to be developed and tested in real time, by proxying an existing webserver such as localhost:5173 from React/Vite. The proxy adds all QDN functionality to the existing server in real time. Needs UI integration before all features can be used.
2023-06-17 13:03:29 +01:00
CalDescent
4b04b99401 Discard changes before setting status. 2023-06-16 14:59:07 +01:00
CalDescent
7e872f7800 Update QDN cache when receiving a metadata file as part of a resource download. 2023-06-16 14:58:33 +01:00
QuickMythril
91dfc5efd0 Merge pull request #119 from QuickMythril/upgrade-tls
Upgraded to TLSv1.3
2023-06-14 10:25:25 -04:00
QuickMythril
1343a88ee3 Merge pull request #124 from QuickMythril/translate-jp
Added Japanese translations (Credit: R M)
2023-06-02 08:01:04 -04:00
QuickMythril
7f7b02f003 Updated Japanese translations (Credit: R M) 2023-06-02 06:17:48 -04:00
QuickMythril
5650923805 Merge pull request #123 from QuickMythril/2023-05-25
Added some fee info & Q-App support for foreign coins
2023-05-31 09:14:46 -04:00
CalDescent
5fb2640a3a Bump version to 4.1.3 2023-05-30 18:05:05 +01:00
CalDescent
66c91fd365 MIN_PEER_VERSION for handshake set to 4.1.1 2023-05-30 17:42:31 +01:00
CalDescent
bfc03db6a9 Default minPeerVersion set to 4.1.2 2023-05-30 17:41:39 +01:00
QuickMythril
a4bb445f3e Added Japanese translations for review 2023-05-29 04:23:22 -04:00
QuickMythril
27afcf12bf Prepared files for Japanese translations 2023-05-29 04:07:52 -04:00
CalDescent
707176a202 Improved detection of an existing arbitrary resources cache. 2023-05-27 11:30:19 +02:00
CalDescent
74a914367f Merge branch 'master' into arbitrary-resources-cache
# Conflicts:
#	src/main/java/org/qortal/controller/Controller.java
#	src/main/java/org/qortal/repository/RepositoryManager.java
#	src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java
2023-05-26 19:37:13 +02:00
CalDescent
eda6ab5701 Fixed some failing unit tests, and ignored some failing BTC ones that have been superseded by LTC. 2023-05-26 18:01:09 +02:00
QuickMythril
13da0e8a7a Adjusted fee info to long format 2023-05-25 17:26:29 -04:00
QuickMythril
d260c0a9a9 Updated info on foreign coin fees 2023-05-25 16:35:08 -04:00
QuickMythril
655073c524 Added 2m timeout for GET_WALLET_BALANCE action 2023-05-25 04:41:03 -04:00
crowetic
c8f3b6918f Update start.sh
Added better defaults for JVM_MEMORY_ARGS and description as to how and when to uncomment the line.
2023-05-24 16:20:05 -07:00
CalDescent
1565a461ac Bump version to 4.1.2 2023-05-24 20:01:18 +01:00
CalDescent
1f30bef4f8 defaultArchiveVersion set to 2 2023-05-24 19:54:36 +01:00
CalDescent
6f0479c4fc Default minPeerVersion set to 4.1.1 2023-05-24 19:47:37 +01:00
CalDescent
b967800a3e Default repositoryConnectionPoolSize set to 240 2023-05-24 19:47:25 +01:00
CalDescent
0b50f965cc Default maxNetworkThreadPoolSize set to 120 2023-05-24 19:47:10 +01:00
CalDescent
90f7cee058 Bump version to 4.1.1 2023-05-21 20:34:04 +01:00
CalDescent
947b523e61 Limit query to 100 so that it doesn't return endless amounts of transaction signatures.
Using a separate database method for now to reduce risk of interfering with other parts of the code which use it. It can be combined later when there is more testing time.
2023-05-21 20:33:33 +01:00
CalDescent
95d72866e9 Use a better method to detect if a transactions table in need of a rebuild.
Should handle cases where a previous rebuild didn't fully complete, or missed a block.
2023-05-21 20:06:09 +01:00
CalDescent
aea1cc62c8 Fixed off-by-one bug (correctly this time) 2023-05-21 20:02:58 +01:00
CalDescent
c763445e6e Log to console if an extra core restart is needed to complete the update process (this needed ins some cases after bootstrapping). 2023-05-21 19:51:22 +01:00
CalDescent
7a6b83aa22 Bump version to 4.1.0 2023-05-21 16:49:59 +01:00
CalDescent
ba555174ba Merge branch 'master' into block-sequence 2023-05-21 15:35:50 +01:00
CalDescent
3763035d4a Default recoveryModeTimeout increased to 24 hours for now.
It doesn't quite work as intended, so it's best that it doesn't interfere right away. 24 hours should be long enough for any issues to be manually resolved.
2023-05-21 15:34:27 +01:00
CalDescent
b1a904a3c7 MIN_PEER_VERSION set to 4.0.0 2023-05-21 15:26:49 +01:00
CalDescent
3c4c5a1457 Default minPeerVersion set to 4.0.0 2023-05-21 15:22:24 +01:00
CalDescent
648fa66f6a Increased default maxPeers to 40. 2023-05-21 15:22:00 +01:00
CalDescent
072aa469e3 Reduce default minBlockchainPeers to 3, ahead of the upcoming reshape. 2023-05-21 15:21:04 +01:00
CalDescent
2b2d6f4e52 Updated message. 2023-05-21 14:02:45 +01:00
CalDescent
c6456669e2 Don't allow core to start if transaction sequences haven't been rebuilt yet. 2023-05-21 12:33:37 +01:00
CalDescent
a74fa15d60 Missing import 2023-05-21 12:31:49 +01:00
CalDescent
68b99c8643 Update status when rebuilding transaction sequences. 2023-05-21 12:28:51 +01:00
CalDescent
b9015217de Fixed bug causing final block to be missed in the reshape. 2023-05-21 11:05:34 +01:00
CalDescent
e1043ceacb Fixed bug causing duplicate AT entries in local array. 2023-05-21 08:41:56 +01:00
CalDescent
8b51590844 Include AT transactions when rebuilding transaction sequences, as these aren't directly included in the block archive. 2023-05-20 20:54:22 +01:00
CalDescent
a8d92805f9 Added extra check for topOnly mode. 2023-05-20 11:33:43 +01:00
CalDescent
2cc5b90306 Merge branch 'master' into block-sequence 2023-05-14 17:28:27 +01:00
CalDescent
f5f82dc3f6 Fixed issues relating to using a separate repository instance when determining the latest status of a resource. 2023-05-13 19:20:18 +01:00
CalDescent
633f73aa86 Removed API key fields from documentation for methods that don't require an API key by default. 2023-05-13 16:32:33 +01:00
CalDescent
a49529ad9b Cache updating moved back to existing threads when processing or importing a transaction, to remove chances of queued updates being lost.
The dedicated cache manager thread is now used for metadata updates only. If metadata ever goes missing from the db, it would be straightforward to have a background thread that corrects any discrepancies between the filesystem and the db. Not adding that until it is needed.
2023-05-13 15:11:32 +01:00
CalDescent
f451bccbf6 Fixed bug causing descriptions to be truncated in the cache. 2023-05-13 14:54:00 +01:00
CalDescent
4cb755a2f1 Added GET /stats/supply/circulating API endpoint, to fetch total QORT minted so far. 2023-05-13 13:48:27 +01:00
CalDescent
5ed3237d2f Clear queue before exiting cache manager thread. 2023-05-13 13:36:40 +01:00
CalDescent
5c7d12f25e Fixed bug causing incorrect creation dates in the cache. 2023-05-13 12:27:56 +01:00
CalDescent
92119b5558 Increased per-name limit for followed names by 4x. 2023-05-12 20:14:14 +01:00
CalDescent
23d211836f Fixed case sensitivity issue when updating status in the cache. 2023-05-12 20:10:51 +01:00
CalDescent
36a731255a Automatically delete cached resources & metadata if there is no longer a latest transaction. 2023-05-12 20:08:53 +01:00
CalDescent
b661d39844 Cache updating moved to a dedicated thread.
Hopeful fix for serialization failures which occurred when updating from various different network threads.
2023-05-12 19:39:31 +01:00
CalDescent
7725c5e21f Always ignore unsupported services when building the cache. 2023-05-12 12:03:32 +01:00
CalDescent
21f01226e9 Merge branch 'master' into arbitrary-resources-cache 2023-05-12 11:50:24 +01:00
CalDescent
8a1bf8b5ec Return full name data in GET /names. 2023-05-12 11:41:15 +01:00
CalDescent
f8233bd05b Added optional after parameter to GET /names. 2023-05-12 11:41:00 +01:00
CalDescent
29480e5664 Added SEARCH_NAMES Q-App action. 2023-05-12 11:17:09 +01:00
CalDescent
5a873f9465 Added prefix parameter to GET /names/search. 2023-05-12 11:11:34 +01:00
CalDescent
dc1289787d Ignore per-name limits when using storagePolicy ALL. 2023-05-12 10:12:38 +01:00
CalDescent
ba4866a2e6 Added GET /crosschain/tradeoffers/hidden endpoint, to show offers that are currently being hidden.
This uses the maxTradeOfferAttempts setting, so modifying this setting will affect the number of offers that are returned.
2023-05-12 10:01:38 +01:00
CalDescent
2cbc5aabd5 Added maxTradeOfferAttempts setting (default 3).
Offers with more than 3 failures will be hidden from the API and websocket, to prevent unbuyable offers from staying in the order books and continuously failing. maxTradeOfferAttempts can be optionally increased on a node to show more trades that would otherwise be hidden.
2023-05-12 09:59:30 +01:00
QuickMythril
e3be43a1e6 Changed get name API call to use reduced name 2023-05-11 12:31:00 -04:00
QuickMythril
1e10bcf3b0 Merge branch 'Qortal:master' into upgrade-tls 2023-05-09 15:38:20 -04:00
QuickMythril
a575ea4423 Merge pull request #120 from QuickMythril/get-votes-api
Created get votes API call
2023-05-09 15:34:54 -04:00
QuickMythril
3e45948646 Added get votes option to return only counts 2023-05-08 23:41:31 -04:00
QuickMythril
49c0d45bc6 Added count to get votes API call 2023-05-08 23:26:23 -04:00
QuickMythril
cda32a47f1 Added API call to get votes 2023-05-08 20:23:54 -04:00
CalDescent
49063e54ec Bump version to 4.0.3 2023-05-08 19:18:38 +01:00
CalDescent
df3c68679f Log the action to the console, instead of the entire event. 2023-05-08 14:43:00 +01:00
CalDescent
c210d63c40 Added "mode" parameter to GET /arbitrary/resources/search, with possible values of LATEST, ALL.
By default, only the latest resource is returned for a name/service combination. All identifiers can be optionally returned by setting `mode` to "ALL".

More search modes can be added in the future, for instance "RELEVANT" or "POPULAR" (these are just ideas, and are not currently supported).
2023-05-08 13:41:23 +01:00
CalDescent
0ec661431c Added optional "before" and "after" params to GET /arbitrary/resources/search 2023-05-08 12:46:15 +01:00
CalDescent
8fa344125c Fixed issue updating cache when receiving metadata via the network. 2023-05-08 12:34:26 +01:00
CalDescent
2fd5bfb11a Support title/description metadata searching in GET /arbitrary/resources/search
"query" searches name, identifier, title and description fields
"title" searches title only
"description" searches description only

All support "&prefix=true", to indicate searching by prefix only.
2023-05-08 12:34:26 +01:00
CalDescent
cdcb268bd9 Exclude status if includeStatus != true 2023-05-08 12:34:26 +01:00
CalDescent
d03a2d7da9 Resource statuses moved to the db, so they don't have to be calculated on demand for every API call. 2023-05-08 12:34:26 +01:00
CalDescent
961aa9eefd Show splash screen when building QDN cache. 2023-05-08 12:34:26 +01:00
CalDescent
865d3d8aff Fixed ordering, to keep consistency with existing approach. 2023-05-08 12:34:26 +01:00
CalDescent
c0f29f848f Fixed more bugs. 2023-05-08 12:34:26 +01:00
CalDescent
94f4c501fa Update caches where possible when processing arbitrary transactions. 2023-05-08 12:34:26 +01:00
CalDescent
200b0f3412 Added POST /arbitrary/resources/cache/rebuild endpoint to allow a rebuild of the cache. 2023-05-08 12:34:25 +01:00
CalDescent
eb7a29dd2e Fixed bugs. 2023-05-08 12:34:25 +01:00
CalDescent
9dba4b2968 Initial attempt at a database cache to hold arbitrary resources and metadata. 2023-05-08 12:34:25 +01:00
CalDescent
81788610c4 Merge branch 'master' of github.com:Qortal/qortal 2023-05-08 12:18:34 +01:00
CalDescent
fc10b61193 Fixed slow validation issue caused by loading the entire resource into memory. 2023-05-08 12:17:44 +01:00
CalDescent
05b4ecd4ed Updated documentation. 2023-05-08 12:16:17 +01:00
CalDescent
aba589c0e0 Added optional "build" parameter to GET_QDN_RESOURCE_STATUS.
This triggers an async build when checking the status.
2023-05-08 12:15:53 +01:00
CalDescent
c682fa89fd Avoid duplicate concurrent QDN builds. 2023-05-08 12:14:00 +01:00
CalDescent
21d1750779 Added more debug logging when building resources. 2023-05-08 12:13:12 +01:00
CalDescent
923e90ebed Fixed occasional NPE 2023-05-08 12:12:40 +01:00
catbref
9490c62242 Improved tx.pl that supports local signing via openssl and "deploy_at" transaction type + other minor fixes 2023-05-08 12:07:02 +01:00
CalDescent
c941bc6024 Catch and log all exceptions when publishing data. 2023-05-07 11:19:42 +01:00
CalDescent
0acf0729e9 Bump version to 4.0.2 2023-05-06 15:10:46 +01:00
CalDescent
1f77ee535f Added link to example Q-App projects. 2023-05-06 12:16:59 +01:00
CalDescent
b693a514fd Fixed warnings, and other improvements. 2023-05-06 12:13:41 +01:00
CalDescent
b571931127 Fixed formatting of services list 2023-05-05 22:35:19 +01:00
CalDescent
92b983a16e Q-Apps documentation updates. 2023-05-05 22:25:12 +01:00
CalDescent
3f71a63512 Increased timeout for other new actions. 2023-05-05 18:30:14 +01:00
CalDescent
86b5bae320 Set timeout of PUBLISH_MULTIPLE_QDN_RESOURCES to 60 mins. 2023-05-05 13:22:14 +01:00
CalDescent
3775135e0c Added helper methods to fetch lists of private or public service objects.
These can ultimately be used to help inform the cleanup manager on the best order to delete files when the node runs out of space. Public data should be given priority over private data (unless the node is part of a data market contract for that data - this isn't developed yet).
2023-05-05 12:39:11 +01:00
CalDescent
c172a5764b Added _PRIVATE services, to allow for publishing/validation of encrypted data.
New additions:

QCHAT_ATTACHMENT_PRIVATE
ATTACHMENT_PRIVATE
FILE_PRIVATE
IMAGE_PRIVATE
VIDEO_PRIVATE
AUDIO_PRIVATE
VOICE_PRIVATE
DOCUMENT_PRIVATE
MAIL_PRIVATE
MESSAGE_PRIVATE
2023-05-05 12:26:18 +01:00
CalDescent
1a5e3b4fb1 Added GET /names/search endpoint, to search names via case insensitive, partial name matching. 2023-05-05 11:24:52 +01:00
CalDescent
f39b6a15da Fixed refresh bug on Windows. 2023-05-05 11:03:13 +01:00
CalDescent
2dfee13d86 Remove all backslashes from vars in HTML parser (correct order this time) 2023-05-03 19:44:54 +01:00
CalDescent
b9d81645f8 Revert "Remove all backslashes from vars in HTML parser."
This reverts commit 9547a087b2.
2023-05-03 19:40:17 +01:00
CalDescent
9547a087b2 Remove all backslashes from vars in HTML parser. 2023-05-03 19:38:31 +01:00
CalDescent
e014a207ef Escape all vars added by HTML parser 2023-05-03 19:28:26 +01:00
CalDescent
611240650e Added GET /chat/messages/count endpoint, which is identical to /chat/messages but returns a count of the messages rather than the messages themselves. 2023-05-03 19:27:59 +01:00
CalDescent
c71dce92b5 Bump version to 4.0.1 2023-05-01 19:34:01 +01:00
CalDescent
34c3adf280 Limit MAIL and MESSAGE to 1MB. 2023-04-29 19:04:17 +01:00
CalDescent
95a1c6bf8b Added "encoding" parameter to the SEARCH_CHAT_MESSAGES action. 2023-04-29 17:48:58 +01:00
CalDescent
36e944d7e2 Added MAIL and MESSAGE services. 2023-04-29 17:45:38 +01:00
CalDescent
f044166b81 More qdnBase improvements, to hopefully handle all cases correctly. 2023-04-29 17:13:50 +01:00
CalDescent
aed1823afb Added support of simple Range headers when requesting QDN data. 2023-04-28 20:36:06 +01:00
CalDescent
6dfaaf0054 Set charset to UTF-8 in various places that bytes are converted to a string. 2023-04-28 13:06:29 +01:00
CalDescent
45bc2e46d6 Improved metadata trimming, to better handle multibyte UTF-8 characters. 2023-04-28 12:48:38 +01:00
CalDescent
46e2e1043d Fixed issue with <base href> introduced in v4.0.0 2023-04-28 12:18:27 +01:00
CalDescent
a3518d1f05 Revert "Fixed bug with base path."
This reverts commit ce52b39495.
2023-04-28 12:13:31 +01:00
CalDescent
0a1ab3d685 Added GET_QDN_RESOURCE_METADATA action. 2023-04-28 10:57:04 +01:00
CalDescent
5dbacc4db3 Added "Accept-Ranges" header when serving arbitrary data.
Allows for video seeking when using URL playback, even though the Range header isn't implemented yet. This could be heavily optimized by adding full support of the Range/Content-Range headers, however this is still a big step forward as it allows for (inefficient) seeking.
2023-04-28 10:12:16 +01:00
CalDescent
1ce2dcfb2b Fixed bug which prevented qortal:// URLs from working properly in most cases. 2023-04-25 08:33:33 +01:00
CalDescent
ed6333f82e Allow for faster and more frequent retries when QDN data fails to be retrieved (thanks to suggestions from @xspektrex) 2023-04-23 19:14:28 +01:00
CalDescent
f27c9193c7 Auto delete any metadata files that are unreadable (e.g. due to being empty, or invalid JSON). 2023-04-23 11:30:42 +01:00
CalDescent
e48529704c Bump version to 4.0.0 2023-04-22 16:08:09 +01:00
CalDescent
53508f9298 Fixed problems in last commit. 2023-04-22 11:33:59 +01:00
CalDescent
33aeec7e87 Added various new service types, in preparation for Q-Apps release. 2023-04-22 11:00:21 +01:00
QuickMythril
8f847d3689 Upgraded to TLSv1.3 2023-04-21 19:30:29 -04:00
CalDescent
16dc23ddc7 Added new actions to gateway handler. 2023-04-21 21:45:16 +01:00
CalDescent
e80494b784 Fixed unit test. 2023-04-21 20:22:18 +01:00
CalDescent
111ec3b483 Fixed typo 2023-04-21 20:05:24 +01:00
CalDescent
db4a9ee880 Return "Resource does not exist" error if requesting a non-existent resource via GET_QDN_RESOURCE_URL. 2023-04-21 19:50:01 +01:00
CalDescent
b1ebe1864b Fixed bug in error handling. 2023-04-21 19:27:24 +01:00
CalDescent
3c251c35ea Fixed divide by zero error in GET /arbitrary/resource/status/* 2023-04-21 18:21:41 +01:00
CalDescent
4954a1744b Fixed case sensitivity bugs. 2023-04-21 17:47:29 +01:00
QuickMythril
a7bbad17d7 Merge pull request #118 from QuickMythril/temp-test
Added missing parameter to test
2023-04-21 11:06:09 -04:00
QuickMythril
8ca9423c52 Added missing parameter to test 2023-04-21 10:58:09 -04:00
CalDescent
32b9b7e578 Use a temporary file when reading on-chain data. 2023-04-21 13:59:29 +01:00
CalDescent
f045e10ada Removed all case sensitivity when searching names. 2023-04-21 12:56:15 +01:00
CalDescent
560282dc1d Added "exactMatchNames" parameter to GET /arbitrary/resources/search 2023-04-21 12:55:59 +01:00
CalDescent
9cd6372161 Improved GET /admin/settings/{setting} further, in order to support all settings (fixes ones such as bitcoinNet). 2023-04-21 12:06:16 +01:00
CalDescent
2370a67b8a Merge branch 'master' into q-apps 2023-04-21 11:07:01 +01:00
CalDescent
0993903aa0 Added GET /settings/{setting} endpoint
Based on work by @QuickMythril, but modified to be generic.
2023-04-21 11:03:24 +01:00
CalDescent
f5e9b91d6b Merge pull request #116 from QuickMythril/restart
Added API call for restarting node
2023-04-21 10:27:58 +01:00
CalDescent
7fe507a497 Merge pull request #117 from QuickMythril/rm-swagger-check
Removed 3rd-party swagger server validation
2023-04-21 10:26:19 +01:00
CalDescent
10f12221c9 Fixed exception in readJson(), and removed some duplicated code. 2023-04-21 09:42:04 +01:00
QuickMythril
85980e4cfc Removed 3rd-party swagger server validation 2023-04-20 16:41:47 -04:00
QuickMythril
7bb6b84e86 Added API call for restarting node 2023-04-20 16:23:57 -04:00
CalDescent
dc25d33739 Merge branch 'master' into q-apps 2023-04-19 20:57:31 +01:00
CalDescent
358e67b050 Added "bindAddressFallback" setting, which defaults to "0.0.0.0".
Should fix problems on systems unable to use IPv6 wildcard (::) for listening, and avoids having to manually specify "bindAddress": "0.0.0.0" in settings.json.
2023-04-19 20:56:47 +01:00
CalDescent
8331241d75 Bump version to 3.9.1 2023-04-18 19:01:45 +01:00
CalDescent
e041748b48 Improved name rebuilding code, to handle some more complex scenarios. 2023-04-16 13:59:25 +01:00
CalDescent
06691af729 Merge pull request #113 from QuickMythril/drew-api
Added API calls to get node settings & get, create, and vote on polls
2023-04-16 11:43:31 +01:00
CalDescent
cfe6dfcd1c If nameFilter contains an empty or nonexistent list, return an empty array. 2023-04-15 18:27:55 +01:00
CalDescent
3f00cda847 "nameListFilter" added to LIST_QDN_RESOURCES and SEARCH_QDN_RESOURCES Q-Apps actions. 2023-04-15 16:02:25 +01:00
CalDescent
a286db2dfd "namefilter" param in GET /arbitrary/resources/search is now exact match, which makes more sense when filtering results by names in a list. 2023-04-15 15:55:52 +01:00
CalDescent
28bd4adcd2 Removed GET /arbitrary/resources/names endpoint, as it's unused and doesn't scale well. 2023-04-15 15:42:47 +01:00
CalDescent
61b7cdd025 Added "followedonly" and "excludeblocked" params to GET /arbitrary/resources and GET /arbitrary/resources/search, as well as LIST_QDN_RESOURCES and SEARCH_QDN_RESOURCES Q-Apps actions. 2023-04-15 15:24:10 +01:00
CalDescent
250245d5e1 Added new list management actions to Q-Apps documentation. 2023-04-15 14:34:30 +01:00
CalDescent
0258d2bcb6 Fixed layout issues recently introduced in documentation. 2023-04-15 14:31:41 +01:00
QuickMythril
735de93848 Removed internal use parameter from API endpoint 2023-04-15 09:25:28 -04:00
QuickMythril
57485bfe36 Removed check from poll tx that creator is owner 2023-04-15 09:11:27 -04:00
CalDescent
ed05560413 Gateway auth alert box replaced with a modal overlay in the lower right hand corner of the screen. 2023-04-15 10:11:33 +01:00
CalDescent
892b667f86 Fixed console errors seen in certain cases. 2023-04-15 09:57:26 +01:00
CalDescent
ea7a2224d3 Allow the name of a list to be specified as a "namefilter" param in GET /arbitrary/resources/search. Any names in the list will be included in the search (same as if they were specified manually via &name=). 2023-04-14 17:44:06 +01:00
CalDescent
20893879ca Allow multiple name parameters to optionally be included in GET /arbitrary/resources/search
Also updated SEARCH_QDN_RESOURCES action, to allow multiple names to be optionally specified via the "names" parameter.
2023-04-14 17:17:05 +01:00
CalDescent
b08e845dbb Updated docs to include sending of foreign coins 2023-04-14 16:24:27 +01:00
CalDescent
e60cd96514 Fixed occasional NPE seen in ArbitraryDataFileMessage 2023-04-14 11:02:27 +01:00
CalDescent
e2a2a1f956 Fixed bug with GET_QDN_RESOURCE_URL action. 2023-04-11 19:03:56 +01:00
CalDescent
7f53983d77 Added support for hash routing in URL shown in address bar. 2023-04-09 18:21:19 +01:00
CalDescent
ce52b39495 Fixed bug with base path. 2023-04-09 17:55:41 +01:00
CalDescent
3296779125 Update address bar when navigating within an app. 2023-04-09 17:11:20 +01:00
CalDescent
3dcd9d237c Added "_qdnBase" global javascript var, for apps to use as a basename / path prefix. 2023-04-07 19:03:28 +01:00
QuickMythril
23ec71d7be Renamed API calls from "voting" to "polls" 2023-04-06 04:48:18 -04:00
QuickMythril
5bbde4dcdb Added API calls to get polls & node settings 2023-04-06 04:41:51 -04:00
QuickMythril
dc2da8b283 Merge pull request #37 from DrewMPeacock/VotingTransactions
Fix up CREATE_POLL and VOTE_ON_POLL transactions to process and valid…
2023-04-06 03:35:58 -04:00
QuickMythril
f3772d19f5 Merge pull request #36 from DrewMPeacock/master
Add API handles to build CREATE_POLL and VOTE_ON_POLL transactions.
2023-04-06 03:34:50 -04:00
CalDescent
35def54ecc Added support for multiple block/follow lists.
Any list with the following prefix will be used in block/follow logic:

blockedNames
blockedAddresses
followedNames

For instance, any names in a list named "blockedNames_CustomBlockList" would also be blocked, along with those in the standard "blockedNames" list.

This will ultimately allow apps to offer custom block/follow lists to users (once list functionality is added to the Q-Apps API).
2023-04-02 14:42:49 +01:00
CalDescent
2086a2c476 Moved block/follow utility methods to a new ListUtils class 2023-04-02 10:58:16 +01:00
CalDescent
4835e5732d Fixed issue which caused UI to lock up when using qortalRequest() 2023-04-02 10:06:02 +01:00
CalDescent
d831972005 Fixed NPE in isMetadataEqual() 2023-03-31 18:41:58 +01:00
CalDescent
f6914821d3 Always use PUT for on-chain data. 2023-03-31 18:30:05 +01:00
CalDescent
073d124aef Add "percentLoaded" to resource statuses. 2023-03-31 17:27:04 +01:00
CalDescent
a83e332c11 Major upgrade of arbitrary data functionality, to support on-chain data for small payloads.
Max size for on-chain data is 239 bytes, due to 16-byte IV. Must be a single file resource, without .qortal folder.
2023-03-31 15:53:08 +01:00
CalDescent
7deb9328fa Don't delete metadata when deleting a resource from the Data Management screen. 2023-03-31 15:36:51 +01:00
CalDescent
e598d7476b Updated documentation, to discourage custom timeouts. 2023-03-31 14:58:30 +01:00
CalDescent
85735fabb2 Block external links. 2023-03-31 13:03:46 +01:00
CalDescent
7392082875 Ignore "qdnAuthBypassEnabled" setting when in gateway mode. 2023-03-31 09:48:25 +01:00
CalDescent
88f8041b05 Updated testnet documentation. 2023-03-31 09:47:38 +01:00
CalDescent
3109c3bb16 Increased timeouts to 5 mins for various actions that require the user to confirm. 2023-03-29 18:47:03 +01:00
CalDescent
8d462dedfa Added routing info to documentation. 2023-03-28 19:18:21 +01:00
CalDescent
fdd9741936 Documentation updates. 2023-03-26 21:19:10 +01:00
CalDescent
929d0ac897 Added "name" filter to GET /arbitrary/resources and LIST_QDN_RESOURCES. 2023-03-24 10:31:35 +00:00
CalDescent
952d18390b Q-Apps documentation updates 2023-03-24 09:53:49 +00:00
CalDescent
bc026d9d1c Merge branch 'master' into q-apps 2023-03-22 21:48:18 +00:00
CalDescent
ea2577d1c3 Include "created" and "updated" timestamps in GET /arbitrary/resource/* API endpoints. 2023-03-22 21:44:32 +00:00
CalDescent
c78593cf15 Reduced MAX_DESCRIPTION_LENGTH from 500 to 240, in preparation for upcoming arbitrary db reshape. 2023-03-22 21:40:26 +00:00
CalDescent
de4523c34e Added support for custom URL routing when using the APP service.
Unhandled requests (where no file exists) are now forwarded to the index file, to allow for custom routing in the app. This applies to the APP service only. For the WEBSITE and other services, unhandled requests will return a 404. In future, we may be able to allow websites to opt in to URL routing too, and maybe even allow both services to specify custom routing rules in a file.
2023-03-22 21:38:02 +00:00
CalDescent
b08329dcf1 Bump version to 3.9.0 2023-03-22 20:14:11 +00:00
CalDescent
668be633c4 arbitraryOptionalFeeTimestamp set to Friday, 31st March 2023 at 16:00:00 2023-03-22 19:58:51 +00:00
CalDescent
ea6225ab9a Include "mimeType" in metadata for single file resources (but only when a metadata file would have otherwise been created). 2023-03-19 16:27:57 +00:00
CalDescent
055b66e835 Fixed gateway prefix bugs. 2023-03-19 11:02:49 +00:00
CalDescent
2a7a2d3220 Added gateway-specific Q-Apps handler. For now, just show a warning alert if an app requires authentication / interactive features. 2023-03-19 10:41:37 +00:00
CalDescent
73a7c1fe7e More improvements to Service handling. 2023-03-19 10:18:13 +00:00
CalDescent
2848ae695c More improvements to Service handling. 2023-03-19 10:17:56 +00:00
CalDescent
713fd4f0c6 Added GET_QDN_RESOURCE_PROPERTIES Q-App action. 2023-03-19 08:56:06 +00:00
CalDescent
519bb10c60 Updated docs for PUBLISH_QDN_RESOURCE, to include "filename" parameter. 2023-03-18 18:15:28 +00:00
CalDescent
3a64336d9f If the MIME type can't be determined from the file's contents, fall back to using the filename. 2023-03-18 17:57:07 +00:00
CalDescent
5ecc633fd7 GET /arbitrary/resource/properties/{service}/{name}/{identifier} can now extract the MIME type from the file's contents as an alternative to using the filename. 2023-03-18 17:50:13 +00:00
CalDescent
1b9afce21f Filename API renamed to GET /resource/properties/{service}/{name}/{identifier}.
Now returns filename, size, and mimeType where available.
2023-03-18 16:39:23 +00:00
CalDescent
f9f34a61ac Treat service as an int in other parts of ArbitraryTransactionData too 2023-03-18 15:44:01 +00:00
CalDescent
46b225cdfb Treat service as an int in other parts of ArbitraryTransactionData too 2023-03-18 15:18:36 +00:00
CalDescent
4ce3b2a786 Added GET /resource/filename/{service}/{name}/{identifier} endpoint.
This allows the filename of single file resources to be returned via the API. Useful to help determine to file format of the data.
2023-03-18 15:16:41 +00:00
CalDescent
87ed49a2ee Added optional "filename" parameter when publishing data from a string or base64-encoded string.
This causes the data to be stored with the requested filename, instead of generating a random one. Also, randomly generated filenames now use a timestamp instead of a random number.
2023-03-18 15:11:53 +00:00
CalDescent
a555f503eb Treat service as an int in ArbitraryTransactionData 2023-03-18 10:41:53 +00:00
CalDescent
50780aba53 Set max size of APP service to 50MB. 2023-03-18 10:41:14 +00:00
CalDescent
2bee3cbb5c Treat service as an int in ArbitraryTransactionData 2023-03-18 10:40:27 +00:00
CalDescent
534a44d0ce Fixed bugs with URL building. 2023-03-17 22:58:14 +00:00
CalDescent
469c1af0ef Added new search features to the SEARCH_QDN_RESOURCES action.
Existing action renamed to LIST_QDN_RESOURCES, which is an alternative for listing QDN resources without using a search query.
2023-03-17 22:11:34 +00:00
CalDescent
5656100197 Added "identifier", "name", and "prefix" parameters to GET /arbitrary/resources/search endpoint.
- "identifier" is an alternative to "query" that will search identifiers only.
- "name" is an alternative to "query" that will search names only.
- "query" remains the same as before - it searches both name and identifier fields.
- "prefix" is a boolean, and when true it will only match the beginning of each field. Works with "identifier", "name", and "query" params.
2023-03-17 19:47:57 +00:00
CalDescent
d9cac6db39 Allow "data:" URLs to be played in app/website media players.
E.g: src="data:video/mp4;base64,VideoContentEncodedInBase64GoesHere"
2023-03-17 19:33:41 +00:00
CalDescent
98b0b1932d Merge branch 'master' into q-apps 2023-03-17 13:17:47 +00:00
CalDescent
9968865d0e Updated parsing of "encoding" in websockets, for consistency with other params. 2023-03-17 13:17:23 +00:00
CalDescent
05eb337367 Added optional limit/offset/reverse query string params to GET /websockets/chat/messages.
Without this, the websocket returns all messages on connection, which is very time consuming.
2023-03-17 13:15:57 +00:00
CalDescent
5386db8a3f Added ping/pong functionality to CHAT websockets. 2023-03-17 13:11:01 +00:00
CalDescent
edae7fd844 Added optional "encoding" query string param for various chat APIs and websockets, as base58 is too slow for the amount of data it is now processing.
Usage:
Add `encoding=BASE64` query string parameter to opt in to base64 encoding of returned chat data. Defaults to BASE58 for backwards support.

Compatible endpoints:
GET /chat/messages
GET /chat/message/{signature}
GET /chat/active/{address}
GET /websockets/chat/active/*
GET /websockets/chat/messages
2023-03-17 12:46:14 +00:00
CalDescent
4840804d32 Fixed qdn utility usage docs. 2023-03-17 10:22:26 +00:00
CalDescent
a4551245cb Improved error logging in BlockArchiveUtils.importFromArchive() 2023-03-12 19:08:57 +00:00
CalDescent
e4f45c1a70 Break out of orphan loop when stopping. 2023-03-12 19:08:07 +00:00
CalDescent
bc44b998dc The transaction sequences reshape now fetches transactions from the archive.
This is required as it's the only place that holds the original order of each block's transactions. We cannot sort them, because the comparator function for transactions has some dependencies on the existing order for AT transactions. As a result, topOnly nodes cannot perform this reshape, and will be unable to run this version.
2023-03-10 21:29:35 +00:00
CalDescent
b89a35ac69 Merge branch 'master' into block-sequence
# Conflicts:
#	src/main/java/org/qortal/controller/Controller.java
#	src/main/java/org/qortal/repository/RepositoryManager.java
2023-03-10 19:52:05 +00:00
CalDescent
b5cb5f1da3 Fixed bug causing cache invalidation to be skipped, due to incorrect message reuse.
The "Data Management" screen should now update correctly without a core restart.
2023-03-10 19:46:58 +00:00
CalDescent
101023ba1d Updated link. 2023-03-10 16:39:14 +00:00
CalDescent
ed73162881 Merge branch 'master' into q-apps 2023-03-10 15:41:31 +00:00
CalDescent
0388626e42 Use a lower file size target (10MB instead of 100MB) when using archive V2, as the average block size is over 90% smaller. 2023-03-10 15:41:07 +00:00
CalDescent
c5c0dcf0f2 Testnet arbitraryOptionalFeeTimestamp set to Sun Mar 12 2023 at 12:00:00 UTC 2023-03-10 14:59:33 +00:00
CalDescent
384f592f59 Added testnet files to testnet/ directory.
This will be maintained with future feature triggers etc.
2023-03-10 14:59:27 +00:00
CalDescent
1528e05e0b Testnet arbitraryOptionalFeeTimestamp set to Sun Mar 12 2023 at 12:00:00 UTC 2023-03-10 14:29:52 +00:00
CalDescent
82c66c0555 Added testnet files to testnet/ directory.
This will be maintained with future feature triggers etc.
2023-03-10 14:28:13 +00:00
CalDescent
b5ce8d5fb3 Merge branch 'master' into q-apps
# Conflicts:
#	src/main/java/org/qortal/api/resource/ArbitraryResource.java
2023-03-10 14:03:08 +00:00
CalDescent
b4a736c5d2 Added optional "sender" filter to GET /chat/messages 2023-03-10 13:53:46 +00:00
CalDescent
4afbca7ed2 Merge branch 'rebuild-archive' 2023-03-10 11:50:09 +00:00
CalDescent
44aa0a6026 Catch ArithmeticException in block minter, so that it retries instead of giving up completely. 2023-03-10 10:00:30 +00:00
CalDescent
b1452bddf3 Added BlockArchiveV2 tests, and updated the V1 tests now that we no longer support bulk archiving/pruning 2023-03-06 17:17:55 +00:00
CalDescent
96ac883515 Throw exception and break out of loop if archive rebuilding fails 2023-03-06 14:40:17 +00:00
CalDescent
b6803490b9 Archive version is now loaded from the version of block 2 in the existing archive, or "defaultArchiveVersion" in settings if not available (default: 1). 2023-03-06 14:13:58 +00:00
CalDescent
3739920ad3 Added support for an optional fee in arbitrary transactions, to give the option for data to be published instantly (i.e. no proof of work / mempow required when fee is sufficient).
Takes effect at a future undecided timestamp.
2023-03-06 13:17:48 +00:00
CalDescent
7f21ea7e00 Added new bootstrap host 2023-03-05 13:16:58 +00:00
CalDescent
83b0ce53e6 Fixed bug in JSON validation. 2023-03-05 13:16:08 +00:00
CalDescent
d6ab9eb066 Rework of service validation, to allow a service to be specified as a single file resource.
This removes some complexity and duplication from custom validation functions. Q-Chat QDN functionality will need a re-test.
2023-03-05 11:39:53 +00:00
CalDescent
ac60ef30a3 Added JSON service, with a maximum size of 25KB, and a requirement that the data must be valid JSON. 2023-03-05 10:51:26 +00:00
CalDescent
94f14a39e3 Ensure theme is transferred when visiting a linked resource. 2023-03-03 18:16:35 +00:00
CalDescent
4b7844dc06 Pass the UI's theme to Q-Apps themselves, so they have the option of adapting to the user's theme.
Variable name is _qdnTheme, and possible values are "dark" or "light"
2023-03-03 17:55:46 +00:00
CalDescent
c40d0cc67b Same fix again but for multi file resources too. 2023-03-03 17:47:14 +00:00
CalDescent
3318093a4f Fixed preview functionality for resources other than websites/apps. 2023-03-03 17:33:15 +00:00
CalDescent
cf0681d7df Only rebuild if transaction has a name. 2023-03-03 17:10:45 +00:00
CalDescent
7d7cea3278 Only rebuild if transaction has a name. 2023-03-03 17:10:14 +00:00
CalDescent
7d38fa909d Rebuild name in ArbitraryTransaction.preProcess() 2023-03-03 16:15:10 +00:00
CalDescent
0b05de22a0 Rebuild name in ArbitraryTransaction.preProcess() 2023-03-03 16:14:43 +00:00
CalDescent
308196250e Updated documentation. 2023-03-03 16:13:49 +00:00
CalDescent
b254ca7706 Added support for optional Base64 encoding in FETCH_QDN_RESOURCE. 2023-03-03 15:39:37 +00:00
CalDescent
9ea2d7ab09 Updated documentation to remove an action that isn't supported in Q-Apps v1. 2023-03-03 14:24:10 +00:00
CalDescent
d166f625d0 Rework of preview mode.
All /arbitrary endpoints responsible for publishing data now support an optional "preview" query string parameter. If true, these endpoints will return a URL path to open the preview, rather than returning transaction bytes.
2023-03-03 14:20:45 +00:00
CalDescent
8e2dd60ea0 Increased default timeout for GET_USER_ACCOUNT from 30 seconds to 1 hour, to give the user more time to grant permissions. 2023-03-03 13:20:17 +00:00
CalDescent
d51f9368ef Fixed bug in HTML parser 2023-03-03 12:39:44 +00:00
CalDescent
b17035c864 Escape QDN vars and prefix with underscores. 2023-03-03 11:57:07 +00:00
CalDescent
fa14568cb9 Fixed issue causing "totalChunkCount" to exclude the metadata file in some cases.
ArbitraryDataFile now has a fileCount() method which returns the total number of files associated with that piece of data - i.e. chunks, metadata, and the complete file in cases where it isn't chunked.
2023-03-03 10:42:43 +00:00
CalDescent
64cd21b0dd Merge branch 'master' into q-apps 2023-02-28 22:03:19 +00:00
CalDescent
abdc265fc6 Removed legacy bulk archiving/pruning code that is no longer needed. 2023-02-26 16:54:14 +00:00
CalDescent
1153519d78 Various fixes as a result of moving to archive version 2. 2023-02-26 16:53:43 +00:00
CalDescent
0af6fbe1eb Added POST /repository/archive/rebuild endpoint to allow local archive to be rebuilt.
When "archiveVersion" is set to 2 in settings, this should allow the archive size to reduce by over 90%. Some nodes might want to maintain an older/larger version, for the purposes of development/debugging, so this is currently opt-in.
2023-02-26 16:52:48 +00:00
CalDescent
d54006caf7 Added "archiveVersion" setting, which specifies the archive version to be used when building. Defaults to 1 for now, but will bump to version 2 at the time of a wider rollout. 2023-02-26 15:59:18 +00:00
CalDescent
e1771dbaea Merge branch 'master' into rebuild-archive 2023-02-26 14:29:37 +00:00
CalDescent
9566bda279 Merge branch 'master' into block-sequence 2023-02-26 12:55:35 +00:00
CalDescent
cc98abeffb Reduced log spam 2023-02-26 12:51:52 +00:00
CalDescent
a3702ac6b0 Revert "Merge pull request #111 from AlphaX-Projects/master"
This reverts commit 69902f7f5b, reversing
changes made to 466c727dee.
2023-02-26 12:45:38 +00:00
CalDescent
c1ffe557e1 Fixed wording in marshaller exceptions. 2023-02-24 13:42:59 +00:00
CalDescent
c310a7c5e8 Added "X-API-VERSION" header support in POST /transactions/process.
Default is version "1". If version "2" is specified, the API will return the full transaction JSON on success, rather than just "true".

Example usage:

curl -X POST "http://localhost:12391/transactions/process" -H  "X-API-VERSION: 2" -d "signedTransactionBytesHere"
2023-02-24 13:41:52 +00:00
CalDescent
c5a0b00cde Q-Apps documentation updates based on UI development progress. 2023-02-24 12:15:22 +00:00
QuickMythril
69902f7f5b Merge pull request #111 from AlphaX-Projects/master
Update hsqldb and grpc
2023-02-24 05:02:32 -05:00
AlphaX-Projects
999e8b8aca Update pom.xml 2023-02-24 09:12:57 +01:00
CalDescent
466c727dee Bump version to 3.8.9 2023-02-22 19:01:10 +00:00
CalDescent
ba9f3b335c Added unit test to reproduce the UPDATE_NAME issue and prove that the fix is working correctly. 2023-02-22 18:59:43 +00:00
CalDescent
148ca0af05 Fixed long term bug with UPDATE_NAME transactions, causing name data to be incorrectly deleted if newName == name. 2023-02-22 09:16:52 +00:00
CalDescent
c39b9c764b Bump version to 3.8.8 2023-02-20 18:12:40 +00:00
CalDescent
d30eb6141a Default minPeerVersion set to 3.8.7 2023-02-20 18:10:21 +00:00
CalDescent
52c806f9e6 Bump version to 3.8.7 2023-02-19 22:44:59 +00:00
CalDescent
b2d31a7e02 Rebuild the name's history before processing a CancelSellNameTransaction. 2023-02-19 22:26:22 +00:00
CalDescent
cfa0b1d8ea Bump version to 3.8.6 2023-02-19 18:02:22 +00:00
CalDescent
edacce1bac Improved logging when creating bootstraps, and catch/log all exceptions. 2023-02-19 17:43:13 +00:00
CalDescent
074cba2266 Added QCHAT_AUDIO and QCHAT_VOICE services (limited to 10MB each) 2023-02-19 17:33:17 +00:00
CalDescent
7f23ef64a2 Updated /arbitrary/metadata/* response when not found. 2023-02-17 17:37:30 +00:00
CalDescent
5b7e9666dc Send URL updates to the UI when pages are loaded. 2023-02-17 15:40:06 +00:00
CalDescent
20d4e88fab Fixed API endpoints relying on getTransactionsFromSignature(), which therefore won't have worked properly since core V2. 2023-02-12 13:21:54 +00:00
CalDescent
a8c27be18a Modified AT and transaction repository queries to use Transactions.block_sequence instead of BlockTransactions.sequence.
The former is available for all blocks, whereas the latter is only available for unpruned blocks.

Also removed joins with the Blocks table - as the Blocks table is also pruned - and instead retrieved the height from the Transactions table.
2023-02-12 13:21:41 +00:00
CalDescent
af6be759e7 Fixed long term issue where logs would report "Repository in use by another process?" when the database actually failed to start for some other reason. It will now log the correct reason. 2023-02-12 13:20:31 +00:00
CalDescent
896d814385 Add block_sequence to Transactions table, and populate all past transactions.
This data was being lost when pruning the BlockTransactions table.

Note: on first run this will reshape the db, which can take several minutes.
2023-02-12 13:20:23 +00:00
QuickMythril
f4a32d19dd Merge pull request #110 from QuickMythril/update-electrumx
Add new ElectrumX servers
2023-02-12 00:10:56 -05:00
QuickMythril
eb6d84c04d Add new ElectrumX servers 2023-02-12 00:10:13 -05:00
CalDescent
26587067d8 Merge pull request #109 from AlphaX-Projects/master
Add more data to active chats websocket
2023-02-11 13:25:13 +00:00
AlphaX-Projects
227d93a31e Merge branch 'Qortal:master' into master 2023-02-11 13:39:26 +01:00
CalDescent
76f17dda53 Merge branch 'master' into rebuild-archive 2023-02-10 17:48:05 +00:00
CalDescent
830bae3dc1 Merge branch 'at-states-fix'
# Conflicts:
#	src/main/java/org/qortal/controller/repository/AtStatesPruner.java
#	src/main/java/org/qortal/controller/repository/AtStatesTrimmer.java
2023-02-10 17:45:22 +00:00
CalDescent
ec09312cc5 Updated AdvancedInstaller project for 3.8.5 2023-02-10 17:42:12 +00:00
AlphaX-Projects
11654ba9c6 Add Chat Data 2023-02-10 11:05:54 +01:00
AlphaX-Projects
ea356d1026 add signatures to websockt 2023-02-10 10:27:28 +01:00
CalDescent
e7a3e511bd Bump version to 3.8.5 2023-02-08 19:37:01 +00:00
CalDescent
6fca30ce75 Added GET /admin/summary/alltime endpoint, to view a summary of chain activity since genesis. 2023-02-07 19:56:54 +00:00
CalDescent
e903e59f7f Merge pull request #107 from QuickMythril/unused-address
Add unused foreign address to API calls
2023-02-06 18:43:22 +00:00
CalDescent
bef170df7e Updated PirateChain lightwallet servers. 2023-02-06 18:42:37 +00:00
QuickMythril
386bfa4e20 Merge pull request #108 from AlphaX-Projects/master
Add electrum servers and fix java reflect error
2023-02-05 07:38:30 -05:00
AlphaX-Projects
6f867031e2 Add electrum servers and fix java reflect error 2023-02-05 12:53:49 +01:00
QuickMythril
8f589391a6 Updated depreciated actions
Node.js 12 actions are deprecated. Please update the following actions to use Node.js 16: actions/checkout@v2, actions/cache@v2, actions/setup-java@v2. For more information see: https://github.blog/changelog/2022-09-22-github-actions-all-actions-will-begin-running-on-node16-instead-of-node12/
2023-02-04 21:57:31 -05:00
QuickMythril
30c9f63cb1 Add unused foreign address to API calls
POST ​/crosschain​/{COIN}/unusedaddress
2023-02-04 21:03:55 -05:00
QuickMythril
952b21d9bd Merge pull request #105 from QuickMythril/update-electrumx
Updated ElectrumX servers
2023-02-04 18:40:39 -05:00
QuickMythril
1f410a503e Updated ElectrumX servers 2023-02-04 18:30:31 -05:00
CalDescent
ae5b713e58 Rework of AT state trimming and pruning, in order to more reliably track the "latest" AT states.
This should fix an edge case where AT states data was pruned/trimmed but it was then later required in consensus. The older state was deleted because it was replaced by a new "latest" state in a brand new block. But once the new "latest" state was orphaned from the block, the old "latest" state was then required again.

This works around the problem by excluding very recent blocks in the latest AT states data, so that it is unaffected by real-time sync activity.

The trade off is that we could end up retaining more AT states than needed, so a secondary cleanup process may need to run at some time in the future to remove these. But it should only be a minimal amount of data, and can be cleaned up with a single query. This would have been happening to a certain degree already.

# Conflicts:
#	src/main/java/org/qortal/controller/repository/AtStatesPruner.java
#	src/main/java/org/qortal/controller/repository/AtStatesTrimmer.java
2023-02-03 12:39:27 +01:00
CalDescent
257ca2da05 Bumped default block archive serialization version to V2. 2023-02-03 12:36:57 +01:00
CalDescent
d27316eb64 Clear cache after rebuilding. 2023-02-02 18:11:56 +01:00
CalDescent
64d8353629 Added V2 support in the block archive, and added feature to rebuild a V1 block archive using V2 block serialization. Should drastically reduce the archive size once rebuilt. 2023-02-02 15:54:03 +01:00
CalDescent
f5e30eeaf5 Merge pull request #104 from QuickMythril/foreign-height
Add foreign chain height to API calls
2023-02-01 20:28:55 +00:00
QuickMythril
21f5d9a3d0 Add foreign chain height to API calls
GET ​/crosschain​/{COIN}/height
2023-01-31 17:23:25 -05:00
CalDescent
3077810ea8 Fixed bugs causing websites to report as "Not published" when listed in the UI. 2023-01-29 18:05:04 +00:00
CalDescent
4ba2f7ad6a Small documentation updates 2023-01-29 17:20:25 +00:00
CalDescent
8eba0f89fe Added to Q-Apps documentation 2023-01-29 17:09:28 +00:00
CalDescent
600f98ddab Fixed bug in extractComponents() 2023-01-29 13:38:08 +00:00
CalDescent
eb07e6613f Fixed small bug 2023-01-29 13:23:12 +00:00
CalDescent
6c445ff646 GET_ACCOUNT_ADDRESS and GET_ACCOUNT_PUBLIC_KEY replaced with a single action: GET_USER_ACCOUNT, as it doesn't make sense to request address and public key separately (they are essentially the same thing). 2023-01-29 13:23:01 +00:00
CalDescent
4d9cece9fa Timeouts are specified by action, rather than using 10 second for every request. This allows certain requests to wait for longer before timing out, such as ones that create transactions. 2023-01-29 13:07:26 +00:00
CalDescent
8beffd4dae Switched to document.querySelectorAll() as otherwise we were only intercepting the first image on the page. 2023-01-29 12:12:47 +00:00
CalDescent
566c6a3f4b Added support for img src updates from a Q-App.
Example:

document.getElementById("logo").src = "qortal://thumbnail/QortalDemo/qortal_avatar";
2023-01-29 12:04:39 +00:00
CalDescent
1be3ae267e Reduce log spam. 2023-01-29 11:45:09 +00:00
CalDescent
7af551fbc5 Added "GET_QDN_RESOURCE_URL" Q-Apps action, to allow a website/app to programmatically determine the URL to retrieve any QDN resource it needs to access.
Examples:

### Get URL to load a QDN resource
```
let url = await qortalRequest({
    action: "GET_QDN_RESOURCE_URL",
    service: "THUMBNAIL",
    name: "QortalDemo",
    identifier: "qortal_avatar"
    // path: "filename.jpg" // optional - not needed if resource contains only one file
});
```

### Get URL to load a QDN website
```
let url = await qortalRequest({
    action: "GET_QDN_RESOURCE_URL",
    service: "WEBSITE",
    name: "QortalDemo",
});
```

### Get URL to load a specific file from a QDN website
```
let url = await qortalRequest({
    action: "GET_QDN_RESOURCE_URL",
    service: "WEBSITE",
    name: "AlphaX",
    path: "/assets/img/logo.png"
});
```
2023-01-29 11:44:59 +00:00
CalDescent
6ba6c58843 Added support for qortal:// protocol links when loading images from the DOM.
Example: <img src="qortal://THUMBNAIL/QortalDemo/qortal_avatar" />
2023-01-29 11:18:00 +00:00
CalDescent
ab34fae810 Merge pull request #90 from QuickMythril/german
Updated German translations
2023-01-28 20:22:11 +00:00
CalDescent
42f2d015b7 Merge branch 'master' into german 2023-01-28 20:22:02 +00:00
CalDescent
2181ece28d Merge pull request #89 from lexandr0s/patch-2
Update ApiError_ru.properties
2023-01-28 20:21:13 +00:00
CalDescent
03a5d0e5f9 Merge pull request #88 from lexandr0s/patch-1
Update SysTray_ru.properties
2023-01-28 20:21:00 +00:00
CalDescent
352f094272 Merge pull request #99 from Nuc1eoN/polish-translation
Add polish translation
2023-01-28 20:20:35 +00:00
CalDescent
ca09dd264f Merge branch 'master' into q-apps 2023-01-28 20:14:35 +00:00
CalDescent
eea98d0bc7 Fixed bugs. 2023-01-28 18:37:04 +00:00
CalDescent
9c58faa7c2 Added LINK_TO_QDN_RESOURCE support in the gateway. 2023-01-28 18:36:55 +00:00
CalDescent
3cdfa4e276 Increased loading screen refresh interval from 1s to 2s. 2023-01-28 18:03:00 +00:00
CalDescent
380ba5b8c2 Show "File not found" on the loading screen when navigating to a non-existent resource. 2023-01-28 18:01:52 +00:00
CalDescent
04f248bcdd Upgraded gateway to support service and identifier.
The URL used to access the gateway is now interpreted, and the most appropriate resource is served. This means it can be used in different ways to retrieve any type of content from QDN. For example:

/QortalDemo
/QortalDemo/minting-leveling/index.html
/WEBSITE/QortalDemo
/WEBSITE/QortalDemo/minting-leveling/index.html
/APP/QortalDemo
/THUMBNAIL/QortalDemo/qortal_avatar
/QCHAT_IMAGE/birtydasterd/qchat_BfBeCz
/ARBITRARY_DATA/PirateChainWallet/LiteWalletJNI/coinparams.json
2023-01-28 17:56:24 +00:00
CalDescent
37b20aac66 Upgraded rendering to support identifiers, as well as single file resources.
This allows any QDN resource (e.g. an IMAGE) to be linked to from a website/app and then rendered on screen. It isn't yet supported in gateway or domain map mode, as these need some more thought.
2023-01-28 16:55:04 +00:00
CalDescent
e1e52b3165 RenderResource moved to restricted resources, as /render/* endpoints shouldn't ever need to be served over the gateway. 2023-01-28 15:52:46 +00:00
CalDescent
c5c826453b Removed unnecessary join when finding MESSAGE transactions, which caused secret to be unavailable when querying pruned blocks. 2023-01-28 15:41:48 +00:00
CalDescent
e86b9b1caf Added additional Litecoin ElectrumX server. 2023-01-28 15:34:30 +00:00
CalDescent
46e8baac98 Added linking between QDN websites / apps.
The simplest way to link to another QDN website is to include a link with the format:
<a href="qortal://WEBSITE/QortalDemo">link text</a>

This can be expanded to link to a specific path, e.g:
<a href="qortal://WEBSITE/QortalDemo/minting-leveling/index.html">link text</a>

Or it can be initiated programatically, via qortalRequest():

let res = await qortalRequest({
    action: "LINK_TO_QDN_RESOURCE",
    service: "WEBSITE",
    name: "QortalDemo",
    path: "/minting-leveling/index.html" // Optional
});

Note that qortal:// links don't yet support identifiers, so the above format is not confirmed.
2023-01-28 15:22:03 +00:00
CalDescent
3b6e1ea27f Added "qdnContext" variable, with possible values of "render", "gateway", or "domainMap".
This is used internally to allow Q-Apps to determine how to handle certain requests.
2023-01-28 14:42:29 +00:00
CalDescent
5a1cc7a0de Fixed/improved logging when an exception is caught whilst adding statuses to resources. 2023-01-28 14:32:17 +00:00
CalDescent
0ec5e39517 Fixed additional NPE 2023-01-28 14:31:04 +00:00
CalDescent
bede5a71f8 Fixed various NPEs when checking statuses of non-existent resources. 2023-01-28 14:17:23 +00:00
CalDescent
5e750b4283 Added new ArbitraryResourceStatus "NOT_PUBLISHED" - for when a non-existent resource is attempted to be loaded. 2023-01-28 14:15:54 +00:00
CalDescent
4a42dc2d00 Don't require prior authorization of QDN resources if qdnAuthBypassEnabled is true. Necessary for resource linking. 2023-01-28 14:14:44 +00:00
CalDescent
7fc170575c Merge branch 'cancel-sell-name-fixes' 2023-01-28 12:11:42 +00:00
CalDescent
876658256f Prevent a P2SH address being funded for a trade if there is an unconfirmed buy or cancel request in progress for it already.
This prevents foreign coins from leaving the local wallet when there is a high probability that the trade will fail, and therefore should reduce the chances of losing transaction fees due to refunds.

Whenever this occurs, the UI will show "Trade has an existing buy request or is pending cancellation." after clicking Buy.
2023-01-28 11:57:15 +00:00
CalDescent
a24ba40d5c Added additional Dogecoin ElectrumX server. 2023-01-28 09:54:15 +00:00
CalDescent
06d8a21714 Added CANCEL_SELL_NAME equivalents to NamesDatabaseIntegrityCheck.java 2023-01-27 19:38:26 +00:00
CalDescent
ae44065d7e Fixed issue with CancelSellName transactions. 2023-01-27 19:34:23 +00:00
CalDescent
6ad0989ea2 Reduce log spam 2023-01-27 18:35:44 +00:00
CalDescent
5962ebd08a More logging improvements in ArbitraryDataReader.decrypt() 2023-01-27 16:56:53 +00:00
CalDescent
bf06d47842 Create an ArbitraryDataResource object when building. Eventually this could be passed in to the reader instead of the individual components (service, name, identifier, etc)
This is now used to improve logging when extracting.
2023-01-27 16:55:43 +00:00
CalDescent
d7b1615d4f qdnAuthBypassEnabled defaulted to true, as it is needed for Q-Apps. 2023-01-27 16:26:36 +00:00
CalDescent
8c708558cb Implemented ElectrumX version negotiation. Fixes issues with DOGE wallet. 2023-01-27 14:33:34 +00:00
CalDescent
6b36d94c6f Removed searchResultsTransactions cache, to simplify code. The hostedTransactions cache is still in place, which limits disk reads when searching, so this additional cache isn't really needed. 2023-01-27 12:48:42 +00:00
CalDescent
8c41a4a6b3 Moved BootstrapResource to restricted resources 2023-01-22 21:08:42 +00:00
CalDescent
8dffe1e3ac Another rewrite of Q-App APIs, which removes the /apps/* redirects and instead calls the main APIs directly.
- All APIs are now served over the gateway and domain map, with the exception of /admin/*
- AdminResource moved to a "restricted" folder, so that it isn't served over the gateway/domainMap ports.
- This opens the door to websites/apps calling core APIs directly for certain read-only functions, as an alternative to using qortalRequest().
2023-01-22 18:59:46 +00:00
CalDescent
932a553b91 Merge branch 'master' into q-apps
# Conflicts:
#	src/main/java/org/qortal/api/resource/ArbitraryResource.java
2023-01-22 16:37:02 +00:00
CalDescent
1d568fa462 Return file lists via /arbitrary/metadata/* endpoints, but exclude it from /arbitrary/resources/* endpoints. 2023-01-22 16:29:23 +00:00
CalDescent
328ba48224 Merge branch 'master' into qdn-file-list 2023-01-22 16:12:54 +00:00
CalDescent
6196841609 Allow files without extensions in QCHAT_ATTACHMENT validation. 2023-01-22 15:59:16 +00:00
CalDescent
9f30571b12 Use a filename without an extension when publishing data from a string (instead of .tmp) 2023-01-22 15:58:53 +00:00
CalDescent
1f7fec6251 Exclude .qortal directory in validation functions, as it was incorrectly failing with "DIRECTORIES_NOT_ALLOWED". 2023-01-20 10:40:20 +00:00
CalDescent
c3f19ea0c1 Don't allow the custom validation methods to evade superclass validation. 2023-01-20 10:21:05 +00:00
CalDescent
e31515b4a2 Fixed bugs preventing single file GIF repositories and QCHAT attachments from passing validation. 2023-01-20 10:14:42 +00:00
CalDescent
8ad46b6344 Fixed/removed incorrect comments 2023-01-20 09:58:28 +00:00
CalDescent
57eacbdd59 Added "GET_PRICE" action. 2023-01-19 20:47:06 +00:00
CalDescent
86d6037af3 Added "SEARCH_TRANSACTIONS" action. 2023-01-19 20:22:29 +00:00
CalDescent
ca80fd5f9c Added "FETCH_BLOCK" and "FETCH_BLOCK_RANGE" Q-Apps actions. 2023-01-19 20:05:46 +00:00
CalDescent
03a54691a1 Merge branch 'master' into q-apps 2023-01-19 19:57:01 +00:00
CalDescent
3c8088e463 Removed all code duplication for Q-Apps API endpoints.
Requests are now internally routed to the existing API handlers. This should allow new Q-Apps API endpoints to be added much more quickly, as well as removing the need to maintain their code separately from the regular API endpoints.
2023-01-19 19:56:50 +00:00
CalDescent
2f7912abce Compute balances for Bitcoin-like coins using unspent outputs. Should fix occasional incorrect balance issue, and speed up loading time. 2023-01-18 19:30:43 +00:00
CalDescent
64529e8abf Added "reverse" and "includeOnlineSignatures" params to GET /blocks/range/{height} endpoint. 2023-01-18 19:04:54 +00:00
CalDescent
9d81ea7744 Bump version to 3.8.4 2023-01-16 20:26:00 +00:00
CalDescent
688acd466c Set checkpoint to block 1136300 2023-01-16 20:23:43 +00:00
CalDescent
81cf46f5dd Disable block signing on topOnly nodes. Minting rewards are still earned on topOnly for now. 2023-01-16 20:18:23 +00:00
CalDescent
4c52d6f0fc Fixed bug causing initial latestATStates data to be discarded. 2023-01-15 21:58:17 +00:00
CalDescent
de47a94677 Fixed bug causing initial latestATStates data to be discarded. 2023-01-15 15:51:10 +00:00
CalDescent
bd4c47dba6 Rework of AT state trimming and pruning, in order to more reliably track the "latest" AT states.
This should fix an edge case where AT states data was pruned/trimmed but it was then later required in consensus. The older state was deleted because it was replaced by a new "latest" state in a brand new block. But once the new "latest" state was orphaned from the block, the old "latest" state was then required again.

This works around the problem by excluding very recent blocks in the latest AT states data, so that it is unaffected by real-time sync activity.

The trade off is that we could end up retaining more AT states than needed, so a secondary cleanup process may need to run at some time in the future to remove these. But it should only be a minimal amount of data, and can be cleaned up with a single query. This would have been happening to a certain degree already.
2023-01-15 14:32:33 +00:00
CalDescent
c03f271825 Keep track of peers which are too divergent, and return an isTooDivergent boolean in /peers APIs.
isTooDivergent will be true or false if a definitive decision has been made, or missing from the response if not yet known. Therefore it should be safe to treat `"isTooDivergent": false` as a peer that is on the same chain.
2023-01-15 12:44:19 +00:00
CalDescent
dfe3754afc Block connections with peers older than 3.8.2, as those versions are nonfunctional due to recent feature triggers. 2023-01-15 12:07:27 +00:00
CalDescent
30105199a2 Default pruneBlockLimit increased from 1450 to 6000 (approx 5 days), to be more similar to the AT states retention time of full nodes. 2023-01-15 12:00:32 +00:00
CalDescent
e91e612b55 Added checkpoint lookup on startup.
Currently enabled for topOnly nodes only. This will detect if the node is on a divergent chain, and will force a bootstrap or resync (depending on settings) in order to rejoin the main chain.
2023-01-15 11:33:16 +00:00
CalDescent
2a55eba1f7 Updated AdvancedInstaller project for v3.8.3 2023-01-15 11:28:37 +00:00
CalDescent
39e59cbcf8 Bump version to 3.8.3 2023-01-14 18:47:46 +00:00
CalDescent
016191bdb0 Reduce log spam when a QDN resource can't be found due to it not being published. 2023-01-14 15:15:48 +00:00
CalDescent
0596a07c7d Reduced ArbitraryDataFileRequestThread count from 10 to 5, to reduce network flooding. 2023-01-14 12:58:35 +00:00
CalDescent
c62c59b445 Use correct timeout (12s) when sending arbitrary data to a peer, and improved logging. 2023-01-14 12:57:44 +00:00
CalDescent
f78101e9cc Updated a default bootstrap host to use a domain instead of its IP. 2023-01-14 11:07:54 +00:00
CalDescent
476fdcb31d Added serialization tests for chatReference, and grouped with other serialization tests into a single package. 2023-01-14 10:38:50 +00:00
CalDescent
02d5043ef7 Added missing calls to electrumX.setBlockchain(instance); for DGB and RVN. Thanks to @QuickMythril for noticing this. 2023-01-13 20:17:27 +00:00
CalDescent
0ad9e2f65b Added QCHAT_ATTACHMENT service, with custom validation function. 2023-01-13 20:08:47 +00:00
CalDescent
4dc0033a5a Added missing chatReferenceTimestamp in unit tests. 2023-01-13 19:45:52 +00:00
CalDescent
745cfe8ea1 chatReferenceTimestamp set to 1674316800000 (Sat, 21 Jan 2023 16:00:00 GMT) 2023-01-13 19:45:38 +00:00
CalDescent
6284a4691c Import test transactions as part of the serialization tests, to catch any issues with db schema data lengths. 2023-01-13 19:28:44 +00:00
CalDescent
41f88be55e Test serialization of CHAT transactions 2023-01-13 19:27:38 +00:00
CalDescent
ba95f8376f Increase CHAT transaction data limits to the maximum (4000 bytes) to allow for upcoming UI features. 2023-01-13 19:27:02 +00:00
CalDescent
8e97c05b56 Added missing feature trigger from unit tests. 2023-01-13 19:25:06 +00:00
CalDescent
2c78f4b45b Fixed typo and reworded "methods" to "actions", for consistency with the code. 2023-01-13 18:25:30 +00:00
CalDescent
613ce84df8 More documentation updates 2023-01-13 18:11:44 +00:00
CalDescent
2822d860d8 Fixed sample app 2023-01-13 18:01:38 +00:00
CalDescent
5a052a4f67 Documentation updates 2023-01-13 17:57:01 +00:00
CalDescent
32c2f68cb1 Initial APIs and core support for Q-Apps 2023-01-13 17:36:27 +00:00
CalDescent
4232616a5f Fixed QDN website preview functionality. 2023-01-13 12:07:24 +00:00
CalDescent
8ddcae249c Added gatewayLoopbackEnabled setting (default false) to allow serving gateway requests via localhost.
Useful for testing, but not recommended for production environments.
2023-01-13 12:05:57 +00:00
CalDescent
eb569304ba Improved refund/refundAll HTLC code, to handle cases where there have been multiple purchase attempts for the same AT. 2023-01-06 10:38:25 +00:00
CalDescent
b0486f44bb Added chat_reference index to speed up searches. 2023-01-02 17:47:36 +00:00
CalDescent
cecf28ab7b Merge branch 'chat-reference'
# Conflicts:
#	src/main/java/org/qortal/block/BlockChain.java
#	src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java
#	src/main/resources/blockchain.json
#	src/test/resources/test-chain-v2-block-timestamps.json
#	src/test/resources/test-chain-v2-disable-reference.json
#	src/test/resources/test-chain-v2-founder-rewards.json
#	src/test/resources/test-chain-v2-leftover-reward.json
#	src/test/resources/test-chain-v2-minting.json
#	src/test/resources/test-chain-v2-qora-holder-extremes.json
#	src/test/resources/test-chain-v2-qora-holder-reduction.json
#	src/test/resources/test-chain-v2-qora-holder.json
#	src/test/resources/test-chain-v2-reward-levels.json
#	src/test/resources/test-chain-v2-reward-scaling.json
#	src/test/resources/test-chain-v2-reward-shares.json
#	src/test/resources/test-chain-v2.json
2023-01-02 17:32:38 +00:00
CalDescent
98b92a5bf1 Introduced "historic threshold" to ARBITRARY transactions in order to save on verification times of older transactions.
This is based on the approach used for PUBLICIZE transactions.
2023-01-02 16:58:50 +00:00
CalDescent
6b45901c47 Fixed validation of existing reward share transactions. 2022-12-31 14:43:37 +00:00
CalDescent
166f9bd079 Bump version to 3.8.2 2022-12-24 21:28:02 +00:00
CalDescent
2f8f896077 Merge remote-tracking branch 'catbref/bugfix-deploy-at' 2022-12-24 16:01:23 +00:00
CalDescent
9a77aff0a6 Reduced difficulty of PUBLICIZE transactions from 15 to 14 (it is now the same as ARBITRARY transactions) 2022-12-24 14:10:49 +00:00
CalDescent
c6d65a88dc Increase mempow difficulty and threshold in ChatTransaction, to match the values in the UI. 2022-12-22 18:19:27 +00:00
CalDescent
4aea29a91b Improved PublicizeTransaction validation. 2022-12-22 18:03:29 +00:00
CalDescent
0e81665a36 Revert "Filter out peers of divergent or significantly inferior chains when syncing."
This reverts commit 1dc7f056f9. To be un-reverted in future when there is more time available for testing.
2022-12-22 15:10:19 +00:00
CalDescent
2a4ac1ed24 Limit to 250 CHAT messages per hour per account. 2022-12-22 15:09:04 +00:00
CalDescent
bb74b2d4f6 MAX_AVG_RESPONSE_TIME for ElectrumX servers increased from 0.5s to 1s. 2022-12-22 14:25:10 +00:00
CalDescent
758a02d71a Log Pirate light client server address if the wallet unable to be initialized. 2022-12-22 14:23:30 +00:00
CalDescent
7ae142fa64 Improved transaction validation. 2022-12-22 14:20:42 +00:00
CalDescent
a75ed0e634 Bump additional expandedAccount level references held in memory. 2022-12-22 14:18:39 +00:00
CalDescent
e40dc4af59 Fixed group ban expiry. 2022-12-22 14:16:57 +00:00
CalDescent
e678ea22e0 Fixed NPE in unit tests. Still need to work out how/when this was introduced. 2022-12-18 18:33:51 +00:00
CalDescent
cf3195cb83 Set "minAccountsToActivateShareBin" to 0 for certain tests. 2022-12-18 18:32:06 +00:00
CalDescent
80048208d1 Moved some test sponsorship utility methods to AccountUtils, so they can be used in other test classes too. 2022-12-15 12:14:42 +00:00
CalDescent
08de1fb4ec Disallow CHAT transactions with timestamps more than 5 minutes in the future. 2022-12-14 16:40:57 +00:00
CalDescent
99d5bf9103 Disallow transactions with timestamps more than 30 mins in the future (reduced from 24 hours) 2022-12-14 16:40:11 +00:00
CalDescent
1dc7f056f9 Filter out peers of divergent or significantly inferior chains when syncing. 2022-12-14 16:39:43 +00:00
CalDescent
cdeb2052b0 Bump version to 3.8.1 2022-12-08 18:26:34 +00:00
CalDescent
5c9109aca9 minPeerVersion set to 3.8.0 2022-12-08 18:25:19 +00:00
CalDescent
ccc1976d00 Added defensiveness 2022-12-08 18:25:03 +00:00
CalDescent
12fb6cd0ad onlineAccountMinterLevelValidationHeight moved forward to block 1092000 2022-12-08 18:24:34 +00:00
CalDescent
6f95e7c1c8 Bump version to 3.8.0 2022-12-05 21:57:32 +00:00
CalDescent
a69618133e Level 0 online account removals moved inside feature trigger, so it is coordinated with the new validation. 2022-12-05 21:34:26 +00:00
CalDescent
51ad0a5b48 onlineAccountMinterLevelValidationHeight set to 1093400 (approx 20 hours later) 2022-12-05 19:38:44 +00:00
CalDescent
45a6f495d2 selfSponsorshipAlgoV1Height set to 1092400 (approx 4pm UTC on Sat 10th December) 2022-12-05 19:38:26 +00:00
CalDescent
4d9964c080 Block connections with peers older than 3.7.0, as this has been released for long enough now. 2022-12-05 18:52:33 +00:00
CalDescent
9afc31a20d selfSponsorshipAlgoV1SnapshotTimestamp set to 1670230000000 2022-12-05 08:52:09 +00:00
CalDescent
d435e4047b SelfSponsorshipAlgoV1 2022-12-05 08:21:45 +00:00
CalDescent
c108afa27c Self sponsorship algo tests 2022-12-04 20:57:36 +00:00
CalDescent
eea42b56ee Added SelfSponsorshipAlgoV1Block, and call it when processing/orphaning a block at an undecided future height. 2022-12-04 18:21:01 +00:00
CalDescent
f4d20e42f3 Disallow TRANSFER_PRIVS transactions if the sending account has a penalty. Again, there will be no penalties until the algo runs, so it's safe without a feature trigger. 2022-12-04 11:54:05 +00:00
CalDescent
f14cc374c6 Include blocksMintedPenalty in effectiveBlocksMinted. This will be zero until the algo runs, so doesn't need a feature trigger. 2022-12-04 11:52:51 +00:00
CalDescent
99ba4caf75 We definitely can't retroactively validate minter levels, because there are confirmed cases in the chains history where this fails. Set to 999999999 until we have decided on a future block height. 2022-12-04 11:50:58 +00:00
catbref
ae991dda4d Fix creatorPublicKey not being unmarshaled when calling POST /at to deploy an AT 2022-11-28 21:52:37 +00:00
CalDescent
2b6ae57a27 Merge branch 'master' into chat-reference
# Conflicts:
#	src/main/java/org/qortal/block/BlockChain.java
#	src/main/resources/blockchain.json
#	src/test/resources/test-chain-v2-block-timestamps.json
#	src/test/resources/test-chain-v2-disable-reference.json
#	src/test/resources/test-chain-v2-founder-rewards.json
#	src/test/resources/test-chain-v2-leftover-reward.json
#	src/test/resources/test-chain-v2-minting.json
#	src/test/resources/test-chain-v2-qora-holder-extremes.json
#	src/test/resources/test-chain-v2-qora-holder-reduction.json
#	src/test/resources/test-chain-v2-qora-holder.json
#	src/test/resources/test-chain-v2-reward-levels.json
#	src/test/resources/test-chain-v2-reward-scaling.json
#	src/test/resources/test-chain-v2-reward-shares.json
#	src/test/resources/test-chain-v2.json
2022-11-27 20:06:55 +00:00
CalDescent
5ff7b3df6d hasInvalidSigner() now only checks the chain tip block, to reduce the amount of unintended side effects that can occur. 2022-11-27 19:59:46 +00:00
CalDescent
76686eca21 Modified reward share creation in test/common/AccountUtils to allow a fee to be specified. 2022-11-27 12:23:17 +00:00
CalDescent
3965f24ab5 Fixed bug 2022-11-26 19:06:02 +00:00
CalDescent
a75fd14e45 Added Account method needed for unit tests. 2022-11-26 17:23:43 +00:00
CalDescent
41cdf665ed Code tidy 2022-11-26 16:53:38 +00:00
CalDescent
6ea3c0e6f7 Give founder accounts as an effective minting level of 0 if they have a penalty. 2022-11-26 16:19:43 +00:00
CalDescent
5f0263c078 Modifications to block minting for unit tests, in order to solve an NPE and give more options to callers.
This shouldn't affect the behaviour of existing tests, other than an NPE being replaced with an assertNotNull().
2022-11-26 16:14:45 +00:00
CalDescent
58e5d325ff Added algo feature triggers to BlockChain.java (at future undecided block height & timestamp) 2022-11-26 16:10:18 +00:00
CalDescent
7003a8274b Added some API endpoints relating to penalties. Relies on some code not yet committed. 2022-11-26 15:51:06 +00:00
CalDescent
ab687af4bb Added new db query to fetch a list of all accounts that have created a non-self-share, based on confirmed transactions.
This will be used as the input dataset for the self sponsorship algo.
2022-11-26 12:21:43 +00:00
CalDescent
f50c0c87dd Account repository modifications for blocksMintedPenalty. 2022-11-26 12:03:13 +00:00
CalDescent
9c3a4d6e37 BlockChain.java additions for onlineAccountMinterLevelValidationHeight, which were missing from commit 68a0923 2022-11-26 11:55:27 +00:00
CalDescent
1c8a6ce204 When synchronizing, filter out peers that have a recent block with an invalid signer.
This avoids the wasted time and consensus confusion causes by syncing and then validation failing. This is significant after the algo has run, as many signers will become invalid.
2022-11-26 11:52:27 +00:00
CalDescent
68a0923582 Disallow level 0 minters in blocks, and exclude them when minting a new block.
The validation is currently set to a feature trigger of height 0, although this will likely be set to a future block, in case there are any cases in the chain's history where this validation may fail (e.g. transfer privs?)
2022-11-26 11:37:02 +00:00
CalDescent
617c801cbd Made Block.ExpandedAccount public, and added some more getters. This is needed for upcoming additional validation and unit tests. 2022-11-26 11:25:44 +00:00
CalDescent
b0c9ce7482 Add blocks minted penalty to Accounts table 2022-11-26 11:19:23 +00:00
CalDescent
4e829a2d05 Bump version to 3.7.0 2022-11-07 21:12:34 +00:00
CalDescent
a7402adfa5 Merge branch 'null-owned-groups' 2022-11-06 21:20:19 +00:00
CalDescent
9255df46cf Script updates to support add/remove dev group admins 2022-11-06 19:46:12 +00:00
CalDescent
db22445948 Include API key automatically in publish-auto-update-v5.pl 2022-11-06 14:52:14 +00:00
CalDescent
818e037e75 Merge branch 'master' into null-owned-groups 2022-11-06 13:08:54 +00:00
CalDescent
9c68f1038a Bump AT version to 1.4.0 2022-11-05 14:02:04 +00:00
CalDescent
10ae383bb6 Merge pull request #102 from catbref/faster-qort-buy-PoC
Proof of concept: speed up QORT buying
2022-11-01 18:55:21 +00:00
catbref
aead9cfcbf Proof of concept: speed up QORT buying
When users buy QORT ("Alice"-side), most of the API time is spent computing mempow for the MESSAGE sent to Bob's AT.
This is the final stage startResponse() and after Alice's P2SH is already broadcast.

To speed this up, the MESSAGE part is moved into its own thread allowing startResponse() to return sooner, improving the user experience.

Caveats:
If MESSAGE importAsUnconfirmed() somehow fails the the buy won't complete and Alice will have to wait for P2SH refund.
If Alice shuts down her node while MESSAGE mempow is being computed then it's possible the shutdown will be blocked until mempow is complete.

Currently only implemented in LitecoinACCTv3TradeBot as this is only proof-of-concept.
Tested with multiple buys in the same block.
2022-11-01 08:55:57 +00:00
CalDescent
055775b13d Include a list of files in the QDN metadata. 2022-10-30 18:54:38 +00:00
CalDescent
985c195e9e Added GIF_REPOSITORY, with custom validation function and unit tests. 2022-10-30 17:33:21 +00:00
CalDescent
0628847d14 Removed QORTAL_METADATA service tests. 2022-10-30 17:25:11 +00:00
CalDescent
4043ae1928 Added QCHAT_IMAGE service (with 500KB file size limit). 2022-10-30 17:23:46 +00:00
CalDescent
fa80c83864 Remove QORTAL_METADATA service as this uses its own protocol instead. 2022-10-30 17:07:56 +00:00
CalDescent
f739d8f5c6 Added increaseOnlineAccountsDifficultyTimestamp feature trigger to unit tests. 2022-10-28 18:06:34 +01:00
CalDescent
166425bee9 Added feature trigger timestamp (TBC) to increase online accounts mempow difficulty (also TBC). 2022-10-28 17:20:39 +01:00
CalDescent
59a804c560 Include "blocks remaining" in systray when syncing from more than 60 minutes away from a peer's chain tip. 2022-10-28 16:57:52 +01:00
CalDescent
b64c053531 Reuse the work buffer when verifying online accounts from the OnlineAccountsManager import queue.
This is a hopeful fix for extra memory usage since mempow activated, due to adding a lot of load to the garbage collector. It only applies to accounts verified from the import queue; the optimization hasn't been applied to block processing. But verifying online accounts when processing blocks is rare and generally would only last a short amount of time.
2022-10-28 16:54:53 +01:00
CalDescent
30cd56165a Speed up syncing blocks in the range of 1-12 hours ago by caching the valid online accounts. 2022-10-28 16:02:25 +01:00
CalDescent
510328db47 Removed unused timestamp value. 2022-10-28 15:50:43 +01:00
CalDescent
9d74f0eec0 Added haschatreference, with possible values of true, false, or null, to allow optional filtering by the presence or absense of a chat reference. 2022-10-24 19:21:29 +01:00
CalDescent
09014d07e0 Fixed issues retrieving chatReference from the db. 2022-10-23 19:29:31 +01:00
CalDescent
f83d4bac7b Reduced online accounts mempow difficulty to 5 on testnets.
This allows testnets to more easily coexist on the same machines that are running a mainnet instance, and still tests the mempow computation and verification in a non-resource-intensive way.
2022-10-23 17:01:58 +01:00
CalDescent
b3273ff01a Removed all mempow feature trigger conditionals.
We no longer need all the code complexity, now that 24 hours have passed since activation. We don't validate online accounts beyond 12 hours, and the data is trimmed after 24 hours.
2022-10-23 16:47:42 +01:00
CalDescent
1dd039fb2d Merge branch 'master' into chat-reference 2022-10-23 14:14:23 +01:00
CalDescent
1d5497e484 Modifications to support a single node testnet:
- Added "singleNodeTestnet" setting, allowing for fast and consecutive block minting, and no requirement for a minimum number of peers.
- Added "recoveryModeTimeout" setting (previously hardcoded in Synchronizer).
- Updated testnets documentation to include new settings and a quick start guide.
- Added "generic" minting account that can be used in testnets (not functional on mainnet), to simplify the process for new devs.
2022-10-23 14:13:38 +01:00
CalDescent
b37aa749c6 Removed onlineAccountsMemPoWEnabled setting as it's no longer needed. 2022-10-22 19:34:24 +01:00
CalDescent
e45ad37eb5 Fixed bug which could prevent invalid accounts being removed from the queue until the next valid one is added. 2022-10-22 19:30:08 +01:00
CalDescent
72985b1fc6 Reduce log spam, especially around the time of node startup before online accounts have been retrieved.
We expect a "Couldn't build a to-be-minted block" log on every startup due to trying to mint before having any accounts. This one has moved from error to info level because error logs can be quite intrusive when using an IDE.
2022-10-22 19:24:54 +01:00
CalDescent
6f27d3798c Improved online accounts processing, to avoid creating keys in the map before validation. 2022-10-22 19:18:41 +01:00
CalDescent
23a5c5f9b4 Fixed bug in original commit - we need to save the chat reference to the db. 2022-10-22 12:50:28 +01:00
CalDescent
a4759a0ef4 Re-ordered chat transaction transformation, to simplify UI code. New additions are now at the end of the data bytes. 2022-10-22 12:43:40 +01:00
CalDescent
910191b074 Added optional chatReference field to CHAT transactions.
This allows one message to reference another, e.g. for replies, edits, and reactions. We can't use the existing reference field as this is used for encryption and generally points to the user's lastReference at the time of signing.

"chatReference" is based on the "nameReference" field used in various name transactions, for similar purposes.

This needs a feature trigger timestamp to activate, and that same timestamp will need to be used in the UI since that is responsible for building the chat transactions.
2022-10-21 15:58:23 +01:00
CalDescent
57125a91cf Bump version to 3.6.4 2022-10-15 18:59:42 +01:00
CalDescent
3c565638c1 onlineAccountsMemoryPoWTimestamp set to Sat Oct 22 2022 16:00:00 UTC 2022-10-15 18:58:13 +01:00
CalDescent
c2d02aead9 Default minPeerVersion set to 3.6.3 2022-10-14 18:44:25 +01:00
CalDescent
0d9aafaf4e Reduced log spam 2022-10-14 17:03:10 +01:00
CalDescent
3844358380 Mark a peer as misbehaved if it fails to respond with a usable block 3 times in a row.
This should help to workaround deserialization and missing response issues.
2022-10-14 16:38:05 +01:00
CalDescent
b4125d2bf1 Fix for NPE in verifyMemoryPoW() 2022-10-14 11:34:46 +01:00
CalDescent
5c223179ed Updated AdvancedInstaller project for v3.6.3 2022-10-13 23:37:21 +01:00
CalDescent
f3cb57417a Merge branch 'master' of github.com:Qortal/qortal 2022-10-13 23:36:27 +01:00
CalDescent
7c7f071eba Bump version to 3.6.3 2022-10-12 08:54:27 +01:00
CalDescent
7c15d88cbc Fix for issue in BLOCK_SUMMARIES_V2 when sending an empty array of summaries.
The BLOCK_SUMMARIES message type would differentiate between an empty response and a missing/invalid response. However, in V2, a response with empty summaries would throw a BufferUnderflowException and be treated by the caller as a null message.

This caused problems when trying to find a common block with peers that have diverged by more than 8 blocks. With V1 the caller would know to search back further (e.g. 16 blocks) but in V2 it was treated as "no response" and so the caller would give up instead of increasing the look-back threshold.

This fix will identify BLOCK_SUMMARIES_V2 messages with no content, and return an empty array of block summaries instead of a null message.

Should be enough to recover any stuck nodes, as long as they haven't diverged more than 240 blocks from the main chain.
2022-10-12 08:52:58 +01:00
CalDescent
d4aaba2293 Bump version to 3.6.2 2022-10-10 19:06:08 +01:00
CalDescent
10d3176e70 Revert "Always use BlockSummariesMessage V1 (instead of V2) when responding to GetBlockSummaries requests."
This reverts commit 2d58118d7c.
2022-10-10 10:28:44 +01:00
CalDescent
36fcd6792a Discard BLOCK_SUMMARIES_V2 messages with an ID (thanks to @catbref for the code)
This is a better fix for the "contaminated chain tip summaries" issue. Need to reduce the logging level to debug before release.
2022-10-10 10:28:36 +01:00
CalDescent
cb1eee8ff5 GenericUnknownMessage.MINIMUM_PEER_VERSION set to 3.6.1.
This should ideally have been set in the 3.6.1 release, but not setting it is unlikely to have caused any problems.
2022-10-09 20:37:39 +01:00
CalDescent
2d58118d7c Always use BlockSummariesMessage V1 (instead of V2) when responding to GetBlockSummaries requests.
This should hopefully fix a potential issue where peer's chain tip data becomes contaminated with other summary data, causing incorrect sync decisions.
2022-10-09 20:11:01 +01:00
CalDescent
e6bb0b81cf Revert "Reduce INITIAL_BLOCK_STEP from 8 to 7."
This reverts commit 0088ba8485.
2022-10-09 19:11:20 +01:00
Nuc1eoN
8ddf4c9f9f Add polish translation 2022-10-09 15:35:19 +02:00
Nuc1eoN
0693e26cda Update/add German translation 2022-10-09 15:34:20 +02:00
CalDescent
77d60fc33f Revert "Skip GET_BLOCK_SUMMARIES requests if it can already be fulfilled entirely from the peer's chain tip block summaries cache."
This reverts commit 8cedf618f4.
2022-10-09 14:11:28 +01:00
CalDescent
504f38b42a Merge pull request #97 from Nuc1eoN/patch-1
Mark start/stop scripts as executables
2022-10-08 19:49:10 +01:00
Nuc1eoN
3a18599d85 Mark start/stop scripts as executables
The `start.sh` & `stop.sh` scripts have already been marked as executables in the source folder... But since we have only piped their contents, we need to set correct file permissions again.
2022-10-07 23:35:35 +02:00
CalDescent
0088ba8485 Reduce INITIAL_BLOCK_STEP from 8 to 7.
This allows the first pass to always be served from the peer's cache of 8 summaries. This allows a maximum of 7 to be returned, because the 8th spot is needed for the parent block's signature.
2022-10-07 14:47:46 +01:00
CalDescent
8cedf618f4 Skip GET_BLOCK_SUMMARIES requests if it can already be fulfilled entirely from the peer's chain tip block summaries cache.
Loading from the cache should speed up sync decisions, particularly when choose which peer to sync from. The greater the number of connected peers, the more significant this optimization will be. It should also reduce wasted network requests and data usage.

Adding this check prior to making a network request is a simple way to introduce the new cached summaries from BLOCK_SUMMARIES_V2 without having to rewrite a lot of the complex sync / peer comparison logic. Longer term we may want to rewrite that logic to read from the cache directly, but it doesn't make sense to introduce that level of risk at this point time, especially as the Synchronizer may be rewritten soon to prefer longer chains.

Even so, this is still quite a high risk commit so lots of testing will be needed.
2022-10-07 14:46:09 +01:00
CalDescent
fdd95eac56 Limit to 240 blocks in syncToPeerChain().
Should fix OutOfMemoryException often seen when syncing from 1000+ blocks behind the chain tip.
2022-10-07 11:05:24 +01:00
CalDescent
10b0f0a054 Catch JSON exceptions in PirateChainWalletController.
This could prevent additional wallets from being initialized if connection was lost while syncing an existing one.
2022-10-05 15:29:29 +01:00
CalDescent
1233ba6703 Bump version to 3.6.1 2022-10-04 20:08:30 +01:00
CalDescent
c35c7180d4 Return empty levels in GET /addresses/online/levels 2022-10-03 10:58:47 +01:00
CalDescent
7080b55aac Reintroduced initial sleep period in block archiver. 2022-09-25 19:43:56 +01:00
CalDescent
3890fa8490 Renamed constant for consistency 2022-09-25 18:46:33 +01:00
CalDescent
a9721bab3d Fixed issue causing startup of various components to be delayed by 30 seconds. 2022-09-25 18:39:56 +01:00
CalDescent
1bb8f1b6d2 Fixed bug in last commit.
We need to track items to remove separately from items to add, otherwise invalid accounts remain in the queue.
2022-09-25 12:36:00 +01:00
CalDescent
765416db71 Yet another attempt to optimize the online accounts import queue processing.
The main difference here is that we now remove items from the onlineAccountsImportQueue in a batch, _after_ they have been imported. This prevents duplicates from being added to the queue in the previous time gap between them being removed and imported.
2022-09-25 12:26:00 +01:00
CalDescent
5989473c8a Revert "Allow duplicate variations of each OnlineAccountData in the import queue, but don't allow two entries that match exactly."
This reverts commit 6d9e6e8d4c.
2022-09-25 12:06:14 +01:00
CalDescent
aa9da45c01 Added optional filtering by reference in GET /chat/messages 2022-09-25 11:38:17 +01:00
CalDescent
4681218416 Include total count in debug trade presence logging 2022-09-24 15:49:29 +01:00
CalDescent
5c746f0bd9 Fixed bug which required a node to hold local trade presences before it would request any.
This caused large gaps with no presence data. They are removed when they expire, causing the local count to drop to zero, and the node would only start requesting them again once a peer had pushed one or more entries proactively.
2022-09-24 15:48:45 +01:00
CalDescent
309f27a6b8 Moved error to debug, as we now get a burst of these soon after startup, due to commit 99858f3.
This also shows that commit 99858f3 now prevents a block candidate with a very small number of online accounts being built immediately after startup.
2022-09-24 15:21:01 +01:00
CalDescent
d2ebb215e6 Fixed Synchronizer.getBlockSummaries() which was expecting BLOCK_SUMMARIES, but updated peers send BLOCK_SUMMARIES_V2 2022-09-24 14:36:49 +01:00
CalDescent
7a60f713ea Fixed error in rebase. 2022-09-24 14:35:02 +01:00
CalDescent
e80dd31fb4 BlockSummariesV2Message.MINIMUM_PEER_VERSION set to 3.6.1 2022-09-24 13:53:27 +01:00
catbref
94cdc10151 Initial work on BLOCK_SUMMARIES_V2, part of a bigger arc to improve synchronization.
Touches quite a few files because:

* Deprecate HEIGHT_V2 because it doesn't contain enough info to be fully useful during sync.
Newer peers will re-use BLOCK_SUMMARIES_V2.

* For newer peers, instead of sending / broadcasting HEIGHT_V2,
send top N block summaries instead, to avoid requests for minor reorgs.

* When responding to GET_BLOCK, and we don't actually have the requested block,
we currently send an empty BLOCK_SUMMARIES message instead of not responding,
which would cause a slow timeout in Synchronizer.

This pattern has spread to other network message response code,
so now we introduce a generic 'unknown' message type for all these cases.

* Remove PeerChainTipData class entirely and re-use BlockSummaryData instead.

* Each Peer instance used to hold PeerChainTipData - essentially single latest block summary - but now holds a List of latest block summaries.

* PeerChainTipData getter/setter methods modified for compatibility at this point in time.

* Repository methods that return BlockSummaryData (or lists of) now try to fully populate them,
including newly added block reference field.

* Re-worked Peer.canUseCommonBlockData() to be more readable

* Cherry-picked patch to Message.fromByteBuffer() to pass an empty, read-only ByteBuffer to subclass fromByteBuffer() methods, instead of null.
This allows natural use of BufferUnderflowException if a subclass tries to use read(), or hasRemaining(), etc. from an empty data-payload message.
Previously this could have caused an NPE.
2022-09-24 13:48:01 +01:00
CalDescent
863a5eff97 Moved various online accounts logs to TRACE level, to make it easier to monitor the queue processing when in DEBUG. 2022-09-24 13:11:28 +01:00
CalDescent
5b81b30974 Modified online accounts request interval, and introduced bursting.
It will now request online accounts every 1 minute instead of every 5 seconds, except for the first 5 minutes following a new online accounts timestamp, in which it will request every 5 seconds (referred to as the "burst" interval). It will also use the burst interval for the first 5 minutes after the node starts.

This is based on the idea that most online accounts arrive soon after a new timestamp begins, and so there is no need to request accounts so frequently after that. This should reduce data usage by a significant amount.

Once mempow is fully rolled out, the "burst" feature can be reduced or removed, since online accounts will be sent ahead of time, generally 15-30 mins prior to the new online accounts timestamp becoming active.
2022-09-24 13:02:27 +01:00
CalDescent
174a779e4c Add accounts from the import queue individually, and then skip future duplicates before unnecessarily validating them again.
This closes a gap where accounts would be moved from onlineAccountsImportQueue to onlineAccountsToAdd, but not yet imported. During this time, there was nothing to stop them from being added to the import queue again, causing duplicate validations.
2022-09-24 10:56:52 +01:00
CalDescent
c7cf33ef78 Set hasOurOnlineAccounts to true if one of our accounts is found before signing. 2022-09-24 10:23:55 +01:00
CalDescent
ea4f4d949b When validating online accounts, enforce mempow if the online account's timestamp is after the feature trigger. 2022-09-23 19:45:59 +01:00
CalDescent
6d9e6e8d4c Allow duplicate variations of each OnlineAccountData in the import queue, but don't allow two entries that match exactly. 2022-09-23 18:46:01 +01:00
CalDescent
99858f3781 Wait 30 seconds after the node starts before computing our online accounts.
This allows some time for initial online account lists to be retrieved, and reduces the chances of the same nonce being computed twice.
2022-09-23 18:28:41 +01:00
CalDescent
84a16157d1 Don't add online accounts to the import queue if they are already validated 2022-09-23 18:02:46 +01:00
CalDescent
49d83650f4 Removed online accounts V2 and V1 messaging, as the V3 format will soon be required due to the nonce values. 2022-09-23 15:25:44 +01:00
CalDescent
951c85faf1 Fixed bug causing error 500 in some cases. 2022-09-20 22:26:30 +01:00
CalDescent
84d42b93e1 Reordered code in Block.mint() to fix potential issue after mempow activates. 2022-09-20 08:50:37 +01:00
CalDescent
b99b1f5d57 Bump version to 3.6.0 2022-09-19 17:29:26 +01:00
CalDescent
952c51ab25 QORA / block reward adjustments set to activate at height 1010000 2022-09-19 17:27:07 +01:00
CalDescent
64ef8ab863 OnlineAccountsV3Message.MIN_PEER_VERSION set to 3.6.0 2022-09-19 16:36:39 +01:00
CalDescent
93fd80e289 Require that add/remove admin transactions can only be created by group members.
For regular groups, we require that the owner adds/removes the admins, so group membership is adequately checked. However for null-owned groups this check is skipped. So we need an additional condition to prevent non-group members from issuing a transaction for approval by the group admins.
2022-09-19 16:34:31 +01:00
CalDescent
5581b83c57 Added initial admin approval features for groups owned by the null account.
* The dev group (ID 1) is owned by the null account with public key 11111111111111111111111111111111
 * To regain access to otherwise blocked owner-based rules, it has different validation logic
 * which applies to groups with this same null owner.
 *
 * The main difference is that approval is required for certain transaction types relating to
 * null-owned groups. This allows existing admins to approve updates to the group (using group's
 * approval threshold) instead of these actions being performed by the owner.
 *
 * Since these apply to all null-owned groups, this allows anyone to update their group to
 * the null owner if they want to take advantage of this decentralized approval system.
 *
 * Currently, the affected transaction types are:
 * - AddGroupAdminTransaction
 * - RemoveGroupAdminTransaction
 *
 * This same approach could ultimately be applied to other group transactions too.
2022-09-19 11:03:06 +01:00
CalDescent
5017072f6c Use path parameter instead of query string. 2022-09-17 13:50:04 +01:00
CalDescent
02ac6dd8c1 Added GET /chat/message/{signature} endpoint.
This will ease the transition to a Q-Chat protocol, where chat messages will no longer be regular transactions.
2022-09-17 13:28:32 +01:00
CalDescent
858269f6cb ChatTransaction MAX_DATA_SIZE increased from 256 to 1024 bytes, to allow for new UI features. 2022-09-17 12:21:56 +01:00
CalDescent
791a9b78ec Added support for Pirate Chain wallets on FreeBSD. 2022-09-17 10:36:25 +01:00
CalDescent
aff49e6bdf Added support for ARRR refunds via /crosschain/htlc/refund/{ataddress} and /crosschain/htlc/refundAll
This could probably be refactored into multiple classes to make the code cleaner, but it is functional for now.
2022-09-17 10:30:10 +01:00
CalDescent
2d29fdca00 Allow BTC trades in redeemAll / refundAll, since most will now be using ACCTv3. 2022-09-16 11:19:10 +01:00
CalDescent
063ef8507b Fix for NPE in last commit 2022-09-11 20:04:51 +01:00
CalDescent
f042b5ca5f If mempow is active, remove any legacy accounts from a to-be-minted block that are missing a nonce. 2022-09-11 11:41:11 +01:00
CalDescent
a10e669554 Allow nonce to be computed for "next" timestamp if mempow is enabled in settings. 2022-09-11 11:36:19 +01:00
CalDescent
501f66ab00 BlockTransformer updates necessary for mempow online accounts.
Using the feature trigger timestamp here should be much less error prone than a whole new block message version. Once mempow has been live for at least 24 hours, the feature trigger can be removed and the code cleaned up, as all online accounts signatures will use the new format from that time onwards (legacy signatures are trimmed after 24 hours).
2022-09-10 18:31:01 +01:00
CalDescent
6003ed3ff7 Revert "Use block's online accounts timestamp (instead of main timestamp) for the mempow hard fork."
This reverts commit 8cca6db316.
2022-09-10 17:23:39 +01:00
CalDescent
03e3619817 Revert "Use onlineAccountTimestamp for all mempow hard fork related code in OnlineAccountsManager too."
This reverts commit 23423102e7.
2022-09-10 17:23:34 +01:00
CalDescent
0e42e7b05a Merge branch 'master' into online-accounts-mempow-v2-block-updates
# Conflicts:
#	src/test/resources/test-chain-v2-no-sig-agg.json
2022-09-10 13:50:26 +01:00
CalDescent
d4fbc1687b Optionally exclude initial data from all trade websockets, using query string parameter excludeInitialData=true
Due to the large amount of data, it can take some time for the request to be processed and data to be transferred. It may make more sense to load the initial state from the standard API, and just use the websockets for updates.
2022-09-09 18:45:51 +01:00
DrewMPeacock
1abceada20 Fix up CREATE_POLL and VOTE_ON_POLL transactions to process and validate.
Added rule to enforce that a poll creator is also its owner.
2022-09-09 11:20:46 -06:00
CalDescent
8ffdc9b369 POST /crosschain/htlc/importarchivedtrades moved to POST /admin/repository/importarchivedtrades, as this is a repository operation not an HTLC one. 2022-09-09 18:01:20 +01:00
CalDescent
c883dd44c8 Remove LitecoinACCTv2 as it is no longer being used, and is unsafe. 2022-09-09 17:40:10 +01:00
CalDescent
667530e202 Remove DogecoinACCTv2 as it is no longer being used, and is unsafe. 2022-09-09 17:12:44 +01:00
CalDescent
5807d6e0dc Merge branch 'shares-by-level-rework' 2022-09-09 11:35:56 +01:00
CalDescent
ba4eeed358 Modified GET /arbitrary/resources endpoint (and underlying db queries) to allow filtering names by a list, e.g. "followedNames" or "blockedNames". 2022-09-07 18:42:24 +01:00
CalDescent
82edc4d9f3 OnlineAccountsV3Message MIN_PEER_VERSION set to 3.5.1 2022-09-07 18:26:30 +01:00
CalDescent
2a0d5746e6 Only compute "next" online account signature if mempow hard fork is active.
This minimizes the amount of differences in the first phase of the mempow rollout.
2022-09-04 13:19:32 +01:00
CalDescent
23423102e7 Use onlineAccountTimestamp for all mempow hard fork related code in OnlineAccountsManager too. 2022-09-04 11:51:24 +01:00
CalDescent
8879ec5bb4 Avoid computing the proof of work for each online account more than once. 2022-09-03 17:11:13 +01:00
CalDescent
8cca6db316 Use block's online accounts timestamp (instead of main timestamp) for the mempow hard fork.
This ensures that we cleanly switch from the old to new online account format, even if a block is minted retroactively.
2022-09-03 16:30:03 +01:00
CalDescent
effe1ac44d Merge branch 'master' into online-accounts-mempow-v2-block-updates
# Conflicts:
#	src/main/java/org/qortal/block/BlockChain.java
#	src/main/resources/blockchain.json
#	src/test/resources/test-chain-v2-no-sig-agg.json
2022-09-03 15:16:01 +01:00
CalDescent
ad4308afdf Added POST /crosschain/htlc/importarchivedtrades API endpoint.
This imports all trades from TradeBotStatesArchive.json into the repository.
2022-09-02 19:12:36 +01:00
CalDescent
6cfd85bdce Skip over ARRR orders in /refundAll and /redeemAll, as ARRR support hasn't been added for these yet.
This should fix 500 error which could prevent forced refund attempts for LTC and other chains. It wouldn't have affected any normal trade-bot refund functionality.
2022-09-02 18:05:12 +01:00
CalDescent
8b61247712 Added shareBinsByLevelV2.
This allows for different share bin distribution starting at an undecided future block height. This height will correspond with the QORA reduction. New values decided in recent community vote.
2022-09-02 16:43:46 +01:00
CalDescent
a9267760eb qoraHoldersShare reworked to qoraHoldersShareByHeight.
This allows the QORA share percentage to be modified at different heights, based on community votes. Added unit test to simulate a reduction.

# Conflicts:
#	src/test/java/org/qortal/test/minting/RewardTests.java
2022-08-29 09:40:20 +01:00
CalDescent
73396490ba Set walletsPath and listsPath to AppData folder for new Windows installs. 2022-08-27 19:44:31 +01:00
CalDescent
0b8fcc0a7b Bump version to 3.5.0 2022-08-26 19:32:58 +01:00
CalDescent
3d3ecbfb15 Merge branch 'master' into pirate-chain 2022-08-23 09:51:15 +01:00
CalDescent
9658f0cdd4 Revert "Rewrite of isNotOldPeer predicate, to fix logic issue."
This rewrite may have been causing problems with connections in the network, due to peers being forgotten too easily. Reverting for now to see if it solves the problem.

This reverts commit d81071f254.

# Conflicts:
#	src/main/java/org/qortal/network/Network.java
2022-08-21 12:18:40 +01:00
CalDescent
b23500fdd0 Attempt to hold peer connections for 1-4 hours instead of 5-60 mins, as the constant disconnections are causing too much data to be sent over and over.
If this proves to not have any significant bad effects on re-orgs, we could consider setting these even higher or even disabling the auto disconnect by default.
2022-08-21 12:16:44 +01:00
CalDescent
a1365e57d8 Don't log network stats if no messages have been received, as otherwise this floods the logs with empty stats due to failed connections. 2022-08-21 12:14:12 +01:00
CalDescent
d8ca3a455d Merge pull request #93 from catbref/peer-message-stats
Log count & total size of peer messages sent & received when a peer is disconnected
2022-08-21 11:41:37 +01:00
CalDescent
dcc943a906 Hopeful fix for InvalidKeyException seen in some JDK implementations. 2022-08-20 15:33:52 +01:00
CalDescent
cd2010bd06 More algo logging. 2022-08-20 12:50:24 +01:00
CalDescent
8cd16792a2 More logging relating to decryption failures. 2022-08-20 12:45:20 +01:00
CalDescent
4d97586f82 Log exception if AES decryption fails using specific algorithm settings. 2022-08-20 12:27:17 +01:00
CalDescent
3612fd8257 Removed unused throw 2022-08-20 11:58:16 +01:00
CalDescent
ff96868bd9 Log full stack trace if loading a QDN resource fails due to a DataException. 2022-08-20 11:57:53 +01:00
CalDescent
1694d4552e ArbitraryDataReader.deleteWorkingDirectory() is now optional. 2022-08-20 11:57:04 +01:00
CalDescent
bb1593efd2 Log full IOException stacktrace when obtaining Pirate Chain libraries. 2022-08-20 09:47:37 +01:00
CalDescent
4140546afb Ignore failures when deleting original QDN compressed file. 2022-08-20 09:45:48 +01:00
CalDescent
19197812d3 Log DataException during transaction validation. 2022-08-19 22:54:52 +01:00
CalDescent
168d32a474 Include memo for outgoing ARRR transactions. 2022-08-17 19:41:43 +01:00
CalDescent
a4fade0157 Validate wallet initialization result when restoring existing wallet. 2022-08-17 19:27:13 +01:00
CalDescent
2ea6921b66 Added more ARRR lightwalletd nodes 2022-08-17 19:26:47 +01:00
CalDescent
11ef31215b Fixed failing test 2022-08-17 19:25:45 +01:00
CalDescent
830a608b14 Include memo for incoming ARRR transactions. 2022-08-17 19:23:54 +01:00
CalDescent
57acf7dffe Updated text when downloading wallet files from QDN 2022-08-15 20:09:16 +01:00
CalDescent
9debebe03e Default birthday for ARRR moved to "arrrDefaultBirthday" (default 2000000).
This allows users to increase their default birthday if they know that no wallets were created before a certain block, to reduce sync time. It also fixed some failed unit tests that relied on transactions between blocks 1900000 and 2000000.
2022-08-13 16:48:03 +01:00
catbref
b17e96e121 Log count & total size of peer messages sent & received when a peer is disconnected. Requires org.qortal.net.Peer logging level set to DEBUG 2022-08-13 15:42:24 +01:00
CalDescent
b46c3cf95f Allow direct connection QDN retries every hour, instead of every 24 hours. 2022-08-13 14:20:10 +01:00
CalDescent
86526507a6 Increase time range and total number of attempts to fetch a QDN resource, as it previously gave up too quickly. 2022-08-12 22:25:35 +01:00
CalDescent
1b9128289f Added initial support to download Pirate wallet libraries from QDN, using hardcoded transaction signature.
Using a hardcoded signature ensures that the libraries cannot be swapped out without a core auto update, which requires the standard dev team approval process.
2022-08-12 21:57:31 +01:00
CalDescent
4a58f90223 Added support for multiple lightwalletd servers. 2022-08-12 21:52:02 +01:00
CalDescent
e68db40d91 Remove sapling params 2022-08-12 21:50:41 +01:00
CalDescent
bd6c0c9a7d qdata utility renamed to qdn 2022-08-12 13:38:30 +01:00
CalDescent
5804b9469c Default qdata port set to 12391 2022-08-12 13:38:03 +01:00
DrewMPeacock
4c463f65b7 Add API handles to build CREATE_POLL and VOTE_ON_POLL transactions. 2022-08-08 15:58:46 -06:00
CalDescent
53b47023ac ARRR default birthday increased to 2000000 2022-08-07 19:15:19 +01:00
CalDescent
22f9f08885 Added Windows amd64 architecture (again with a temporary path) 2022-08-07 19:14:21 +01:00
CalDescent
f26267e572 Don't sync, save, or load null seed wallets. They only act as a placeholder wallet when redeeming/refunding a P2SH. 2022-08-07 18:38:27 +01:00
CalDescent
e8c29226a1 Merge branch 'master' into pirate-chain 2022-08-06 16:41:18 +01:00
CalDescent
94f48f8f54 Don't allow QORT to be listed on the ARRR market unless the Pirate light wallet library is loaded. 2022-08-06 11:48:51 +01:00
CalDescent
3aac580f2c Add amd64 architecture (using a temporary path) 2022-08-06 11:47:36 +01:00
CalDescent
2d0b035f98 Updated AdvancedInstaller project for v3.4.3 2022-08-01 19:58:19 +01:00
CalDescent
075385d3ff Bump version to 3.4.3 2022-07-31 19:50:31 +01:00
CalDescent
6ed8250301 onlineAccountsModulusV2Timestamp set to Sat, 06 Aug 2022 16:00:00 UTC 2022-07-31 19:50:05 +01:00
CalDescent
d10ff49dcb Replaced arm architecture with aarch64, as 32 bit is unsupported. 2022-07-31 19:43:49 +01:00
CalDescent
4cf34fa932 Merge branch 'master' into pirate-chain 2022-07-31 16:00:52 +01:00
CalDescent
06b5d5f1d0 Merge branch 'increase-online-timestamp-modulus' 2022-07-31 13:00:47 +01:00
CalDescent
d6d2641cad Added "bitcoinjLookaheadSize" setting (default 50).
This replaces the WALLET_KEY_LOOKAHEAD_INCREMENT_BITCOINJ constant.
2022-07-31 12:07:44 +01:00
CalDescent
e71f22fd2c Added "gapLimit" setting.
This replaces the previously hardcoded "numberOfAdditionalBatchesToSearch" variable, and specifies the minimum number of empty consecutive addresses required before a set of wallet transactions is considered complete. Used for foreign transaction lists and balances.
2022-07-31 11:59:14 +01:00
CalDescent
c996633732 Added trace level logging. 2022-07-30 19:06:04 +01:00
CalDescent
55f973af3c Ensure all online accounts timestamps are a multiple of the online timestamp modulus.
This is a simple way to discard the 5-minute online account timestamps (from out of date nodes) once the switch to 30-minute online account timestamps has taken place.
2022-07-30 18:15:16 +01:00
CalDescent
fe9744eec6 Fixed missing feature trigger in testchain config 2022-07-30 14:52:47 +01:00
CalDescent
410fa59430 Merge branch 'master' into increase-online-timestamp-modulus
# Conflicts:
#	src/main/java/org/qortal/block/BlockChain.java
2022-07-30 12:23:25 +01:00
CalDescent
522ae2bce7 Updated AdvancedInstaller project for v3.4.2 2022-07-27 22:52:48 +01:00
CalDescent
a6e79947b8 Bump version to 3.4.2 2022-07-23 18:07:35 +01:00
CalDescent
b9bf945fd8 Removed aggregateSignatureTimestamp. All online account signatures are aggregated - there is no need for backwards support as signatures are trimmed from blocks after 24 hours. testOnlineAccountsModulusV2() had to be removed as this relied on pre-aggregation signatures. 2022-07-22 13:57:40 +01:00
CalDescent
85a27c14b8 Revert incorrect genesis timestamp that somehow made it into the stashed code. 2022-07-20 10:38:58 +01:00
CalDescent
46c40ca9ca Committed stashed code that is functional but probably too messy for production use.
Online account nonces are appended to the onlineAccountsSignatures to avoid the need for a new field, but it probably makes more sense to separate them.
2022-07-20 10:27:09 +01:00
CalDescent
fcd0d71cb6 Merge branch 'share-bin-activation' 2022-07-17 19:20:19 +01:00
catbref
275bee62d9 Revert BlockMinter to using long-lifetime repository session.
Although BlockMinter could reattach a repository session to its cache of potential blocks,
and these blocks would in turn reattach that repository session to their transactions,
further transaction-specific fields (e.g. creator PublicKeyAccount) were not being updated.

This would lead to NPEs like the following:

Exception in thread "BlockMinter" java.lang.NullPointerException
        at org.qortal.repository.hsqldb.HSQLDBRepository.cachePreparedStatement(HSQLDBRepository.java:587)
        at org.qortal.repository.hsqldb.HSQLDBRepository.prepareStatement(HSQLDBRepository.java:569)
        at org.qortal.repository.hsqldb.HSQLDBRepository.checkedExecute(HSQLDBRepository.java:609)
        at org.qortal.repository.hsqldb.HSQLDBAccountRepository.getBalance(HSQLDBAccountRepository.java:327)
        at org.qortal.account.Account.getConfirmedBalance(Account.java:72)
        at org.qortal.transaction.MessageTransaction.isValid(MessageTransaction.java:200)
        at org.qortal.block.Block.areTransactionsValid(Block.java:1190)
        at org.qortal.block.Block.isValid(Block.java:1137)
        at org.qortal.controller.BlockMinter.run(BlockMinter.java:301)

where the Account has an associated repository session which is now obsolete.

This commit reverts BlockMinter back to obtaining a repository session before entering main loop.
2022-07-17 13:53:07 +01:00
CalDescent
97221a4449 Added test to simulate level 7-8 reward tier activation, including orphaning. 2022-07-17 13:37:52 +01:00
CalDescent
508a34684b Revert "qoraHoldersShare reworked to qoraHoldersShareByHeight."
This reverts commit 90e8cfc737.

# Conflicts:
#	src/test/java/org/qortal/test/minting/RewardTests.java
2022-07-16 18:45:49 +01:00
CalDescent
3d2144f303 Check orphaning in levels 7-8 and 9-10 reward tests. This would have been tested in orphanCheck() anyway, but this makes the testing a bit more granular. 2022-07-16 17:29:16 +01:00
CalDescent
3c7fbed709 Fixed build error due to merge. 2022-07-15 11:53:30 +01:00
CalDescent
fb9a155e4c Merge branch 'master' into pirate-chain
# Conflicts:
#	src/main/java/org/qortal/api/resource/CrossChainTradeBotResource.java
#	src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java
2022-07-15 11:46:45 +01:00
CalDescent
fbcc870d36 Added informational test to compare ConsiceSet size against an int array for online account nonce arrays. 2022-07-15 10:23:14 +01:00
CalDescent
020e59743b Fixed failing test(s) due to merge. 2022-07-10 19:49:24 +01:00
CalDescent
0904de3f71 Merge branch 'master' into online-accounts-mempow-v2
# Conflicts:
#	src/main/java/org/qortal/block/BlockChain.java
2022-07-10 16:50:28 +01:00
CalDescent
35f3430687 Added share bin activation feature.
To prevent a single or very small number of minters receiving the rewards for an entire tier, share bins can now require "activation". This adds the requirement that a minimum number of accounts must be present in a share bin before it is considered active. When inactive, the rewards and minters are added to the previous tier.

Summary of new functionality:

- If a share bin has more than one, but less than 30 accounts present, the rewards and accounts are shifted to the previous share bin.
- This process is iterative, so the accounts can shift through multiple tiers until the minimum number of accounts is met, OR the share bin's starting level is less than shareBinActivationMinLevel.
- Applies to level 7+, so that no backwards support is needed. It will only take effect once the first account reaches level 7.

This requires hot swapping the sharesByLevel data to combine tiers where needed, so is a considerable shift away from the immutable array that was in place previously.

All existing and new unit tests are now passing, however a lot more testing will be needed.
2022-07-10 12:09:44 +01:00
CalDescent
90e8cfc737 qoraHoldersShare reworked to qoraHoldersShareByHeight.
This allows the QORA share percentage to be modified at different heights, based on community votes. Added unit test to simulate a reduction.
2022-07-08 11:12:58 +01:00
CalDescent
57bd3c3459 Merge remote-tracking branch 'catbref/auto-update-fix' 2022-07-07 18:48:39 +01:00
CalDescent
ad0d8fac91 Bump version to 3.4.1 2022-07-05 20:56:40 +01:00
CalDescent
a8b58d2007 Reward share limit activation timestamp set to 1657382400000 (Sat Jul 09 2022 16:00:00 UTC) 2022-07-05 20:34:23 +01:00
CalDescent
a099ecf55b Merge branch 'reduce-reward-shares' 2022-07-04 19:58:48 +01:00
CalDescent
6b91b0477d Added version query string param to /blocks/signature/{signature}/data API endpoint, to allow for optional V2 block serialization (with a single combined AT states hash).
Version can only be specified when querying unarchived blocks; archived blocks require V1 for now (and possibly V2 in the future).
2022-07-04 19:57:54 +01:00
CalDescent
fe2c63e8e4 Generate random nonces for test accounts.
These don't have to be valid for unit tests, because they are treated as "cached already valid accounts" in the block validation.
2022-07-02 17:30:31 +01:00
CalDescent
a3febdf00e Pass timestamp to OnlineAccountsManager.isMemoryPoWActive() so that block timestamp can be used. 2022-07-02 17:26:53 +01:00
CalDescent
4ca174fa0b Fixed bug in isMemoryPoWActive() which affected the ability to enable mempow via settings. 2022-07-02 13:45:39 +01:00
CalDescent
294582f136 Added mempow support in OnlineAccountsManager.
- This adds mempow requirements to online account importing (after activation timestamp), however doesn't yet add any requirements to block validation.
- It also causes the 'next' online accounts timestamp to be computed in addition to the 'current', so that the computed nonce value is ready when the next online accounts timestamp window begins.
2022-07-02 12:33:02 +01:00
CalDescent
d7e7c1f48c Fixed bugs from merge conflict, causing incorrect systray statuses in some cases. 2022-07-02 10:33:00 +01:00
CalDescent
215800fb67 Added optional "timeout" parameter to MemoryPoW.compute2().
This can be used to give up after the specified number of milliseconds.
2022-07-01 22:36:51 +01:00
CalDescent
b05d428b2e Added onlineAccountsMemPoWEnabled setting (for beta testing) 2022-07-01 22:31:51 +01:00
CalDescent
d2adadb600 Added onlineAccountsMemoryPoWTimestamp to blockchain.json 2022-07-01 22:31:30 +01:00
CalDescent
8e8c0b3fc5 Added OnlineAccountsV3Message, along with optional nonce Integer in OnlineAccountData.
This could potentially be released ahead of the other mempow code, splitting the rollout into multiple smaller phases.
2022-07-01 22:29:05 +01:00
CalDescent
65d63487f3 Merge branch 'master' into increase-online-timestamp-modulus 2022-07-01 17:46:35 +01:00
CalDescent
7c5932a512 GET /admin/status now returns online account submission status for "isMintingPossible", instead of BlockMinter status.
Online account credit is a more useful definition of "minting" than block signing, from the user's perspective. Should bring UI minting/syncing status in line with the core's systray status.
2022-07-01 17:29:15 +01:00
CalDescent
610a3fcf83 Improved order in getNodeType() 2022-07-01 16:48:57 +01:00
CalDescent
b329dc41bc Updated incorrect ONLINE_ACCOUNTS_V3_PEER_VERSION to 3.4.0 2022-07-01 13:36:56 +01:00
CalDescent
ff78606153 Merge branch 'master' into increase-online-timestamp-modulus 2022-07-01 13:14:15 +01:00
CalDescent
ef249066cd Updated another reference of SimpleTransaction::getTimestamp 2022-07-01 13:13:55 +01:00
CalDescent
80188629df Don't aggregate signatures when running OnlineAccountsTests, as it's too difficult to check how many unique signatures exist over a given period of time. 2022-07-01 13:13:22 +01:00
CalDescent
f77093731c Merge branch 'master' into increase-online-timestamp-modulus
# Conflicts:
#	src/main/java/org/qortal/block/Block.java
#	src/main/java/org/qortal/controller/OnlineAccountsManager.java
2022-07-01 13:03:19 +01:00
CalDescent
ca7d58c272 SimpleTransaction.timestamp is now in milliseconds instead of seconds.
Should fix 1970 timestamp issue in UI for foreign transactions, and also maintains consistency with QORT wallet transactions.
2022-07-01 12:46:20 +01:00
CalDescent
08f3351a7a Reward share transaction modifications:
- Reduce concurrent reward share limit from 6 to 3 (or from 5 to 2 when including self share) - as per community vote.
- Founders remain at 6 (5 when including self share) - also decided in community vote.
- When all slots are being filled, require that at least one is a self share, so that not all can be used for sponsorship.
- Activates at future undecided timestamp.
2022-07-01 12:18:48 +01:00
QuickMythril
f499ada94c Merge pull request #91 from qortish/master
Update SysTray_sv.properties
2022-06-29 08:44:47 -04:00
qortish
f073040c06 Update SysTray_sv.properties
proper
2022-06-29 14:04:27 +02:00
CalDescent
49bfb43bd2 Updated AdvancedInstaller project for v3.4.0 2022-06-28 22:56:11 +01:00
CalDescent
425c70719c Bump version to 3.4.0 2022-06-28 19:29:09 +01:00
CalDescent
1420aea600 aggregateSignatureTimestamp set to 1656864000000 (Sun Jul 03 2022 16:00:00 UTC) 2022-06-28 19:26:00 +01:00
CalDescent
4543062700 Updated blockchain.json files in unit tests to include an already active "aggregateSignatureTimestamp" 2022-06-28 19:22:54 +01:00
CalDescent
722468a859 Restrict relays to v3.4.0 peers and above, in attempt to avoid bugs causing older peers to break relay chains. 2022-06-27 19:38:30 +01:00
CalDescent
492a9ed3cf Fixed more message rebroadcasts that were missing IDs. 2022-06-26 20:02:08 +01:00
CalDescent
420b577606 No longer adding inferior chain signatures in comparePeers() as it doesn't seem 100% reliable in some cases. It's better to re-check weights on each pass. 2022-06-26 18:24:33 +01:00
CalDescent
434038fd12 Reduced online accounts log spam 2022-06-26 16:34:04 +01:00
CalDescent
a9b154b783 Modified BlockMinter.higherWeightChainExists() so that it checks for invalid blocks before treating a chain as higher weight. Otherwise minting is slowed down when a higher weight but invalid chain exists on the network (e.g. after a hard fork). 2022-06-26 15:54:41 +01:00
CalDescent
a01652b816 Removed hasInvalidBlock filtering, as this was unnecessary risk now that the original bug in comparePeers() is fixed. 2022-06-26 10:09:24 +01:00
CalDescent
4440e82bb9 Fixed long term bug in comparePeers() causing peers with invalid blocks to prevent alternate valid but lower weight candidates from being chosen. 2022-06-25 16:34:42 +01:00
CalDescent
a2e1efab90 Synchronize hasInvalidBlock predicate, as it wasn't thread safe 2022-06-25 14:12:21 +01:00
CalDescent
7e1ce38f0a Fixed major bug in hasInvalidBlock predicate 2022-06-25 14:11:25 +01:00
CalDescent
a93bae616e Invalid signatures are now stored as ByteArray instead of String, to avoid regular Base58 encoding and decoding, which is very inefficient. 2022-06-25 13:29:53 +01:00
CalDescent
a2568936a0 Synchronizer: filter out peers reporting to hold invalid block signatures.
We already mark peers as misbehaved if they returned invalid signatures, but this wasn't sufficient when multiple copies of the same invalid block exist on the network (e.g. after a hard fork). In these cases, we need to be more proactive to avoid syncing with these peers, to increase the chances of preserving other candidate blocks.
2022-06-25 12:45:19 +01:00
CalDescent
23408827b3 Merge remote-tracking branch 'catbref/schnorr-agg-BlockMinter-fix' into schnorr-agg-BlockMinter-fix
# Conflicts:
#	src/main/java/org/qortal/block/BlockChain.java
#	src/main/java/org/qortal/controller/OnlineAccountsManager.java
#	src/main/java/org/qortal/network/message/BlockV2Message.java
#	src/main/resources/blockchain.json
#	src/test/resources/test-chain-v2.json
2022-06-24 11:47:58 +01:00
CalDescent
ae6e2fab6f Rewrite of isNotOldPeer predicate, to fix logic issue (second attempt - first had too many issues)
Previously, a peer would be continuously considered not 'old' if it had a connection attempt in the past day. This prevented some peers from being removed, causing nodes to hold a large repository of peers. On slower systems, this large number of known peers resulted in low numbers of outbound connections being made, presumably because of the time taken to iterate through dataset, using up a lot of allKnownPeers lock time.

On devices that experienced the problem, it could be solved by deleting all known peers. This adds confidence that the old peers were the problem.
2022-06-24 10:36:06 +01:00
CalDescent
3af36644c0 Revert "Rewrite of isNotOldPeer predicate, to fix logic issue."
This reverts commit d81071f254.
2022-06-24 10:26:39 +01:00
CalDescent
db8f627f1a Default minPeerVersion set to 3.3.7 2022-06-24 10:13:55 +01:00
CalDescent
5db0fa080b Prune peers every 5 minutes instead of every cycle of the Controller thread.
This should reduce the amount of time the allKnownPeers lock is held.
2022-06-24 10:13:36 +01:00
CalDescent
d81071f254 Rewrite of isNotOldPeer predicate, to fix logic issue.
Previously, a peer would be continuously considered not 'old' if it had a connection attempt in the past day. This prevented some peers from being removed, causing nodes to hold a large repository of peers. On slower systems, this large number of known peers resulted in low numbers of outbound connections being made, presumably because of the time taken to iterate through dataset, using up a lot of allKnownPeers lock time.

On devices that experienced the problem, it could be solved by deleting all known peers. This adds confidence that the old peers were the problem.
2022-06-24 10:11:46 +01:00
QuickMythril
ba148dfd88 Added Korean translations
credit: TL (Discord username)
2022-06-23 02:35:42 -04:00
QuickMythril
ff40b8f8ab Updated German translations 2022-06-23 01:43:33 -04:00
CalDescent
dbcb457a04 Merge branch 'master' of github.com:Qortal/qortal 2022-06-20 22:51:33 +01:00
CalDescent
b00e1c8f47 Allow online account submission in all cases when in recovery mode. 2022-06-20 22:50:41 +01:00
CalDescent
899a6eb104 Rework of systray statuses
- Show "Minting" as long as online accounts are submitted to the network (previously it related to block signing).
- Fixed bug causing it to regularly show "Synchronizing 100%".
- Only show "Synchronizing" if the chain falls more than 2 hours behind - anything less is unnecessary noise.
2022-06-20 22:48:32 +01:00
CalDescent
6e556c82a3 Updated AdvancedInstaller project for v3.3.7 2022-06-20 22:25:40 +01:00
CalDescent
35ce64cc3a Bump version to 3.3.7 2022-06-20 21:52:41 +01:00
CalDescent
09b218d16c Merge branch 'master' of github.com:Qortal/qortal 2022-06-20 21:51:39 +01:00
CalDescent
7ea451e027 Allow trades to be initiated, and QDN data to be published, as long as the latest block is within 60 minutes of the current time. Again this should remove negative effects of larger re-orgs from the UX. 2022-06-20 21:26:48 +01:00
CalDescent
ffb27c3946 Further relaxed min latest block timestamp age to be considered "up to date" in a few places, from 30 to 60 mins. This should help reduce the visible effects of larger re-orgs if they happen again. 2022-06-20 21:25:14 +01:00
QuickMythril
6e7d2b50a0 Added Romanian translations
credit: Ovidiu (Telegram username)
2022-06-20 01:27:28 -04:00
QuickMythril
bd025f30ff Updated ApiError German translations
credit: CD (Discord username)
2022-06-20 00:56:19 -04:00
CalDescent
c6cbd8e826 Revert "Sync behaviour changes:"
This reverts commit 8a76c6c0de.
2022-06-19 19:10:37 +01:00
CalDescent
b85afe3ca7 Revert "Keep existing findCommonBlocksWithPeers() and comparePeers() behaviour prior to consensus switchover, to reduce the number of variables."
This reverts commit fecfac5ad9.
2022-06-19 19:10:30 +01:00
CalDescent
5a4674c973 Revert "newConsensusTimestamp set to Sun Jun 19 2022 16:00:00 UTC"
This reverts commit 55a0c10855.
2022-06-19 19:09:57 +01:00
CalDescent
769418e5ae Revert "Fixed unit tests due to missing feature trigger"
This reverts commit 28f9df7178.
2022-06-19 19:09:52 +01:00
CalDescent
38faed5799 Don't submit online accounts if node is more than 2 hours of sync. 2022-06-19 18:56:08 +01:00
catbref
10a578428b Improve intermittent auto-update failures, mostly under non-Windows environments.
Symptoms are:
* AutoUpdate trying to run new ApplyUpdate process, but nothing appears in log-apply-update.?.txt
* Main qortal.jar process continues to run without updating
* Last AutoUpdate line in log.txt.? is:
	2022-06-18 15:42:46 INFO  AutoUpdate:258 - Applying update with: /usr/local/openjdk11/bin/java -Djava.net.preferIPv4Stack=false -Xss256k -Xmx1024m -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=127.0.0.1:5005 -cp new-qortal.jar org.qortal.ApplyUpdate

Changes are:
* child process now inherits parent's stdout / stderr (was piped from parent)
* child process is given a fresh stdin, which is immediately closed
* AutoUpdate now converts -agentlib JVM arg to -DQORTAL_agentlib
* ApplyUpdate converts -DQORTAL_agentlib to -agentlib

The latter two changes are to prevent a conflict where two processes try to reuse the same JVM debugging port number.
2022-06-19 18:25:00 +01:00
CalDescent
96cdf4a87e Updated AdvancedInstaller project for v3.3.6 2022-06-19 11:41:50 +01:00
CalDescent
c0b1580561 Bump version to 3.3.6 2022-06-17 20:42:51 +01:00
CalDescent
28f9df7178 Fixed unit tests due to missing feature trigger 2022-06-17 13:07:33 +01:00
CalDescent
55a0c10855 newConsensusTimestamp set to Sun Jun 19 2022 16:00:00 UTC 2022-06-17 13:04:04 +01:00
CalDescent
7c5165763d Merge branch 'sync-long-tip' 2022-06-17 13:02:08 +01:00
CalDescent
d2836ebcb9 Make sure to set the ID when rebroadcasting messages. Hopeful fix for QDN networking issues. 2022-06-16 22:39:51 +01:00
CalDescent
fecfac5ad9 Keep existing findCommonBlocksWithPeers() and comparePeers() behaviour prior to consensus switchover, to reduce the number of variables. 2022-06-16 18:38:27 +01:00
CalDescent
5ed1ec8809 Merge remote-tracking branch 'catbref/sync-long-tip' into sync-long-tip 2022-06-16 18:35:16 +01:00
catbref
431cbf01af BlockMinter will discard block candidates that turn out to be invalid just prior to adding transactions, to be potentially reminted in the next pass 2022-06-16 17:47:08 +01:00
CalDescent
af792dfc06 Updated AdvancedInstaller project for v3.3.5 2022-06-15 21:11:12 +01:00
CalDescent
d3b6c5f052 Bump version to 3.3.5 2022-06-14 23:18:46 +01:00
CalDescent
f48eb27f00 Revert "Temporarily limit block minter as first stage of multipart minting fix. This will be reverted almost immediately after release."
This reverts commit 0eebfe4a8c.
2022-06-14 22:42:44 +01:00
CalDescent
b02ac2561f Revert "Safety check - also to be removed shortly."
This reverts commit 8b3f9db497.
2022-06-14 22:42:38 +01:00
CalDescent
1b2f66b201 Bump version to 3.3.4 2022-06-13 23:40:39 +01:00
CalDescent
e992f6b683 Increase column size to allow for approx 10x as many online accounts in each block. 2022-06-13 22:51:48 +01:00
CalDescent
8b3f9db497 Safety check - also to be removed shortly. 2022-06-13 22:41:10 +01:00
CalDescent
0eebfe4a8c Temporarily limit block minter as first stage of multipart minting fix. This will be reverted almost immediately after release. 2022-06-13 22:40:54 +01:00
lexandr0s
c03344caae Update ApiError_ru.properties 2022-06-04 23:57:25 +04:00
lexandr0s
237b39a524 Update SysTray_ru.properties 2022-06-04 23:50:03 +04:00
CalDescent
12b3fc257b Updated AdvancedInstaller project for v3.3.3 2022-06-04 19:50:43 +01:00
CalDescent
66a3322ea6 Bump version to 3.3.3 2022-06-04 19:16:01 +01:00
CalDescent
4965cb7121 Set BlockV2Message min peer version to 3.3.3 2022-06-04 15:50:17 +01:00
CalDescent
b92b1fecb0 disableReferenceTimestamp set to 1655222400000 (Tuesday, 14 June 2022 16:00:00 UTC) 2022-06-04 14:51:37 +01:00
CalDescent
43a75420d0 Merge branch 'disable-reference' 2022-06-04 14:46:51 +01:00
catbref
e85026f866 Initial work on reducing network load for transferring blocks.
Reduced AT state info from per-AT address + state hash + fees to AT count + total AT fees + hash of all AT states.
Modified Block and Controller to support above. Controller needs more work regarding CachedBlockMessages.
Note that blocks fetched from archive are in old V1 format.
Changed Triple<BlockData, List<TransactionData>, List<ATStateData>> to BlockTransformation to support both V1 and V2 forms.

Set min peer version to 3.3.203 in BlockV2Message class.
2022-06-04 14:43:46 +01:00
CalDescent
ba7b9f3ad8 Added feature to allow repository to be kept intact after running certain tests 2022-06-04 14:17:01 +01:00
catbref
4eb58d3591 BlockTimestampTests to show results from changing blockTimingsByHeight 2022-06-04 12:36:36 +01:00
catbref
8d8e58a905 Network$NetworkProcessor now has its own LOGGER 2022-06-04 12:36:36 +01:00
catbref
8f58da4f52 OnlineAccountsManager:
Bump v3 min peer version from 3.2.203 to 3.3.203
No need for toOnlineAccountTimestamp(long) as we only ever use getCurrentOnlineAccountTimestamp().
Latter now returns Long and does the call to NTP.getTime() on behalf of caller, removing duplicated NTP.getTime() calls and null checks in multiple callers.
Add aggregate-signature feature-trigger timestamp threshold checks where needed, near sign() and verify() calls.
Improve logging - but some logging will need to be removed / reduced before merging.
2022-06-04 12:36:36 +01:00
catbref
a4e2aedde1 Remove debug-hindering "final" modifier from effectively final locals 2022-06-04 12:36:36 +01:00
catbref
24d04fe928 Block.mint() always uses latest timestamped online accounts 2022-06-04 12:36:35 +01:00
catbref
0cf32f6c5e BlockMinter now only acquires repository instance as needed to prevent long HSQLDB rollbacks 2022-06-04 12:35:54 +01:00
catbref
84d850ee0b WIP: use blockchain feature-trigger "aggregateSignatureTimestamp" to determine when online-accounts sigs and block sigs switch to aggregate sigs 2022-06-04 12:35:51 +01:00
catbref
51930d3ccf Move some private key methods to Crypto class 2022-06-04 12:35:15 +01:00
catbref
c5e5316f2e Schnorr public key and signature aggregation for 'online accounts'.
Aggregated signature should reduce block payload significantly,
as well as associated network, memory & CPU loads.

org.qortal.crypto.BouncyCastle25519 renamed to Qortal25519Extras.
Our class provides additional features such as DH-based shared secret,
aggregating public keys & signatures and sign/verify for aggregate use.

BouncyCastle's Ed25519 class copied in as BouncyCastleEd25519,
but with 'private' modifiers changed to 'protected',
to allow extension by our Qortal25519Extras class,
and to avoid lots of messy reflection-based calls.
2022-06-04 12:35:15 +01:00
catbref
829ab1eb37 Cherry-pick minor fixes from another branch to resolve "No online accounts - not even our own?" issues 2022-06-04 12:35:03 +01:00
catbref
d9b330b46a OnlineAccountData no longer uses signature in equals() or hashCode() because newer aggregate signatures use random nonces and OAD class doesn't care about / verify sigs 2022-06-04 10:49:59 +01:00
catbref
c032b92d0d Logging fix: size() was called on wrong collection, leading to confusing logging output 2022-06-04 10:49:59 +01:00
catbref
ae92a6eed4 OnlineAccountsV3: slightly rework Block.mint() so it doesn't need to filter so many online accounts
Slight optimization to BlockMinter by adding OnlineAccountsManager.hasOnlineAccounts():boolean instead of returning actual data, only to call isEmpty()!
2022-06-04 10:49:59 +01:00
catbref
712c4463f7 OnlineAccountsV3:
Move online account cache code from Block into OnlineAccountsManager, simplifying Block code and removing duplicated caches from Block also.
This tidies up those remaining set-based getters in OnlineAccountsManager.
No need for currentOnlineAccountsHashes's inner Map to be sorted so addAccounts() creates new ConcurentHashMap insteaad of ConcurrentSkipListMap.

Changed GetOnlineAccountsV3Message to use a single byte for count of hashes as it can only be 1 to 256.
256 is represented by 0.

Comments tidy-up.
Change v3 broadcast interval from 10s to 15s.
2022-06-04 10:49:59 +01:00
catbref
fbdc1e1cdb OnlineAccountsV3:
Adding support for GET_ONLINE_ACCOUNTS_V3 to Controller, which calls OnlineAccountsManager.

With OnlineAccountsV3, instead of nodes sending their list of known online accounts (public keys),
nodes now send a summary which contains hashes of known online accounts, one per timestamp + leading-byte combo.
Thus outgoing messages are much smaller and scale better with more users.
Remote peers compare the hashes and send back lists of online accounts (for that timestamp + leading-byte combo) where hashes do not match.

Massive rewrite of OnlineAccountsManager to maintain online accounts.
Now there are three caches:
1. all online accounts, but split into sets by timestamp
2. 'hashes' of all online accounts, one hash per timestamp+leading-byte combination
Mainly for efficient use by GetOnlineAccountsV3 message constructor.
3. online accounts for the highest blocks on our chain to speed up block processing
Note that highest blocks might be way older than 'current' blocks if we're somewhat behind in syncing.

Other OnlineAccountsManager changes:
* Use scheduling executor service to manage subtasks
* Switch from 'synchronized' to 'concurrent' collections
* Generally switch from Lists to Sets - requires improved OnlineAccountData.hashCode() - further work needed
* Only send V3 messages to peers with version >= 3.2.203 (for testing)
* More info on which online accounts lists are returned depending on use-cases

To test, change your peer's version (in pom.xml?) to v3.2.203.
2022-06-04 10:49:59 +01:00
catbref
f2060fe7a1 Initial work on online-accounts-v3 network messages to drastically reduce network load.
Lots of TODOs to action.
2022-06-04 10:49:59 +01:00
catbref
6950c6bf69 Initial work on reducing network load for transferring blocks.
Reduced AT state info from per-AT address + state hash + fees to AT count + total AT fees + hash of all AT states.
Modified Block and Controller to support above. Controller needs more work regarding CachedBlockMessages.
Note that blocks fetched from archive are in old V1 format.
Changed Triple<BlockData, List<TransactionData>, List<ATStateData>> to BlockTransformation to support both V1 and V2 forms.

Set min peer version to 3.3.203 in BlockV2Message class.
2022-06-04 10:49:11 +01:00
catbref
8a76c6c0de Sync behaviour changes:
* Prefer longer chains
* Compare chain tips
* Skip pre-sync all-peer sorting (for now)
2022-06-04 10:23:31 +01:00
CalDescent
ef51cf5702 Added defensiveness in getOnlineTimestampModulus(), just in case NTP.getTime() returns null 2022-06-02 12:46:11 +01:00
CalDescent
0c3988202e Merge branch 'master' into increase-online-timestamp-modulus 2022-06-02 12:38:05 +01:00
CalDescent
987446cf7f Updated AdvancedInstaller project for v3.3.2 2022-06-02 11:31:56 +01:00
CalDescent
6dd44317c4 Sync pirate wallet every 30 seconds instead of 60, to match behaviour of official wallet. 2022-05-31 10:09:37 +02:00
CalDescent
d2fc705846 Merge remote-tracking branch 'catbref/UnsupportedMessage' 2022-05-31 09:09:08 +02:00
CalDescent
e393150e9c Require references to be the correct length post feature-trigger 2022-05-30 22:40:39 +02:00
CalDescent
43bfd28bcd Revert "Discard unsupported messages instead of disconnecting the peer."
This reverts commit d086ade91f.
2022-05-30 21:46:16 +02:00
catbref
ca8f8a59f4 Better forwards compatibility with newer message types so we don't disconnect newer peers 2022-05-30 20:41:45 +01:00
CalDescent
85a26ae052 Another rework of null seed wallets, to allow them to be saved and loaded.
A full sync is unavoidable for P2SH redeem/refund, so we need to be able to save our progress. Creating a new null seed wallet each time isn't an option because it relies on having a recent checkpoint to avoid having to sync large amounts of blocks every time (sync is per wallet, not per node).
2022-05-30 19:31:55 +02:00
CalDescent
c30b1145a1 Improved ensureSynchronized() as it would often not notice an unsynced wallet. 2022-05-30 18:14:26 +02:00
CalDescent
d086ade91f Discard unsupported messages instead of disconnecting the peer. 2022-05-30 15:27:42 +02:00
CalDescent
64d4c458ec Fixed logging error 2022-05-30 15:16:44 +02:00
CalDescent
2478450694 Revert "Removed "consecutive blocks" limitation in block minter."
This reverts commit f41fbb3b3d.
2022-05-30 13:49:15 +02:00
CalDescent
9f19a042e6 Added test to ensure short (1 byte) references can be imported. 2022-05-30 13:46:00 +02:00
CalDescent
922ffcc0be Modified post-trigger last reference checking, to now require a non-null value
This allows for compatibility with TRANSFER_PRIVS validation in commit 8950bb7, which treats any account with a non-null reference as "existing". It also avoids possible unknown side effects from trying to process and store transactions with a null reference - something that wouldn't have been possible until the validation was removed.
2022-05-30 13:22:15 +02:00
CalDescent
f887fcafe3 Disable last reference validation after feature trigger timestamp (not yet set).
This should prevent the failed transactions that are encountered when issuing two or more in a short space of time. Using a feature trigger (hard fork) to release this, to avoid potential consensus confusion around the time of the update (older versions could consider the main chain invalid until updating).
2022-05-30 13:06:55 +02:00
CalDescent
48b562f71b Auto update check interval slowed from 10s to 30s, to hopefully reduce the chance of encountering "repository in use by another process?" error. 2022-05-28 14:33:37 +02:00
CalDescent
5203742b05 Started work on architecture-specific lite wallet library loading. Paths are not yet correct. 2022-05-28 14:31:38 +02:00
CalDescent
f14b494bfc "Disposable" wallets renamed to "null seed" wallets, as this is a better description of what they are. 2022-05-28 14:28:42 +02:00
CalDescent
9a4ce57001 Increase blockchain lock wait time from 30 to 60 seconds in /transactions/process.
This will hopefully reduce the number of failed tradeoffer listings that result in a nonfunctional tradebot (and subsequent PENDING status shown in the UI)
2022-05-28 13:33:09 +02:00
CalDescent
10af961fdf Consider a node with a block in the last 30 mins to be "up to date" when trading. 2022-05-27 22:31:21 +02:00
CalDescent
33cffe45fd Bump version to 3.3.2 2022-05-27 17:23:53 +02:00
CalDescent
a0ce75a978 Minimum BTC order amount set to 0.001 BTC. Anything lower than that will result in greater than 10% fees. 2022-05-27 16:59:34 +02:00
QuickMythril
8d168f6ad4 Override default Bitcoin trade fee
Reduced to 20 sats/byte.
2022-05-27 06:57:38 -04:00
CalDescent
0875c5bf3b Fix ConcurrentModificationException in getCachedSigValidTransactions() 2022-05-27 10:34:26 +02:00
CalDescent
b17b28d9d6 Catch NoSuchMethodError in ElectrumX, and log it, just in case we ever reencounter a dependency issue. 2022-05-27 10:33:37 +02:00
CalDescent
e95249dc1b Reduced bouncycastle version to 1.69, as 1.70 was having compatibility issues with the ElectrumX code. 2022-05-27 10:30:28 +02:00
CalDescent
bb4bdfede5 Added concept of a "disposable" pirate chain wallet.
This is needed to allow redeem/refund of P2SH without having an actively synced and initialized wallet. It also ultimately avoids us having to retain the wallet entropy in the trade bot states. Various safety checks have been introduced to make sure that a disposable wallet is never used for anything other than P2SH redeem/refund.
2022-05-27 10:29:24 +02:00
CalDescent
e2b241d416 Fix ConcurrentModificationException in getCachedSigValidTransactions() 2022-05-27 10:15:41 +02:00
CalDescent
aeb94fb879 Merge branch 'master' into pirate-chain 2022-05-27 09:16:38 +02:00
QuickMythril
8e71cbd822 Reduce static Bitcoin trade fee 2022-05-26 16:25:07 -04:00
QuickMythril
9896ec2ba6 Fix typo 2022-05-26 15:37:38 -04:00
QuickMythril
9f9a74809e Add Bitcoin ACCTv3
This provides support for restoring BTC in the Trade Portal.
2022-05-26 15:34:51 -04:00
QuickMythril
acce81cdcd Add tray menu item to show Build Version
Core build version in a message dialog for OS which cannot display the entire tooltip.
2022-05-26 15:10:04 -04:00
CalDescent
d72953ae78 Drop expired transactions from the import queue before they are considered "sig valid".
This should prevent expired transactions from being kept alive, adding unnecessary load to the import queue.
2022-05-25 19:06:08 +01:00
CalDescent
32213b1236 Catch all exceptions in PirateLightClient 2022-05-24 20:48:57 +01:00
CalDescent
761d461bad Bump bouncycastle version to 1.70, necessary for ALPN support in some JDKs. 2022-05-24 20:35:19 +01:00
CalDescent
774a3b3dcd Catch RuntimeException, so that the gRPC client is shutdown. 2022-05-24 19:52:23 +01:00
CalDescent
30567d0e87 Correctly handle shutdown of gRPC managed channel on error. 2022-05-24 19:32:41 +01:00
CalDescent
6b53eb5384 Pirate Chain uses the 'b' prefix for P2SH addresses, but the light wallet library is configured to use 't3' (from Zcash), so it's easiest to just derive a different prefix for each destination.
This could be simplified by configuring the light wallet library to use the correct 'b' prefix, but this didn't work when first attempted.
2022-05-23 23:14:54 +01:00
CalDescent
767ef62b64 Added PirateChain.isValidWalletKey() 2022-05-23 22:11:23 +01:00
CalDescent
f7e6d1e5c8 Merge branch 'master' into pirate-chain 2022-05-23 21:55:49 +01:00
CalDescent
551686c2de Updated AdvancedInstaller project for v3.3.1 2022-05-23 21:54:25 +01:00
CalDescent
b73c041cc3 Bump version to 3.3.1 2022-05-23 20:31:36 +01:00
CalDescent
9e8d85285f Removed extra unnecessary digest after writing new data. 2022-05-22 16:13:08 +01:00
CalDescent
f41fbb3b3d Removed "consecutive blocks" limitation in block minter. 2022-05-22 16:13:04 +01:00
CalDescent
3f5240157e Return more detailed errors in trade portal APIs. 2022-05-22 16:11:20 +01:00
CalDescent
7c807f754e First draft of Pirate Chain trade bot. Not tested yet.
This has been modified to a) use full public keys instead of PKH, and b) hand off all transaction building, signing, and broadcasting to the (heavily customized) Pirate light wallet library.
2022-05-22 16:07:48 +01:00
CalDescent
9e1b23caf6 Removed unused constants. 2022-05-22 16:01:12 +01:00
CalDescent
c2bad62d36 Updated status text when not initialized. 2022-05-22 16:00:37 +01:00
CalDescent
4516d44cc0 UnspentOutput additions to support latest PirateChainHTLC methods 2022-05-22 15:55:40 +01:00
CalDescent
9c02b01318 Added PirateChainHTLC.getUnspentFundingTxid(), allowing a minimum amount to be specified.
This will ensure that the correct fundingTxid can be redeemed or refunded by the trade bot.
2022-05-22 15:55:01 +01:00
CalDescent
08fab451d2 Added PirateChainHTLC.getFundingTxid(), to lookup the txid that funded a P2SH. 2022-05-22 15:49:14 +01:00
CalDescent
d47570c642 First draft of Pirate Chain ACCT, with modifications to allow for 33 byte public keys instead of 20 byte PKH. Pirate Chain HTLCs are required to use full public keys rather than hashes. 2022-05-22 12:23:01 +01:00
CalDescent
4547386b1f Increase receiving_account_info column size from 32 to 128 bytes, to allow for Pirate Chain sapling shielded addresses, which are much longer. 2022-05-21 15:24:37 +01:00
CalDescent
ab01dc5e54 Implemented address validation for Pirate Chain 2022-05-21 09:22:49 +01:00
CalDescent
380c742aad Pirate chain minimum order size temporarily decreased to 0.0001, for testing only. 2022-05-21 09:11:24 +01:00
CalDescent
368359917b Use LegacyZcashAddress (copied and modified from bitcoinj) to derive and encode Pirate Chain P2SH addresses.
It wasn't possible to use bitcoinj directly because Zcash-style P2SH addresses have 2 prefix characters (t3), which isn't supported by bitcoinj.
2022-05-20 21:27:38 +01:00
CalDescent
1c1b570cb3 Added Pirate TLC and raw transaction tests 2022-05-20 21:22:18 +01:00
CalDescent
3fe43372a7 Added PirateChainHTLC - uses a different HTLC format suitable for Pirate Chain.
Also removes transaction building code, since this is handled by the light wallet library.
2022-05-20 21:21:51 +01:00
CalDescent
c7bc1d7dcd Hardcode fee to MAINNET_FEE when sending coins. 2022-05-20 21:16:12 +01:00
CalDescent
a7ea6ec80d Added wrapper methods for Pirate LiteWalletJni P2SH funding/redeeming/refunding.
These require a heavily customized version of both piratewallet-light-cli and librustzcash.
2022-05-20 21:14:50 +01:00
CalDescent
9cf574b9e5 Added support for Pirate Chain transaction lookups and deserialization, necessary for HTLC status checks. 2022-05-20 21:07:54 +01:00
CalDescent
20e63a1190 Removed "consecutive blocks" limitation in block minter. 2022-05-20 13:38:56 +01:00
CalDescent
f6fc5de520 Removed extra unnecessary digest after writing new data. 2022-05-20 13:36:56 +01:00
CalDescent
0b89118cd1 Fixed bug in Pirate wallet transaction parsing. 2022-05-20 13:35:52 +01:00
CalDescent
fa3a81575a Reduce wasted time that could otherwise be spent validating queued transaction signatures. 2022-05-14 12:53:36 +01:00
CalDescent
6990766f75 Speed up unconfirmed transaction propagation.
Currently, new transactions take a very long time to be included in each block (or reach the intended recipient), because each node has to obtain a repository lock and import the transaction before it notifies its peers. This can take a long time due to the lock being held by the block minter or synchronizer, and this compounds with every peer that the transaction is routed through.

Validating signatures doesn't require a lock, and so can take place very soon after receipt of a new transaction. This change causes each node to broadcast a new transaction to its peers as soon as its signature is validated, rather than waiting until after the import.

When a notified peer then makes a request for the transaction data itself, this can now be loaded from the sig-valid import queue as an alternative to the repository (since they won't be in the repository until after the import, which likely won't have happened yet).

One small downside to this approach is that each unconfirmed transaction is now notified twice - once after the signature is deemed valid, and again in Controller.onNewTransaction(), but this should be an acceptable trade off given the speed improvements it should achieve. Another downside is that it could cause invalid transactions (with valid signatures) to propagate, but these would quickly be added to each peer's invalidUnconfirmedTransactions list after the import failure, and therefore be ignored.
2022-05-14 12:43:54 +01:00
CalDescent
e1e1a66a0b Improvements and fixes to PirateWallet 2022-05-14 12:10:35 +01:00
CalDescent
e552994f68 Merge branch 'master' into pirate-chain 2022-05-13 13:23:26 +01:00
CalDescent
b33afd99a5 Bump invalid transaction import logging from trace to debug. 2022-05-13 13:22:58 +01:00
CalDescent
3c2ba4a0ea Improved logging when importing transactions. 2022-05-13 12:31:48 +01:00
CalDescent
ab0fc07ee9 Refactored transaction importer, to separate signature validation from importing.
Importing has to be single threaded since it requires the database lock, but there's nothing to stop us from validating signatures on multiple threads, as no lock is required. So it makes sense to separate these two functions to allow for possible multi threaded signature validation in the future, to speed up the process.

Everything remains single threaded in this commit. It should be functionally the same as before, to reduce risk.
2022-05-13 12:31:19 +01:00
CalDescent
001650d48e Fixed typo in currently unused getSignaturesInvolvingAddress() method. 2022-05-13 11:28:55 +01:00
CalDescent
659431ebfd Added "txTypes" parameter to GET /transactions/unconfirmed, to allow optional filtering of unconfirmed transactions by one or more types 2022-05-13 11:28:21 +01:00
CalDescent
0a419cb105 Added "creator" parameter to GET /transactions/unconfirmed, to allow filtering unconfirmed transactions by creator's public key 2022-05-13 11:13:39 +01:00
CalDescent
a4d4d17b82 Added /wallets to .gitignore, in preparation for pirate chain support. 2022-05-12 19:34:53 +01:00
CalDescent
0829ff6908 Fixed bugs in tooltip, introduced in lite node branch 2022-05-12 19:33:12 +01:00
CalDescent
2d1b0fd6d0 Merge branch 'lite-node' 2022-05-12 08:51:05 +01:00
CalDescent
122539596d Merge branch 'qdn-direct-connections' 2022-05-12 08:45:45 +01:00
CalDescent
86015e59a1 Updated AdvancedInstaller project for v3.3.0 2022-05-10 08:27:17 +01:00
CalDescent
107a23f1ec Switch to PirateChainMainNetParams 2022-05-10 08:05:32 +01:00
CalDescent
abce068b97 Catch UnsatisfiedLinkError when initializing Pirate Chain library 2022-05-10 08:05:06 +01:00
CalDescent
28fd9241d4 Fixed issues with merge 2022-05-10 08:01:07 +01:00
CalDescent
3fc4746a52 Merge branch 'master' into pirate-chain
# Conflicts:
#	src/test/java/org/qortal/test/crosschain/DigibyteTests.java
#	src/test/java/org/qortal/test/crosschain/RavencoinTests.java
2022-05-10 07:54:47 +01:00
CalDescent
1ea1e00344 Bump version to 3.3.0 2022-05-09 18:47:19 +01:00
CalDescent
598f219105 Don't require a full synchronization in order to return the ARRR wallet address. 2022-05-08 08:26:00 +01:00
CalDescent
bbf7193c51 Fixed bugs in wallet initialization 2022-05-08 08:24:54 +01:00
CalDescent
adecb21ada Added mainnet xpub addresses for DGB and RVN tests, as testnet support isn't fully implemented yet. 2022-05-07 17:40:36 +01:00
CalDescent
fa4679dcc4 Updated DGB and RVN tests due to conflict. 2022-05-07 17:05:10 +01:00
CalDescent
58917eeeb4 "walletsPath" is now configurable in the settings. 2022-05-07 17:01:19 +01:00
CalDescent
f36e193650 Merge branch 'master' into pirate-chain
# Conflicts:
#	src/main/java/org/qortal/settings/Settings.java
2022-05-07 16:50:32 +01:00
CalDescent
dac484136f Fixed bug in name rebuilding. 2022-05-07 16:46:10 +01:00
CalDescent
999ad857ae Fixed typo 2022-05-07 16:33:10 +01:00
CalDescent
d073b9da65 Added support for Pirate Chain wallets.
Note: this relies on (a modified version of) liblitewallet-jni which is not included, but will ultimately be compiled for each supported architecture and hosted on QDN.

LiteWalletJni code is based on https://github.com/PirateNetwork/cordova-plugin-litewallet - thanks to @CryptoForge for the help in getting this up and running.
2022-05-07 16:32:04 +01:00
CalDescent
aaa0b25106 Make sure to set Peer.isDataPeer() to false as well as true, to prevent bugs due to object reuse.
Also designate a peer as a "data peer" when making an outbound connection to request data from it.
2022-05-02 10:20:23 +01:00
CalDescent
f7dabcaeb0 Increase ONLINE_ACCOUNTS_MODULUS from 5 to 30 mins at a future undecided timestamp.
Note: it's important that this timestamp is set on a 1-hour boundary (such as 16:00:00) to ensure a clean switchover.

# Conflicts:
#	src/main/java/org/qortal/block/BlockChain.java
2022-05-02 08:50:08 +01:00
CalDescent
3409086978 Merge branch 'master' into lite-node 2022-05-02 08:31:43 +01:00
CalDescent
6c201db3dd Merge branch 'EPC-fixes' 2022-05-02 08:28:14 +01:00
CalDescent
da47df0a25 Fixed merge issue. 2022-05-01 16:41:56 +01:00
CalDescent
eea215dacf Merge branch 'ravencoin' into new-coins
# Conflicts:
#	pom.xml
#	src/main/java/org/qortal/controller/tradebot/TradeBot.java
#	src/main/java/org/qortal/crosschain/SupportedBlockchain.java
#	src/main/java/org/qortal/settings/Settings.java
2022-05-01 16:28:22 +01:00
CalDescent
0949271dda Merge branch 'digibyte' into new-coins 2022-05-01 16:25:40 +01:00
CalDescent
6bb9227159 Removed RavencoinACCTv1
Also removed CrossChainRavencoinACCTv1Resource - same as Digibyte.
2022-05-01 16:17:00 +01:00
CalDescent
a95a37277c Removed DigibyteACCTv1 and v2
Also removed CrossChainDigibyteACCTv1Resource, since this is unused, and it seems excessive to maintain support of this for every coin (and potentially every ACCT version).
2022-05-01 16:12:12 +01:00
CalDescent
48b9aa5c18 Allow images to be displayed in QDN websites via data: and blob: 2022-05-01 14:37:57 +01:00
CalDescent
1d7203a6fb Bug fixes found when testing previous commits. 2022-05-01 14:29:24 +01:00
CalDescent
1030b00f0a Keep track of peers requesting data for which we have at least one chunk. Then allow subsequent incoming connections from that peer through, up to a maximum of maxDataPeers.
Direct connections for arbitrary data are currently unlikely to succeed, because those allowing incoming connections generally have their slots maxed out and have reached maxPeers. The idea here is that some connections remain reserved for dedicated arbitrary data transfers, therefore temporarily circumventing the limit (up to a defined maximum number of reserved connections).

Arbitrary data connections will auto disconnect after 2 minutes (we might be able to reduce this at a later date), and it also probably makes sense for the requesting node to disconnect as soon as it has all the chunks that it needs (this part isn't implemented yet).

One downside of this feature is that the listen socket is now going to be accepting connections most of the time, since it is unlikely that we will regularly have 4 data peers connected. This could be improved by modifying the OP_ACCEPT behaviour based on whether we are expecting any data peers to connect. In most cases, this would allow it to remain closed. But for the sake of simplicity I will leave that optimization for a future commit.
2022-05-01 14:02:44 +01:00
CalDescent
0c16d1fc11 Added "maxDataPeerConnectionTime" setting (default 2 mins).
This is used to force a quick disconnect for peers that are only connecting for the purposes of requesting data for a specific arbitrary transaction signature.
2022-05-01 14:02:44 +01:00
CalDescent
ed04375385 Increased default maxPeers from 32 to 36 to compensate - otherwise the network will lose a considerable amount of inbound capacity. 2022-05-01 14:02:44 +01:00
CalDescent
6e49d20383 Added "maxDataPeers" setting to reserve 4 connections by default for direct QDN data requests. 2022-05-01 14:02:44 +01:00
CalDescent
dc34eed203 Include our address when requesting QDN data 2022-05-01 14:02:44 +01:00
CalDescent
fbe4f3fad8 Fixed incorrect minOutboundPeers conditional 2022-05-01 14:01:58 +01:00
CalDescent
e7ee3a06c7 Merge branch 'EPC-fixes' into lite-node 2022-04-30 15:33:07 +01:00
CalDescent
599877195b Merge branch 'master' into EPC-fixes 2022-04-30 15:32:44 +01:00
CalDescent
7f9d267992 Improved lite node response error logging. 2022-04-30 15:32:23 +01:00
CalDescent
52904db413 Migrated new lite node message types to new format. 2022-04-30 15:22:50 +01:00
CalDescent
5e0bde226a Merge branch 'EPC-fixes' into lite-node
# Conflicts:
#	src/main/java/org/qortal/network/message/Message.java
2022-04-30 13:25:02 +01:00
CalDescent
0695039ee3 Fixed long term bug causing last line to be missed out. 2022-04-30 12:08:10 +01:00
CalDescent
a4bcd4451c Added "tail" parameter to GET /admin/logs to allow returning the last X (limit) lines.
This should make it easy to display core logs in the UI.
2022-04-30 12:07:47 +01:00
CalDescent
e5b4b61832 Fixed bugs causing "Hash ... does not match file digest ..." errors 2022-04-30 11:26:05 +01:00
CalDescent
dd55dc277b Updated AdvancedInstaller project for v3.2.5 2022-04-27 08:50:40 +01:00
CalDescent
81ef1ae964 Bump version to 3.2.5 2022-04-26 20:17:57 +01:00
CalDescent
46701e4de7 Revert "Remove peers with unknown height, lower height or same height and same block signature (unless we don't have their block signature)"
This reverts commit 895f02f178.
2022-04-26 19:52:08 +01:00
QuickMythril
0f52ccb433 add Ravencoin ACCTs 2022-04-26 13:51:19 -04:00
QuickMythril
8aed84e6af add Digibyte ACCTs 2022-04-26 11:40:42 -04:00
CalDescent
568497e1c5 Updated AdvancedInstaller project for v3.2.4 2022-04-25 09:03:57 +01:00
CalDescent
f3f8e0013d Bump version to 3.2.4 2022-04-24 17:48:22 +01:00
CalDescent
d03c145189 Added to testRegisterNameFeeIncrease() test to catch the recently detected bug. 2022-04-24 17:41:24 +01:00
CalDescent
682a5fde94 Revert "Attempt to fix core startup problems on some systems (GNOME Desktop?) by adding defensiveness to GUI elements."
This reverts commit 311f41c610.
2022-04-24 15:54:20 +01:00
CalDescent
cca5bac30a Fixed logic bug in name registration fee calculation. 2022-04-24 15:36:36 +01:00
CalDescent
64e102a8c6 Name registration fee reduction to 1.25 QORT set to Sun, 01 May 2022 16:00:00 GMT 2022-04-24 15:27:21 +01:00
CalDescent
f9972f50e0 Updated altcoinj 2022-04-24 15:08:43 +01:00
QuickMythril
05d9a7e820 Switched to Qortal fork of altcoinj, using RavencoinMainNetParams 2022-04-23 08:28:12 -04:00
CalDescent
df290950ea Reduce log spam in BlockMinter 2022-04-23 12:32:06 +01:00
CalDescent
ae64be4802 Retry scheduled repository maintenance up to 5 times, as it's common for it to timeout waiting for the repository. Subsequent retries normally succeed. 2022-04-23 12:31:15 +01:00
CalDescent
348f3c382e Merge branch 'master' into lite-node 2022-04-22 20:40:47 +01:00
CalDescent
d98678fc5f Renamed SECRET_LENGTH to SECRET_SIZE_LENGTH. Thanks to catbref for finding this. 2022-04-22 20:40:13 +01:00
CalDescent
1da157d33f Added separate AT serialization tests, based on generic transaction serialization tests. This allows for testing both MESSAGE-type and PAYMENT-type AT transactions. 2022-04-22 20:36:22 +01:00
CalDescent
de4f004a08 Bump to transaction version 6 at a future undecided timestamp. 2022-04-22 20:35:17 +01:00
CalDescent
522ef282c8 Added support for deserialization of MESSAGE-type AT transactions (requires transaction version 6) 2022-04-22 20:34:42 +01:00
CalDescent
b5522ea260 Added support for PAYMENT-type AT transactions in serialization tests 2022-04-22 20:06:37 +01:00
CalDescent
b1f184c493 Use DigibyteMainNetParams 2022-04-22 16:31:44 +01:00
CalDescent
d66dd51bf6 Switched to Qortal fork of altcoinj, with Digibyte support 2022-04-22 16:31:32 +01:00
QuickMythril
0baed55a44 add DGB wallet 2022-04-21 11:40:17 -04:00
QuickMythril
390b359761 add RVN wallet 2022-04-21 11:38:49 -04:00
CalDescent
311f41c610 Attempt to fix core startup problems on some systems (GNOME Desktop?) by adding defensiveness to GUI elements. 2022-04-20 08:41:37 +01:00
CalDescent
0a156c76a2 Fix for NPE observed on the EPC-fixes branch (but putting the fix on master in case unrelated) 2022-04-20 08:38:59 +01:00
CalDescent
70eaaa9e3b Merge remote-tracking branch 'catbref/EPC-fixes' into EPC-fixes 2022-04-19 20:50:32 +01:00
catbref
3e622f7185 EPC-fixes: catch CancelledKeyExceptions thrown in short window between nextSelectionKey.isValid() and nextSelectionKey.isXXXable() calls 2022-04-18 14:33:05 +01:00
CalDescent
3f12be50ac Merge remote-tracking branch 'catbref/EPC-fixes' into EPC-fixes 2022-04-18 09:20:55 +01:00
catbref
68412b49a1 EPC-fixes: use bindAddress from Settings for outgoing peer connections, not just listen socket 2022-04-17 19:38:50 +01:00
catbref
c9b2620461 EPC-fixes: fix constructing GET_ONLINE_ACCOUNTS_V2 message for case where onlineAccount args is empty list 2022-04-17 19:37:28 +01:00
CalDescent
337b03aa68 Catch java.util.ServiceConfigurationError in Gui.loadImage() 2022-04-17 17:59:29 +01:00
catbref
df3f16ccf1 EPC-fixes: Improve Network shutdown by exiting fast during broadcast and skipping callbacks during peer disconnect. 2022-04-17 14:50:15 +01:00
catbref
22aa5c41b5 WIP: EPC-fixes
BlockMessage was broken because the repository 'connection' associated with the message's Block object was closed between message queuing and message sending.

The fix was to serialize Message subclasses on construction, thus freeing reliance on objects passed into constructor.
The serialized byte[] is held by the message between queuing and sending.
This forces messages into one of two 'modes': outgoing or incoming.
Outgoing messages contain serialized byte[] whereas incoming messages unpack a ByteBuffer into Message subclass fields.
As a result, all network message types have been refactored in this way.
More details in Message's class comment.

A knock-on effect is that incoming messages cannot then be sent out - a new message needs to be constructed.
Some changes needed to Arbitrary controller package classes in this respect.

Bonus: Network no longer needs broadcast threads because 'broadcasting' is now simply the act of queuing a message for many peers.
2022-04-17 14:50:15 +01:00
catbref
8e09567221 EPC-fixed: avoiding some CancelledKeyExceptions 2022-04-17 14:50:15 +01:00
catbref
3505788d42 Another chunk of improvements to networking / EPC.
Instead of synchronizing/blocking in Peer.sendMessage(),
we queue messages in a concurrent blocking TransferQueue, with timeout.

In EPC, ChannelWriteTasks consume from TransferQueue, unblocking callers to Peer.sendMessage().

If a new message is to be sent, or socket output buffer is full,
then OP_WRITE is used to wait for socket to become writable again.

Only one ChannelWriteTask per peer can be active/pending at a time.
Each ChannelWriteTask tries to send as much as it can in one go.

Other minor tidy-ups.
2022-04-17 14:50:15 +01:00
catbref
91e0c9b940 More improvements to networking:
As per work done by szisti in PR#45:
Extracted network 'Tasks' to their own classes.
Network.NetworkProcessor reduced to only producing Tasks.

Improved usage of SocketChannel interest-ops.
Eventually this might lead to reducing task-producing synchronization lock into more granular locks.
Work still needed to convert sending messages to a queue and to make use of OP_WRITE instead of sleeping to wait for socket buffer to empty.

Disabled PeerConnectTask producer from checking against connected peers via DNS as it's too slow.

Swapped Peer's replyQueues from SynchronizedMap(wrapped HashMap) to ConcurrentHashMap.

Other minor changes within networking.
2022-04-17 14:50:15 +01:00
catbref
00996b047f Networking work-in-progress:
As per work done by szisti in PR#45:
Extracted MessageException from inside Message into its own class.
Extracted MessageType from inside Message into its own class.

Converted reflection-based Message.fromByteBuffer method call to non-reflection, functional interface, method-reference.
This should have minor performance improvement but stronger method signature and type enforcement, as well as better IDE integration.

Message.fromByteBuffer method 'contract' tightened up to:
1. throw BufferUnderflowException if there are not enough bytes to deserialize message
2. throw MessageException if bytes contain invalid data
3. should not return null

Message.toData method 'contract' tightened up to:
1. return null if the message has no payload to serialize
2. throw IOException directly - no need to try-catch in each subclass

Several Message-subclass fields now marked 'final' as per IDE suggestion.
Several Message-subclass fromByteBuffer() method signatures have changed 'throws' list.
Several bytes.remaining() != some-value changed to bytes.remaining() < some-value as per new contract.

Some bytes.remaining() checks removed for fixed-length messages because we can rely on ByteBuffer throwing BufferUnderflowException.
Some bytes.remaining() checks retained for variable-length messages, or messages that read a large amount of data, to prevent wasted memory allocations.

Other minor tidying up
2022-04-17 14:50:15 +01:00
catbref
44fc0f367d Networking work-in-progress:
Temporarily increase sleep from 1ms to 100ms when waiting for outgoing socket buffer to empty.

Real fix is to rewrite using an outgoing message queue and OP_WRITE interest op.
2022-04-17 14:50:15 +01:00
catbref
b0e6259073 Networking work-in-progress:
De-register a peer's socket channel OP_READ interest op when producing a ChannelTask for that peer.
This should prevent duplicate ChannelTasks for the same peer.

Re-register OP_READ once node has read from peer's channel.
2022-04-17 14:50:15 +01:00
catbref
6255b2a907 Networking work-in-progress:
When node has reached max connections, Network will ignore pending incoming connections by:
1. not calling accept()
2. de-registering OP_ACCEPT 'interest op' on the listen socket's channel

When a peer disconnects, Network might re-register OP_ACCEPT interest op on listen socket.
2022-04-17 14:50:15 +01:00
catbref
a5fb0be274 Fix Network.disconnectPeer(PeerAddress) to prevent removeIf() on UnmodifiableList throwing UnsupportedOperationException 2022-04-17 14:50:15 +01:00
catbref
e835f6d998 ExecuteProduceConsume:
Slight reworking of EPC to simplify when producer can block
and generally make some of the conditional code more readable.

Improved logging with task class names and logging level editable during runtime!
Use /peer/enginestats?newLoggingLevel=DEBUG (or TRACE or back to INFO) to change.
2022-04-17 14:50:15 +01:00
CalDescent
54ff564bb1 Set name for transaction importer thread 2022-04-16 23:32:42 +01:00
CalDescent
f8a5ded0ba Fix for bug introduced in commit cfe9252 2022-04-16 23:20:49 +01:00
CalDescent
a1be66f02b Temporarily ease the filtering of lite node peers, in order to make development easier. 2022-04-16 20:52:31 +01:00
CalDescent
0815ad2cf0 Added AT transaction deserialization, to all them to be sent in messages for lite nodes.
Note that it is currently not easy to distinguish between MESSAGE-type and PAYMENT-type AT transactions, so PAYMENT-type is currently the only one supported (and used). A hard fork will likely be needed in order to specify the type within each message.
2022-04-16 20:52:31 +01:00
CalDescent
3484047ad4 Added GET /transactions/address/{address} API endpoint
This is a more standardized alternative to using GET /transactions/search?address=xyz. This avoids the need to build full transaction search ability into the lite node protocols right away.
2022-04-16 20:52:31 +01:00
CalDescent
a63fa1cce5 Added GET_ACCOUNT_TRANSACTIONS message, as well as a generic TRANSACTIONS message for responses. 2022-04-16 20:52:31 +01:00
CalDescent
59119ebc3b Added GET_NAME message to allow lookups from name to owner (or any other name data). 2022-04-16 20:52:31 +01:00
CalDescent
276f1b7e68 Fixed small errors in earlier commits. 2022-04-16 20:52:31 +01:00
CalDescent
c482e5b5ca Added GET_ACCOUNT_NAMES message to request names for an address, and a generic NAMES message to return a list of NameData objects. The generic NAMES message can be reused for many other responses, such as requesting the various lists of names that the API supports. 2022-04-16 20:52:31 +01:00
CalDescent
8c3e0adf35 Added message types to fetch account details and account balances, and use these in various APIs.
This should bring in enough data for very basic chat and wallet functionality (using addresses rather than registered names).

Data currently comes from a single random peer, however this can be expanded to request from multiple peers to gain confidence in the accuracy of the data. If bad data is returned from a peer, it's not the end of the world since the transaction would just be considered invalid by full nodes and would be thrown out. But this should be mostly avoidable by taking data from multiple sources to improve confidence in its accuracy.
2022-04-16 20:52:30 +01:00
CalDescent
64ff3ac672 Improved comment 2022-04-16 20:52:30 +01:00
CalDescent
cfe92525ed Disable various core functions when running as a lite node.
Lite nodes can't sync or mint blocks, and they also have a very limited ability to verify unconfirmed transactions due to a lack of contextual information (i.e. the blockchain). For now, most validation is skipped and they simply act as relays to help get transactions around the network. Full and topOnly nodes will disregard any invalid transactions upon receipt as usual, and since the lite nodes aren't signing any blocks, there is little risk to the reduced validation, other than the experience of the lite node itself. This can be tightened up considerably as the lite nodes become more powerful, but the current approach works as a PoC.
2022-04-16 20:52:30 +01:00
CalDescent
0e3a9ee2b2 Return the node "type" (full / topOnly / lite) in GET /admin/info endpoint.
This can used by the UI to hide features that aren't supported on lite nodes.
2022-04-16 20:50:27 +01:00
CalDescent
a921db2cc6 Added "lite" setting to designate the core as a lite node. 2022-04-16 20:50:27 +01:00
CalDescent
3d99f86630 Improved logging 2022-04-16 20:50:00 +01:00
CalDescent
8d1a58ec06 POW_DIFFICULTY_NO_QORT reduced from 14 to 12 (around 4x faster) 2022-04-16 12:36:32 +01:00
CalDescent
2e5a7cb5a1 Adapted Blockchain.java to use lookup table for name registration fees, to more easily support fee adjustments.
This is currently for name registration transactions only, but can be adapted (or duplicated) for other transaction types when needed.

Note: this switches from a greater-than (>) to a greater-than-or-equal (>=) timestamp comparison, as it makes more sense this way. It shouldn't affect the previous transition since there were no REGISTER_NAME transactions at that exact timestamp.
2022-04-16 12:20:03 +01:00
CalDescent
895f02f178 Remove peers with unknown height, lower height or same height and same block signature (unless we don't have their block signature)
Adapted from code originally written by catbref from before genesis, and essentially prevents syncing backwards. This needs significant testing on testnet.
2022-04-16 11:30:07 +01:00
CalDescent
c59869982b Fix for system-wide QDN issues occuring when the metadata file has an empty chunks array.
It is quite likely that existing resources with both metadata and an empty chunks array will need to be republished, because this bug may have led to incorrect file deletions.
2022-04-16 11:25:44 +01:00
CalDescent
3b3368f950 Merge pull request #85 from QuickMythril/member-count
Add member count to each group returned by GET /member/{address}
2022-04-16 11:00:35 +01:00
QuickMythril
3f02c760c2 Add member count to each group returned by GET /member/{address} 2022-04-15 06:23:10 -04:00
CalDescent
fee603e500 Add member count to each group returned by GET /groups (expanded on code written by QuickMythril) 2022-04-15 10:19:43 +01:00
QuickMythril
ad31d8014d get memberCount with Group Data
works for lookup by groupId
2022-04-14 22:08:52 -04:00
CalDescent
58a0ac74d2 Merge pull request #84 from catbref/ByteArray
Improvements to ByteArray to leverage Java 11 'native' Arrays methods
2022-04-14 21:30:59 +01:00
CalDescent
184984c16f Added ElectrumX equivalent for Pirate Chain, to communicate with a remote lightwalletd
This will most likely be used for by the trade bot, rather than for any wallet functionality.
2022-04-14 20:18:58 +01:00
CalDescent
2cf7a5e114 Generated java gRPC protocol buffers for Pirate Chain network comms.
The command used was:

./protoc --plugin=protoc-gen-grpc-java=/Users/user/Downloads/protoc-gen-grpc-java-1.45.1-osx-x86_64.exe -I=src/main/resources/proto/zcash/ --java_out=src/main/java/ --grpc-java_out=src/main/java/ src/main/resources/proto/zcash/service.proto

Then repeat, replacing service.proto with compact_formats.proto and darkside.proto

Darkside isn't needed for mainnet functionality, but included for completeness, and might be useful for testing.
2022-04-14 20:07:42 +01:00
QuickMythril
8388aa9c23 update Russian translation
credit: Alexander45 & malina
2022-04-10 15:50:29 -04:00
catbref
c1894d8c00 Improvements to ByteArray to leverage Java 11 'native' Arrays.hashCode and Arrays.compareUnsigned for speed.
Also modified ambiguous ByteArray::new and ByteArray::of to ByteArray::wrap and ByteArray::copyOf.
Modifications to other classes that use ByteArray.
2022-04-10 16:38:02 +01:00
QuickMythril
f7f9cdc518 Merge pull request #83 from aldum/feature/hungarian_translation
fixup grammar; add missing translations
2022-04-09 00:10:37 -04:00
QuickMythril
850d7f8220 add/update translations
credit: johnnyfg (sv), schizo (it), IsBe (nl), Eduardo9999 (es)
2022-04-08 23:57:54 -04:00
aldum
051043283c fixup grammar; add missing translations 2022-04-06 23:21:49 +02:00
QuickMythril
15bc69de01 Merge pull request #82 from JaymenChou/patch-8
Update SysTray_zh_CN.properties
2022-04-05 13:38:13 -04:00
QuickMythril
ee3cfa4d6d fix typo 2022-04-05 13:26:02 -04:00
QuickMythril
df1f3079a5 Merge pull request #81 from JaymenChou/patch-7
Update SysTray_zh_TW.properties
2022-04-05 13:25:06 -04:00
QuickMythril
d9ae8a5552 Merge branch 'master' into patch-8 2022-04-05 13:23:19 -04:00
QuickMythril
2326c31ee7 Merge branch 'master' into patch-7 2022-04-05 13:11:14 -04:00
QuickMythril
91cb0f30dd Updated TransactionValidity translations
added some missing entries, and sorted alphabetically.
2022-04-05 12:51:49 -04:00
QuickMythril
c0307c352c Updated ApiError translations
removed some duplicate entries, and standardized the order
2022-04-05 11:46:32 -04:00
QuickMythril
8fd7c1b313 formatting fix 2022-04-05 11:09:30 -04:00
QuickMythril
b8147659b1 Updated SysTray translations
added some missing entries, and sorted alphabetically.
2022-04-05 10:48:43 -04:00
JaymenChou
7a1bac682f Update SysTray_zh_TW.properties
Add the missing term "PERFORMING_DB_MAINTENANCE" and translate it to Traditional Chinese
2022-04-04 20:36:48 +08:00
JaymenChou
9fdb7c977f Update SysTray_zh_CN.properties
Translate remaining terms to Simple Chinese
2022-04-04 20:33:59 +08:00
JaymenChou
4f3948323b Update SysTray_zh_TW.properties
Translate the remaining terms to Traditional Chinese
2022-04-04 20:31:19 +08:00
QuickMythril
70fcc1f712 Merge pull request #78 from JaymenChou/patch-4
Create ApiError_zh_CN.properties
2022-04-04 02:49:00 -04:00
JaymenChou
f20fe9199f Update ApiError_zh_CN.properties 2022-04-04 14:36:55 +08:00
QuickMythril
91dee4a3b8 Merge pull request #80 from JaymenChou/patch-6
Create TransactionValidity_zh_CN.properties
2022-04-04 02:17:35 -04:00
QuickMythril
0b89b8084e Merge pull request #79 from JaymenChou/patch-5
Create TransactionValidity_zh_TW.properties
2022-04-04 02:17:24 -04:00
QuickMythril
a5a80302b2 Merge pull request #77 from JaymenChou/patch-3
Create ApiError_zh_TW.properties
2022-04-04 02:17:02 -04:00
QuickMythril
e61a24ee7b removed electrum-ltc.bysh.me
this server often gives a false positive for phishing by some antivirus software.
2022-04-03 22:32:57 -04:00
JaymenChou
55ed342b59 Create TransactionValidity_zh_CN.properties
Add Simple Chinese For better understanding of logs
2022-04-03 13:27:52 +08:00
JaymenChou
3c6f79eec0 Create TransactionValidity_zh_TW.properties
Add Traditional Chinese For TransactionValidity Logs.
2022-04-03 13:25:32 +08:00
JaymenChou
590800ac1d Create ApiError_zh_CN.properties
Add Simple Chinese Support For API Error Message 
Hope it helps in understanding the API !
2022-04-03 12:43:18 +08:00
JaymenChou
95c412b946 Create ApiError_zh_TW.properties
Add Traditional Chinese support to API Responses
2022-04-03 12:40:27 +08:00
CalDescent
a232395750 Merge branch 'master' of github.com:Qortal/qortal 2022-04-01 11:24:56 +01:00
QuickMythril
6edbc8b6a5 add decimal precision to download progress 2022-03-31 13:46:40 -04:00
QuickMythril
f8ffeed302 updated BTC electrumx servers
added new and removed TCP, closed servers, and versions older than 1.16.0
2022-03-31 11:32:55 -04:00
QuickMythril
e2ee68427c removed TCP electrumx servers 2022-03-31 11:29:54 -04:00
QuickMythril
74ff23239d removed TCP electrumx servers 2022-03-31 11:27:56 -04:00
QuickMythril
f1fa2ba2f6 added SSL electrumx servers 2022-03-31 10:02:31 -04:00
QuickMythril
e1522cec94 updated LTC electrumx servers 2022-03-31 09:58:53 -04:00
QuickMythril
8841b3cbb1 add spanish translations 2022-03-31 08:44:33 -04:00
CalDescent
94260bd93f Decreased the number of retries for missing metadata, to reduce broadcast spam. 2022-03-30 08:23:22 +01:00
CalDescent
15ff8af7ac Don't process trade bots or broadcast presence timestamps if our chain is more than 30 minutes old 2022-03-30 08:11:02 +01:00
CalDescent
d420033b36 Revert "Revert "Add Qortal AT FunctionCodes for getting account level / blocks minted + tests""
This reverts commit 59025b8f47.
2022-03-30 08:07:07 +01:00
CalDescent
bda63f0310 Removed hardcoded "qortal-backup/TradeBotStates.json" from POST /admin/repository/data API, as it's no longer needed now that API keys are required. 2022-03-30 08:06:09 +01:00
QuickMythril
54add26ccb fixed typo 2022-03-25 23:39:41 -04:00
CalDescent
089b068362 Updated AdvancedInstaller project for v3.2.3 2022-03-19 22:38:58 +00:00
CalDescent
fe474b4507 Bump version to 3.2.3 2022-03-19 20:44:41 +00:00
CalDescent
bbe15b563c Added unit test to simulate recent issue.
This fails with the 3.2.2 code but now passes when using the latest fixes.
2022-03-19 20:41:38 +00:00
CalDescent
59025b8f47 Revert "Add Qortal AT FunctionCodes for getting account level / blocks minted + tests"
This reverts commit eb9b94b9c6.
2022-03-19 19:52:14 +00:00
CalDescent
1b42c5edb1 Fixed NPE in runIntegrityCheck()
This feature is disabled by default so can be tidied up later. For now, the unhandled scenario is logged and the checking continues on.

One name's transactions are too complex for the current integrity check code to verify (MangoSalsa), but it has been verified manually. All other names pass the automated test.
2022-03-19 19:22:16 +00:00
CalDescent
362335913d Fixed infinite loop in name rebuilding.
If an account is renamed and then at some point renamed back to one of the original names, it confused the names rebuilding code. The current solution is to track the linked names that have already been rebuilt, and then break out of the loop once a name is encountered a second time.
2022-03-19 18:55:19 +00:00
CalDescent
4340dac595 Fixed recently introduced issue in name rebuilding code causing transactions to be unordered.
This is the likely cause of inconsistent name entries across different nodes, as we can't guarantee that every environment will return the same transaction order from the SQL queries.
2022-03-19 18:44:16 +00:00
CalDescent
f3e1fc884c Merge pull request #63 from catbref/master
Add Qortal AT FunctionCodes for getting account level / blocks minted
2022-03-19 11:32:39 +00:00
CalDescent
39c06d8817 Merge pull request #75 from catbref/name-unicode
Unicode / NAME updates.
2022-03-19 11:32:22 +00:00
CalDescent
91cee36c21 Catch and log all exceptions in addStatusToResources()
Some users are seeing 500 errors deriving from this code. This should hopefully allow more info to be obtained, as well as causing it to omit the status for resources that encounter problems.
2022-03-19 11:08:42 +00:00
CalDescent
6bef883942 Removed OpenJDK 11 reference in build-release.sh, as it seems that checksums will not match by default due to timestamps and file orderings.
See: https://dzone.com/articles/reproducible-builds-in-java
2022-03-19 11:05:51 +00:00
CalDescent
25ba2406c0 Updated AdvancedInstaller project for v3.2.2 2022-03-16 19:53:22 +00:00
CalDescent
e4dc8f85a7 Bump version to 3.2.2 2022-03-15 19:57:02 +00:00
CalDescent
12a4a260c8 Handle new sync result case. 2022-03-14 22:04:11 +00:00
CalDescent
268f02b5c3 Added automated test to ensure that the core's default bootstrap hosts are functional.
Whilst not strictly a unit test, this should allow issues with the core's bootstrap servers to be caught quickly.
2022-03-14 21:52:54 +00:00
CalDescent
13eff43b87 Fixed synchronizer issues which caused large re-orgs
Peers without a recent block are removed at the start of the sync process, however, due to the time lag involved in fetching block summaries and comparing the list of peers, some of these could subsequently drop back to a non-recent block and still be chosen as the next peer to sync with. The end result being that nodes could unnecessarily orphan as many as 20 blocks due to syncing with a peer that doesn't have a recent block (but has a couple of high weight blocks after the common block).

This commit adds some additional filtering to avoid this situation.

1) Peers without a recent block are removed as candidates in comparePeers(), allowing for alternate peers to be chosen.
2) After comparePeers() completes, the list is filtered a second time to make sure that all are still recent.
3) Finally, the peer's state is checked one last time in syncToPeerChain(), just before any orphaning takes place.

Whilst just one of the above would probably have been sufficient, the consequences of this bug are so severe that it makes sense to be very thorough.

The only exception to the above is when the node is in "recovery mode", in which case peers without recent blocks are allowed to be included. Items 1 and 3 above do not apply in recovery mode. Item 2 does apply, since the entire comparePeers() functionality is already skipped in a recovery situation due to our chain being out of date.
2022-03-14 21:47:37 +00:00
catbref
e604a19bce Unicode / NAME updates.
Fix UPDATE_NAME not processing empty 'newName' transactions correctly.
Fix some emoji code-points not being processed correctly.
Updated tests.
Now included ICU4J v70.1 - WARNING: this could add around 10MB to JAR size!
Bumped homoglyph to v1.2.1.
2022-03-14 08:45:32 +00:00
CalDescent
e63e39fe9a Updated AdvancedInstaller project for v3.2.1 2022-03-13 19:39:58 +00:00
CalDescent
584c951824 Bump version to 3.2.1 2022-03-13 18:53:54 +00:00
CalDescent
f0d9982ee4 Made arbitraryDataFileHashResponses final, and use .sort() rather than .stream() to avoid new instance creation. 2022-03-12 09:43:56 +00:00
CalDescent
c65de74d13 Revert "Synchronize arbitrary data list removals, as it seems that SynchronizedList and SynchronizedMap aren't overriding removeIf() with a thread-safe version."
This reverts commit e5f88fe2f4.
2022-03-12 09:40:13 +00:00
CalDescent
df0a9701ba Improved logging in onNetworkGetArbitraryDataFileListMessage() 2022-03-11 16:51:19 +00:00
CalDescent
4ec7b1ff1e Removed time estimations that are no longer correct or relevant. 2022-03-11 16:50:34 +00:00
CalDescent
7d3a465386 Including the number of hashes (even if zero) is now required in GetArbitraryDataFileListMessage, to allow for additional fields. Enough peers should have updated by now for this to be ok. 2022-03-11 16:50:11 +00:00
CalDescent
30347900d9 Tidied up one last place that was accessing immutableConnectedPeers directly. This makes no difference, but helps with code consistency. 2022-03-11 15:28:54 +00:00
CalDescent
e5f88fe2f4 Synchronize arbitrary data list removals, as it seems that SynchronizedList and SynchronizedMap aren't overriding removeIf() with a thread-safe version. 2022-03-11 15:22:34 +00:00
CalDescent
0d0ccfd0ac Small refactor for code simplicity. 2022-03-11 15:11:07 +00:00
CalDescent
9013d11d24 Report as 100% synced if the latest block is within the last 30 mins
This should hopefully reduce confusion due to APIs reporting 99% synced even though up to date. The systray should never show this since it already treats blocks in the last 30 mins as synced.
2022-03-11 14:53:10 +00:00
CalDescent
fc5672a161 Use a more tolerant latest block timestamp in the isUpToDate() calls below to reduce misleading systray statuses.
Any block in the last 30 minutes is considered "up to date" for the purposes of displaying statuses.
2022-03-11 14:49:02 +00:00
CalDescent
221c3629e4 Don't refetch the file list if the fileListCache is empty, since an empty list now means that there are likely to be no files available on disk. 2022-03-11 13:08:37 +00:00
CalDescent
76fc56f1c9 Fetch the file list in getFilenameForHeight() if needed. 2022-03-11 13:07:16 +00:00
CalDescent
8e59aa2885 Peer getter methods renamed to include "immutable", for consistency with underlying lists and also to make it clearer to the callers. 2022-03-11 13:00:47 +00:00
CalDescent
0738dbd613 Avoid direct access to this.connectedPeers, as we need to use the immutable copy. 2022-03-11 12:58:11 +00:00
CalDescent
196ecffaf3 Skip calls to this.logger.trace() in ExecuteProduceConsume.run() if trace logging isn't enabled.
This could very slightly reduce load due to skipping the internal filtering inside log4j. Given that this method is causing major problems with CPU at times, I'm trying to make it as optimized as possible.
2022-03-11 11:59:18 +00:00
CalDescent
a0fedbd4b0 Implemented suggestions from catbref to avoid potential thread safety issue in peer arrays. 2022-03-11 11:27:13 +00:00
CalDescent
7c47e22000 Set fileListCache to null when invalidating. 2022-03-11 11:01:29 +00:00
CalDescent
6aad6a1618 fileListCache is now an immutable Map, which is thread safe. Thanks to catbref for this idea. 2022-03-11 10:59:07 +00:00
CalDescent
b764172500 Revert "Hopeful fix for ConcurrentModificationException in BlockArchiveReader.getFilenameForHeight()"
This reverts commit a12ae8ad24.
2022-03-11 10:55:22 +00:00
CalDescent
c185d79672 Loop through all available direct peer connections and try each one in turn.
Also added some extra conditionals to avoid repeated attempts with the same port.
2022-03-09 20:55:27 +00:00
CalDescent
76b8ba91dd Only add an entry to directConnectionInfo if one with this peer-signature combination doesn't already exist. 2022-03-09 20:50:03 +00:00
CalDescent
0418c831e6 Direct connections with peers now prefer those with the highest number of chunks for a resource. Once a connection has been attempted with a peer, remove it from the list so that it isn't attempted again in the same round. 2022-03-09 20:15:26 +00:00
CalDescent
4078f94caa Modified GetArbitraryDataFileListMessage to allow requesting peer's address to be optionally included.
This can ultimately be used to notify the serving peer to expect a direct connection from the requesting peer (to allow it to temporarily bypass maxConnections for long enough for the files to be retrieved). Or it could even possibly be used to trigger a reverse connection (from the serving peer to the requesting peer).
2022-03-09 19:58:02 +00:00
CalDescent
a12ae8ad24 Hopeful fix for ConcurrentModificationException in BlockArchiveReader.getFilenameForHeight() 2022-03-09 19:46:50 +00:00
CalDescent
498ca29aab Wait until a successful connection with a peer before tracking the direct request. 2022-03-08 23:07:08 +00:00
CalDescent
ba70e457b6 Default chunk size reduced from 1MB to 0.5MB 2022-03-08 22:44:43 +00:00
CalDescent
d62808fe1d Don't attempt to create the data directory every time an ArbitraryDataFile instance is instantiated. This was using excessive amounts of CPU and disk I/O. 2022-03-08 22:42:07 +00:00
CalDescent
6c14b79dfb Removed bootstrap host that is no longer functional. 2022-03-08 22:30:01 +00:00
CalDescent
631a253bcc Added support for dark theme in loading screen. 2022-03-08 22:29:37 +00:00
CalDescent
4cb63100d3 Drop the ArbitraryPeers table as it's no longer needed 2022-03-06 13:01:09 +00:00
CalDescent
42fcee0cfd Removed all code that interfaced with the ArbitraryPeers table 2022-03-06 13:00:11 +00:00
CalDescent
829a2e937b Removed all arbitrary signature broadcasts 2022-03-06 12:58:01 +00:00
CalDescent
5d7e5e8e59 Dropped support of ARBITRARY_SIGNATURES message handling, as this feature has been superseded by the peerAddress in file list requests. 2022-03-06 12:46:06 +00:00
CalDescent
6f0a0ef324 Small refactor 2022-03-06 12:42:19 +00:00
CalDescent
f7fe91abeb sendOurOnlineAccountsInfo() moved to its own thread, in preparation for mempow 2022-03-06 12:41:54 +00:00
CalDescent
7252e8d160 Deleted presence tests, as they are no longer relevant, and aren't easily adaptable to the new approach. 2022-03-06 12:03:18 +00:00
CalDescent
2630c35f8c Chunk validation now uses MAX_CHUNK_SIZE rather than CHUNK_SIZE, to allow for a smaller CHUNK_SIZE value to be optionally used, without failing the validation of existing resources. 2022-03-06 11:43:28 +00:00
CalDescent
49f466c073 Added missing break; 2022-03-06 11:21:55 +00:00
CalDescent
c198f785e6 Added significant CPU optimizations to ArbitraryDataManager
- Slow down loops that query the db
- Check for new metadata every 5 minutes instead of constantly
- Check for new data every 1 minute instead of constantly

This could be further improved in the future by having block.process() notify the ArbitraryDataManager that there is new data to process. This would avoid the need for the frequent checks/loops, and only a single complete sweep would be needed on node startup (as long as failures are then retried). But I will avoid this additional complexity for now.
2022-03-06 11:21:39 +00:00
CalDescent
5be093dafc Fix for "Synchronizing null%" systray bug introduced in 3.2.0 2022-03-06 11:00:53 +00:00
CalDescent
2c33d5256c Added code accidentally missed out of commit 1b036b7 2022-03-05 20:44:01 +00:00
CalDescent
4448e2b5df Handle case when metadata isn't returned. 2022-03-05 17:39:13 +00:00
CalDescent
146d234dec Additional defensiveness in ArbitraryDataFile.fromHash() to avoid similar future bugs. 2022-03-05 17:25:48 +00:00
CalDescent
18d5c924e6 Fixed bug cased by fetchAllMetadata() 2022-03-05 17:25:14 +00:00
CalDescent
b520838195 Increased default maxNetworkThreadPoolSize from 20 to 32
This will hopefully offset some of the additional network demands from arbitrary data requests.
2022-03-05 17:24:55 +00:00
CalDescent
1b036b763c Major CPU optimization to block minter
Load sorted list of reward share public keys into memory, so that the indexes can be obtained. This is around 100x faster than querying each index separately (and the savings will increase as more keys are added).

For 4150 reward share keys, it was taking around 5000ms to query individually, vs 50ms using this approach.

The main trade off is that these 4150 keys require around 130kB of additional memory when minting (and this will increase proportionally with more minters). However, this one query was often accounting for 50% of the entire core's CPU usage, so the additional memory usage seems insignificant by comparison.

To gain confidence, I ran both old and new approaches side by side, and confirmed that the indexes matched exactly.
2022-03-05 16:10:43 +00:00
CalDescent
8545a8bf0d Automatically fetch metadata for all resources that have it. 2022-03-05 13:00:49 +00:00
CalDescent
f0136a5018 Include the external port when responding ArbitraryDataFileListRequests 2022-03-05 13:00:17 +00:00
CalDescent
6697b3376b Direct peer connections now use the on-demand data retrieved from file list requests, rather than the stale and incomplete ArbitraryPeerData. 2022-03-05 12:59:13 +00:00
CalDescent
ea785f79b8 Removed unnecessary synchronization 2022-03-04 19:02:30 +00:00
CalDescent
0352a09de7 New online accounts are now verified on the OnlineAccountsManager thread rather than on network threads. This is an attempt to reduce the amount of blocked network threads due to signature verification, and is necessary for the upcoming mempow addition. 2022-03-04 17:58:06 +00:00
CalDescent
5b4f15ab2e Transaction importing code moved to TransactionImporter controller class
As with online accounts, no logic changes other than moving transaction queue processing from the controller thread to its own dedicated thread.
2022-03-04 16:47:21 +00:00
CalDescent
fd37c2b76b Moved all online accounts code to a new OnlineAccountsManager controller class
There are no logic changes here other than moving performOnlineAccountsTasks() onto its own thread, so that it's not subject to anything that might be slowing down the main controller thread.
2022-03-04 16:24:04 +00:00
CalDescent
924aa05681 Optimized peer lists
- Removed synchronization from connectedPeers, and replaced it with an unmodifiableList.
- Added additional immuatable caches: handshakedPeers and outboundHandshakedPeers

This should greatly reduce the amount of time spent waiting around for access to the connectedPeers array, since it is now immediately accessible without needing to obtain a lock. It also removes calls to stream() which were consuming large amounts of CPU to constantly filter the connected peers down to a list of handshaked peers.

Thanks to @catbref for these great suggestions.
2022-03-04 15:14:12 +00:00
CalDescent
84b42210f1 Use ArbitraryDataFileRequestThreads only - instead of reusing file list response threads. 2022-03-04 13:34:16 +00:00
CalDescent
941080c395 Rework of arbitraryDataFileHashResponses to use a list rather than a map (limited to 1000) items. Sort the list by routes with the lowest number of peer hops first, to try and prioritize those which are easiest and quickest to reach. 2022-03-04 13:33:17 +00:00
CalDescent
35d9a10cf4 Avoid logging if there are no remaining transaction signatures to validate. There was too much log spam, none of which was particularly useful. 2022-03-04 12:03:58 +00:00
CalDescent
7c181379b4 Added more granularity to logging, to differentiate between signature validation and general processing/importing, as well as showing counts of the transactions being processed in each round. 2022-03-04 11:12:23 +00:00
CalDescent
f9576d8afb Further optimizations to Controller.processIncomingTransactionsQueue()
- Signature validation is now able to run concurrently with synchronization, to reduce the chances of the queue building up, and to speed up the propagation of new transactions. There's no need to break out of the loop - or avoid looping in the first place - since signatures can be validated without holding the blockchain lock.
- A blockchain lock isn't even attempted if a sync request is pending.
2022-03-04 11:05:58 +00:00
CalDescent
6a8a113fa1 Merge pull request #74 from catbref/presence-txns-removal
PRESENCE transactions changed to always fail signature validation
2022-03-04 10:33:11 +00:00
CalDescent
ef59c34165 Added missing "break" which was causing additional unnecessary debug logging. Originally introduced due to a merge conflict with the metadata branch. 2022-03-04 10:28:44 +00:00
CalDescent
a19e1f06c0 Merge pull request #73 from catbref/incoming-txns-rework
Reworking of Controller.processIncomingTransactionsQueue()
2022-03-04 09:45:29 +00:00
catbref
a9371f0a90 In Controller.processIncomingTransactionsQueue(), don't bother with 2nd-phase of locking blockchain and importing if there are no valid signature transactions to actually import 2022-03-03 20:32:27 +00:00
catbref
a7a94e49e8 PRESENCE transactions changed to always fail signature validation 2022-03-03 20:25:58 +00:00
catbref
affd100298 Reworking of Controller.processIncomingTransactionsQueue()
Main changes are:
* Check transaction signature validity in initial round, without blockchain lock
* Convert List of incoming transactions to Map so we can record whether we have validated transaction signature before to save rechecking effort
* Add invalid signature transactions to invalidUnconfirmedTransactions map with INVALID_TRANSACTION_RECHECK_INTERVAL expiry (~60min)
* Other minor changes related to List->Map change and Java object synchronization
2022-03-03 20:21:04 +00:00
CalDescent
fd6ec301a4 Updated AdvancedInstaller project for v3.2.0 2022-03-03 20:02:30 +00:00
CalDescent
5666e6084b Bump version to 3.2.0 2022-03-02 20:04:49 +00:00
CalDescent
69309c437e Tightened up the content security policy for non HTML files. 2022-03-01 20:36:34 +00:00
CalDescent
e392e4d344 Allow eval(), setTimeout(), etc, to enable various QDN sites to function correctly. The existing sandboxing should be locking this down enough already. Limited to .html and .htm files only. 2022-03-01 20:35:56 +00:00
CalDescent
bd53856927 Disabled auto fetching of metadata. To be re-enabled at a later date. 2022-03-01 20:26:09 +00:00
CalDescent
cbd1018ecf Allow identical data to be published if the metadata differs. 2022-03-01 20:22:47 +00:00
CalDescent
46606152eb /arbitrary/metadata/* endpoint now returns ArbitraryResourceMetadata rather than a raw JSON string. 2022-03-01 20:22:20 +00:00
CalDescent
e6f93e0a08 Added categoryName to ArbitraryResourceMetadata, along with the existing category ID 2022-03-01 20:19:08 +00:00
CalDescent
8d81f1822f Merge branch 'master' into qdn-metadata
# Conflicts:
#	src/main/java/org/qortal/controller/Controller.java
#	src/main/java/org/qortal/network/message/Message.java
2022-02-28 20:10:39 +00:00
CalDescent
5903607363 Merge pull request #72 from catbref/presence-v2
Presence v2
2022-02-27 22:01:59 +00:00
catbref
590a8f52db Remove future work comment from Controller 2022-02-27 16:57:26 +00:00
catbref
ecac47d1bc Also notify TradePresenceWebsocket (using TradePresenceEvent) when bridging old PRESENCE txns 2022-02-27 16:56:17 +00:00
catbref
3b477ef637 Fix JAXB marshalling error (duplicate tradeAddress) in TradePresenceWebSocket. No need to send signature. Make sure publicKey is sent in Base58, not Base64. 2022-02-27 16:56:17 +00:00
catbref
e2ef5b2ef3 Missed change from last commit: incorrect logic in TradePresenceWebSocket 2022-02-27 16:56:17 +00:00
catbref
1d59feeb72 Created /websockets/crosschain/tradepresence to replace /websockets/presence 2022-02-27 16:55:30 +00:00
catbref
c53dd31765 Tidy up of trade presence timestamp generation & checking. Added tests. Renamed "online trades" to "trade presences" 2022-02-27 16:54:42 +00:00
catbref
4c02081992 Tidy up TradeBot presence logging. Decorate API endpoints /crosschain/tradeoffers and /crosschain/trade with presence expiry timestamps 2022-02-27 16:54:42 +00:00
catbref
cb57af3c53 Bugfixes to online trade sigs + bridging from PRESENCE transactions 2022-02-27 16:54:42 +00:00
catbref
01d810fc00 Initial effort at migrating PRESENCE transactions to dedicated network messages 2022-02-27 16:54:42 +00:00
CalDescent
8c2a9279ee Return metadata in various /arbitrary APIs if the "includemetadata" parameter is included.
This is very inefficient and will soon be replaced with dedicated ArbitraryResources / ArbitraryMetadata tables. But this is acceptable in the short term, especially if limit and offset are used.
2022-02-27 09:09:18 +00:00
CalDescent
0d65448f3d Request all metadata automatically. 2022-02-27 08:20:39 +00:00
CalDescent
9da2b3c11a Don't respond to file list requests with just the metadata file.
We have the separate metadata protocol for this now.
2022-02-27 07:28:11 +00:00
CalDescent
95400da977 Fixed typo in various tests (copy and paste error) 2022-02-26 22:10:55 +00:00
CalDescent
dc41dc4c69 Tags now use an array of strings, rather than a single string. 2022-02-26 22:09:07 +00:00
CalDescent
a5c11d4c23 Reduced "Ignoring hash list request" logs from DEBUG to TRACE 2022-02-26 16:10:44 +00:00
CalDescent
878394535e Improvements relating to fetching metadata
- Rate limiter is disabled when using the API
- fetchArbitraryMetadata() returns the actual metadata content rather than a boolean
- Exceptions are thrown on certain errors, rather than returning null
2022-02-26 16:10:26 +00:00
CalDescent
35dba27a55 Fixed issue due to not updating arbitraryMetadataRequests when receiving the metadata file. 2022-02-26 16:07:06 +00:00
CalDescent
f22ad13fa9 Merge branch 'master' into qdn-metadata
This involved a slight rewrite to remove the "includeMetadataOnly" boolean. Metadata is now always excluded, otherwise it complicates the caching too much.

# Conflicts:
#	src/main/java/org/qortal/api/resource/ArbitraryResource.java
#	src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java
2022-02-26 14:39:20 +00:00
CalDescent
aa2e5cb87b Merge branch 'hosted-resources-search' 2022-02-26 14:05:52 +00:00
CalDescent
7740f3da7e Small formatting tweaks, for consistency with existing code. 2022-02-26 14:05:28 +00:00
CalDescent
badb576991 Fixed exception when identifier is null. Also handling null names as this may be a future scenario. 2022-02-26 14:04:35 +00:00
CalDescent
c65a63fc7e Fixed "query" parameter error in swagger documentation 2022-02-26 13:59:53 +00:00
CalDescent
0111747016 Added debug logging of new file list stats. 2022-02-25 13:30:07 +00:00
CalDescent
eac4b0d87b Maintain backwards support for pre-3.2.0 peers by only including new file list message params when sending to newer peers.
These params are optional and the process will function without them, just less efficiently.
2022-02-25 12:24:02 +00:00
CalDescent
3dadce4da4 Renamed a reference 2022-02-25 12:24:02 +00:00
CalDescent
1864468818 Prefer the route with the least number of hops when relaying. 2022-02-25 12:24:02 +00:00
CalDescent
1a59379162 Optionally include requestTime, requestHops, peerAddress, and isRelayPossible flag in ArbitraryDataFileListMessage 2022-02-25 12:24:02 +00:00
CalDescent
31d34c3946 Updated testnet documentation 2022-02-25 11:08:37 +00:00
CalDescent
3cc394f02d Merge pull request #70 from catbref/synchronizer-newchaintipevent
Modify TradeBot to trigger when chain tip changes instead of with every new block
2022-02-24 20:04:26 +00:00
CalDescent
53c4fe9e80 Fixed another ElectrumX issue found in unit tests.
Peers that were thought to be missing output address data may actually have just been using a different key - "address" instead of "addresses". Now reading the addresses from both keys, which may remove the need for the previously added checks.
2022-02-24 20:01:56 +00:00
CalDescent
d5521068b0 Fixed issue in earlier commit, found in unit tests. 2022-02-24 19:45:37 +00:00
CalDescent
a63ef4010d Disabled expired transaction data deletion code for now, as this was often causing data to be incorrectly deleted.
This will need to be re-enabled at some point, but only after it's modified to be much less aggressive.
2022-02-24 19:05:29 +00:00
CalDescent
cec3e86eef Merge pull request #71 from catbref/transferprivs-fix
Very slightly relax validity checks for TRANSFER_PRIVS…
2022-02-24 19:02:20 +00:00
catbref
8950bb7af9 Very slightly relax validity checks for TRANSFER_PRIVS to allow for skeletal account records, e.g. due to CHAT transactions, but account last reference still needs to be null. Example at block height 736196 / 7 2022-02-24 09:13:51 +00:00
catbref
9e6fe7ceb9 Modify TradeBot, some related websockets, to trigger when chain tip changes instead of with every new block 2022-02-24 09:06:21 +00:00
CalDescent
c333d18cd0 Merge branch 'segwit' 2022-02-23 20:07:07 +00:00
CalDescent
0271ef69c9 When submitting a new transaction, treat the chain as "synced" if the latest block is less than 30 minutes old. Increased from around 7.5 minutes. 2022-02-23 20:06:55 +00:00
CalDescent
2d493a4ea2 Added logging when no addresses are returned for a bitcoiny transaction output. 2022-02-23 09:29:16 +00:00
CalDescent
e339ab856f Skip over Electrum servers that don't return any output addresses. Hopeful fix for BTC transactions that report a zero value due to incomplete data being returned from certain ElectrumX peers. 2022-02-23 08:34:38 +00:00
proto
782904a971 improvement to the search on hosted resources
1) use the cached version instead of rescanning all the files
2) separating the loading (which include files scanning) and listing logic
2022-02-22 17:54:08 +01:00
proto
a3753c01bc Add search functionality to hosted resources 2022-02-22 15:50:46 +01:00
CalDescent
d5c3921846 Only show the red "synchronizing" systray icon if the latest block isn't recent.
This should fix issue where the icon unnecessarily jumps between synced and synchronizing.
2022-02-21 22:34:13 +00:00
CalDescent
a2c462b3da Add <meta charset="UTF-8"> tag to websites. Fixed issue rendering emojis and other special characters. 2022-02-21 22:28:59 +00:00
CalDescent
8673c7ef6e Fixed bug in GET /peers/summary API 2022-02-21 22:28:18 +00:00
CalDescent
8d7be7757f Fixed incorrectly named tag. 2022-02-21 22:27:44 +00:00
proto
6b83927048 Persist MintingAccounts.json on minting accounts add/remove
this fix the behavior of the node, After adding or removing a minting account, allowing it to persist it to the backup folder
2022-02-21 16:17:17 +01:00
proto
e07adbd60e online accounts api call, fix level zero accounts
Added online zero level accounts to the response of /addresses/online/levels api endpoints
2022-02-21 15:40:10 +01:00
CalDescent
7798b8dcdc Keep items in arbitraryDataFileHashResponses if they are currently being requested by another thread. This should help to locate the higher numbered chunks from larger resources. 2022-02-20 11:33:09 +00:00
CalDescent
146e7970bf Synchronize this.allKnownPeers and this.connectedPeers in Network.requestDataFromPeer(), to make the method thread-safe.
This could be further improved by taking an immutable copy, but I'll leave this until the same approach is applied to other Network methods.
2022-02-20 11:04:33 +00:00
CalDescent
f4f7cc58e3 Removed unused import. 2022-02-20 10:44:59 +00:00
CalDescent
21b4b494e7 Renamed method. 2022-02-20 10:44:38 +00:00
CalDescent
7307844bee If UPnP is disabled in settings, close the existing external listenPort if a UPnP rule exists. 2022-02-20 10:44:20 +00:00
CalDescent
5d419dd4ec Handle case where funds are sent to and from the same bitcoiny deterministic wallet. 2022-02-19 17:45:24 +00:00
CalDescent
6d0db7cc5e Catch UncheckedIOException in findAllHostedPaths() which was seen when a file was deleted by another thread. 2022-02-19 17:18:51 +00:00
CalDescent
8de606588c Attempt to open the listen port (default 12392) using UPnP, if the local network supports it. 2022-02-18 20:11:00 +00:00
CalDescent
5842b1272d Add WaifUPnP-1.1 jar to project.
For future reference, the command used was:

mvn install:install-file -Dfile=/Users/user/Downloads/waifupnp-1.1/WaifUPnP.jar -DgroupId=com.dosse -DartifactId=WaifUPnP -Dversion=1.1 -Dpackaging=jar -DlocalRepositoryPath=lib
2022-02-18 20:05:14 +00:00
CalDescent
35b0a85818 Increased WALLET_KEY_LOOKAHEAD_INCREMENT_BITCOINJ to 50 2022-02-18 18:42:06 +00:00
CalDescent
fcdd85af6c Try a lookahead size of 20 (instead of 3) when asking Bitcoinj for the balance. 2022-02-18 18:39:47 +00:00
CalDescent
5aac2dc9df ONLINE_ACCOUNTS_V2_PEER_VERSION set to 3.2.0 2022-02-18 18:34:00 +00:00
CalDescent
17a9b4e442 Merge pull request #66 from catbref/network-online-accounts-v2
New network messages ONLINE_ACCOUNTS_V2 and GET_ONLINE_ACCOUNTS_V2.
2022-02-18 18:28:52 +00:00
CalDescent
becb0b37e6 Merge branch 'master' into network-online-accounts-v2 2022-02-18 17:12:44 +00:00
CalDescent
67ca876567 Log if we're unable to process the received file. 2022-02-18 15:35:37 +00:00
CalDescent
464ce66fd5 Moved deletion retry code into ArbitraryDataFile 2022-02-18 15:09:50 +00:00
CalDescent
3e505481fe Default minPeerVersion also increased to 3.1.0 2022-02-18 14:50:46 +00:00
CalDescent
c90c3a183e Block peers below 3.1.0 2022-02-18 14:50:29 +00:00
CalDescent
d1a7e734dc Updated AdvancedInstaller project for v3.1.1 2022-02-15 23:36:38 +00:00
CalDescent
6054982379 Bump version to 3.1.1 2022-02-15 20:02:21 +00:00
CalDescent
85b3278c8a Don't throttle the arbitrary data file request threads when there are items to process. 2022-02-15 19:39:26 +00:00
CalDescent
c90c287601 Increased ARBITRARY_REQUEST_TIMEOUT from 10 to 12 seconds, as some were coming back around 9-10 seconds later. 2022-02-15 19:38:30 +00:00
CalDescent
6ee395ed12 Stop bulk arbitrary signature broadcasts, as they were creating a lot of network traffic, and are in the process of being replaced with a better method. 2022-02-15 19:10:20 +00:00
CalDescent
6275ac2b81 Increased numberOfAdditionalBatchesToSearch from 5 to 7.
This is the equivalent of increasing the max address gap from 15 to 21. The electrum standalone wallet uses 20, so this should be the most we will ever need.
2022-02-14 22:58:37 +00:00
CalDescent
fd0a6ec71f Fix for invalid balance (and transaction amount) when there are no outputs relating to this wallet. 2022-02-14 22:53:30 +00:00
CalDescent
6c1c814aca Updated AdvancedInstaller project for v3.1.0 2022-02-14 20:15:37 +00:00
CalDescent
43791f00aa Wait 2 minutes on node startup before trying to fetch followed QDN data. 2022-02-14 19:33:58 +00:00
CalDescent
538ac30b4e Request only the missing hashes, not all of them. 2022-02-14 19:33:36 +00:00
CalDescent
58f11489db Bump version to 3.1.0 2022-02-13 20:32:19 +00:00
CalDescent
acddf36467 Handle missing includeMetadata parameter. 2022-02-13 19:27:12 +00:00
CalDescent
166d32032a Fixed message IDs. 2022-02-13 19:22:20 +00:00
CalDescent
e4238a62c9 Exclude metadata-only transactions in the data management page (but added an API parameter to allow them to optionally be included).
This ensures that the list will only show resources where there is at least 1 chunk.
2022-02-13 19:21:16 +00:00
CalDescent
ad9c466712 Fall back to UNCATEGORIZED if the parsed category doesn't match any available categories.
This allows for deletion of categories, as the resources will just move into UNCATEGORIZED until they are next updated.
2022-02-13 18:10:56 +00:00
CalDescent
a3d31bbaf1 Category updates based on feedback so far. 2022-02-13 17:56:47 +00:00
CalDescent
4821139501 Merge branch 'master' into qdn-metadata
# Conflicts:
#	src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java
2022-02-13 15:50:12 +00:00
CalDescent
83213800b9 Use the timestamp from the registerNameTransactionData in unit tests, rather than the current time. 2022-02-13 15:05:28 +00:00
CalDescent
265ae19591 Fixed other failing tests due to increased REGISTER_NAME transaction fee. At some point we should determine the correct fee inside of generateBase(), but setting it explicitly adds confidence in testing for now. 2022-02-13 14:31:21 +00:00
CalDescent
c1598d20b5 Name registration fee increase timestamp set to Sunday, 20 February 2022 16:00:00 UTC 2022-02-13 13:47:00 +00:00
CalDescent
0712259057 Implemented REGISTER_NAME transaction fee increase from 0.001 to 5 QORT (average value based on community vote). 2022-02-13 13:45:48 +00:00
CalDescent
ea42a5617f Fixed ElectrumX log spam and errors 2022-02-13 10:58:45 +00:00
CalDescent
58a690e2c3 Route through new getAddressTransactions() wrapper. 2022-02-11 18:15:27 +00:00
CalDescent
3ae2f0086e Removed unusably slow electrum peer 2022-02-11 18:13:45 +00:00
CalDescent
19c83cc54d MAX_AVG_RESPONSE_TIME reduced to 500, as one peer regularly takes around 600ms to reply. 2022-02-11 18:12:34 +00:00
CalDescent
8ac298e07d Allow 3 retries for getTransaction() and getAddressTransactions() requests 2022-02-11 18:11:00 +00:00
CalDescent
9b43e4ea3d Time electrum requests, and move on to another server if one takes more than 1000ms on average to respond (measured over the last 5 requests). 2022-02-11 18:02:56 +00:00
CalDescent
dbacfb964b Increased TX_CACHE_SIZE from 200 to 1000, to speed up loading times on large wallets. 2022-02-11 16:55:29 +00:00
CalDescent
a664a6a790 Added more LTC Electrum peers from https://1209k.com/bitcoin-eye/ele.php?chain=ltc 2022-02-11 16:44:34 +00:00
CalDescent
ee1f072056 Improvement to last commit, so that caller class names are preserved. 2022-02-11 15:34:31 +00:00
CalDescent
a6aabaa7f0 Reduce build queue log spam by only logging high priority items (5 and above). 2022-02-11 15:28:41 +00:00
CalDescent
49b307db60 Treat a null priority as 0 2022-02-11 15:17:02 +00:00
CalDescent
f7341cd9ab Increased /arbitrary priority to 1 2022-02-11 15:13:53 +00:00
CalDescent
6932fb9935 Added "priority" property to build queue items.
/render APIs use priority 10, whereas /arbitrary use priority 0, to prevent thumbnail downloads from holding up website loading. The priorities can be adjusted later, with maybe some service types being given higher priority than others.
2022-02-11 15:08:12 +00:00
CalDescent
2343e739d1 Handle case where a file cannot be unzipped. 2022-02-11 14:35:46 +00:00
CalDescent
fc82f0b622 Use 5 builder threads, so that one slow resource (e.g. a thumbnail) doesn't hold up the other queued build items.
This can be replaced with a task-based approach longer term.
2022-02-11 13:58:45 +00:00
CalDescent
c0c50f2e18 Updated bootstrap hosts 2022-02-11 13:33:25 +00:00
CalDescent
9332d7207e Fixed bug in cache clearing logic, which was often preventing resource updates from being detected. 2022-02-10 09:22:54 +00:00
CalDescent
a8c79b807b Discard any uncommitted changes as a result of the higher weight chain detection 2022-02-10 08:16:30 +00:00
CalDescent
2637311ef5 Prevent potential ConcurrentModificationException in the build queue 2022-02-09 20:20:30 +00:00
CalDescent
06b5b8f793 Reduced time between processing build tasks, to prevent builds with invalid criteria from holding up legitimate builds too much. 2022-02-09 20:17:56 +00:00
CalDescent
61f58173cb Revert "Removed transaction caching. Can be reintroduced later."
This reverts commit 9804eccbf0.
2022-02-09 19:46:20 +00:00
CalDescent
b7b66f6cba Revert "Removed getWalletTransactions() synchronization. Again, can be re-added later."
This reverts commit 70c864bc2f.
2022-02-09 19:46:16 +00:00
CalDescent
dda2316884 Revert "Try a lookahead size of 20 (instead of 3) when asking Bitcoinj for the balance."
This reverts commit d7658ee9f9.
2022-02-09 19:46:10 +00:00
CalDescent
b782679d1f Revert "Revert "Calculate wallet balances from the transactions (ElectrumX) rather than using bitcoinj.""
This reverts commit 214f49e356.
2022-02-09 19:46:06 +00:00
CalDescent
b0f19f8f70 Merge branch 'block-minter-updates'
# Conflicts:
#	src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileRequestThread.java
2022-02-09 19:42:39 +00:00
CalDescent
de5f31ac58 Don't process file hashes if we're stopping 2022-02-09 19:40:20 +00:00
CalDescent
214f49e356 Revert "Calculate wallet balances from the transactions (ElectrumX) rather than using bitcoinj."
This reverts commit 892612c084.

# Conflicts:
#	src/main/java/org/qortal/crosschain/Bitcoiny.java
2022-02-08 18:29:32 +00:00
CalDescent
d7658ee9f9 Try a lookahead size of 20 (instead of 3) when asking Bitcoinj for the balance. 2022-02-08 18:27:44 +00:00
CalDescent
70c864bc2f Removed getWalletTransactions() synchronization. Again, can be re-added later. 2022-02-08 18:27:08 +00:00
CalDescent
9804eccbf0 Removed transaction caching. Can be reintroduced later. 2022-02-08 18:26:15 +00:00
CalDescent
d1f24d45da Added defensiveness in convertToSimpleTransaction() 2022-02-08 18:24:42 +00:00
CalDescent
9630625449 Rework of processIncomingTransactionsQueue() so that it no longer holds the lock while processing.
This should fix an issue where network threads could be blocked when new transactions arrived, due to waiting for the incomingTransactions lock to free up.
2022-02-08 09:18:14 +00:00
CalDescent
b72153f62b Renamed main thread from "Controller" to "Qortal" 2022-02-08 09:02:20 +00:00
CalDescent
0a88a0c95e Perform the base58 decoding outside of the arbitraryDataFileHashResponses lock, to reduce the amount of waiting around by other threads. 2022-02-08 08:45:58 +00:00
CalDescent
ab4ba9bb17 Don't re-fetch unconfirmed transactions that are already in the queue 2022-02-08 08:36:45 +00:00
CalDescent
a49218a840 Optimized ArbitraryDataFileRequestThread - only start a database transaction when there's something to process. 2022-02-07 22:06:45 +00:00
CalDescent
b6d633ab24 Break out of incoming transactions processing loop if we need to sync. 2022-02-07 22:05:13 +00:00
CalDescent
133943cd4e Reduce log spam 2022-02-07 22:03:41 +00:00
CalDescent
f8ffb1a179 Updated thread names 2022-02-07 22:03:26 +00:00
CalDescent
41c4e0c83e Merge branch 'master' into block-minter-updates 2022-02-06 18:40:32 +00:00
CalDescent
99f6bb5ac6 Reorganized some controller methods. 2022-02-06 18:32:43 +00:00
CalDescent
3e0306f646 Increased minPeerConnectionTime and maxPeerConnectionTime to reduce the chances of forced connections during relays.
An alternate option would be to avoid force disconnecting while relays are in progress, but some nodes could have active relays 100% of the time and therefore would never recycle their peers. So it is simpler to just increase the average peer connection time for everyone.
2022-02-06 17:29:00 +00:00
CalDescent
84e4f9a1c1 Rework of arbitraryRelayMap to keep track of multiple responses.
Previously, only one peer's response for a hash would be remembered, even if multiple others reported back too. This would cause useful mapping to be lost.
2022-02-06 17:20:01 +00:00
CalDescent
cd5ce6dd5e Don't remove from the relay map after a file is requested, as it may be needed by other peers.
It will be cleaned up automatically after 60 seconds, so it is best to keep the data intact until then.
2022-02-06 16:04:54 +00:00
CalDescent
9ec4e24ef6 Slightly optimized logic in fetchArbitraryDataFiles() 2022-02-06 15:45:40 +00:00
CalDescent
fa447ccded Builder thread updates. 2022-02-06 15:37:21 +00:00
CalDescent
ef838627c4 Stop asking for hashes from a peer if one fails.
This fixes the request looping that occurs on when a peer is unable to serve files.
2022-02-06 15:37:08 +00:00
CalDescent
b8aaf14cdc Introduced ArbitraryDataFileRequestThread to allow for multiple concurrent file requests.
This is likely a short term solution (to allow existing code to be repurposed) until replaced with a task-based approach, as this will allow for a much greater number of threads.
2022-02-06 15:34:06 +00:00
CalDescent
2740543abf Added "async" and "attempts" parameters to GET /arbitrary/{service}/{name}* endpoints.
async = fail immediately with 404 if missing, and request in the background
attempts = the number of times to request the data (synchronous mode only for now)
2022-02-06 13:04:58 +00:00
CalDescent
3c526db52e Fixed bug in build manager which would prevent future builds until the core was restarted. 2022-02-06 13:03:01 +00:00
CalDescent
cfe0414d96 Small rework of invalidUnconfirmedTransactions to specify the expiry time instead of the time added.
This allows TIMESTAMP_TOO_OLD transactions to be tracked for a shorter time (10 minutes) than the other invalid transactions (60 minutes). Should reduce network traffic and db load around the time that transactions are expiring, as there is a lag before they are noticed and removed from each node. Due to the variance, it could cause other peers to request them again after deleting. They are now ignored for 10 minutes to avoid request spam.
2022-02-06 12:35:41 +00:00
CalDescent
08e06ba11a Fixed bugs preventing invalidUnconfirmedTransactions from working as intended. 2022-02-06 12:09:44 +00:00
CalDescent
8c03164ea5 Don't add expired transactions to invalidUnconfirmedTransactions, as there is no need to keep track of these. 2022-02-06 11:55:07 +00:00
CalDescent
0fe2f226bc Added invalidUnconfirmedTransactions map
An incoming invalid unconfirmed transaction will be added to this map if its timestamp is more than 30 minutes old. This should allow enough time and opportunities for it to be imported and included in a block (allowing for re-orgs which could switch its status from invalid to valid).

Once added, it will be removed after an hour to allow for another chance to be requested from any peers that still have it. If invalid again, it's added back to the map for another hour.

This fixes a 24 hour long loop, where invalid transactions are requested over and over from peers that have already imported them. It could be improved further by periodically removing invalid unconfirmed transactions from the database, but this will be a higher risk.

The results of this feature should be less network traffic, and less blockchain locks (which should ultimately increase the responsiveness of the synchronizer).
2022-02-06 11:23:28 +00:00
CalDescent
55b5702158 Invalidate last low weight block signature whenever the previous block data changes. 2022-02-05 17:54:36 +00:00
CalDescent
a4cbbb3868 Moved block minter sleep to later in the process, otherwise it can remain there for longer than expected. 2022-02-05 17:54:36 +00:00
CalDescent
816b01c1fc Fixed issue in rebase 2022-02-05 17:54:36 +00:00
CalDescent
483e7549f8 Revert "Moved log from INFO to DEBUG, as now the synchronizer is on its own thread it can occur more often than before."
This reverts commit e2e87766fa.
2022-02-05 17:54:35 +00:00
CalDescent
60d71863dc Allow 3 seconds for the synchronizer to obtain a blockchain lock, to reduce missed attempts. 2022-02-05 17:54:35 +00:00
CalDescent
170244e679 More work on higher weight chain detection in the block minter.
Added 30 second timeout, so that any errors should self correct.
2022-02-05 17:54:35 +00:00
CalDescent
472e1da792 Added debug level logging in higherWeightChainExists() for better visibility. 2022-02-05 17:54:35 +00:00
CalDescent
cbf03d58c8 Made synchronizer method public as it is now also used by the block minter. 2022-02-05 17:54:35 +00:00
CalDescent
ba41d84af9 Initial attempt to avoid an unnecessary block submission if one of our peers already has a higher weight chain. 2022-02-05 17:54:35 +00:00
CalDescent
98831a9449 Break out of the various loops in the cleanup manager if the thread is stopping. 2022-02-05 17:53:49 +00:00
CalDescent
9692539a3f Don't include fee in balance calculation (it looks like it could be double counting at the moment). 2022-02-05 17:24:33 +00:00
CalDescent
76df332b57 Check for null IP address before notifying of an external IP update. 2022-02-05 11:51:54 +00:00
CalDescent
c6405340bc minAccountLevelForBlockSubmissions reduced from 6 to 5 2022-02-05 11:33:28 +00:00
CalDescent
775e3c065e Invalidate ElectrumX transactions cache when switching accounts. 2022-02-05 10:23:25 +00:00
CalDescent
8937b3ec86 Don't allow duplicate transaction in the incoming transactions queue.
This should reduce database load slightly, as it won't have to check the same transaction multiple times in each batch.
2022-02-05 10:19:26 +00:00
CalDescent
3fbb86fded Added indexes, to make looking up name transactions by name around 5x faster. 2022-02-04 16:27:31 +00:00
CalDescent
0cf2f7f254 Missing import from last commit 2022-02-04 16:18:00 +00:00
CalDescent
9e571b87e8 Yet another rewrite of fetchAllTransactionsInvolvingName() - this time over 1000x faster since it doesn't involve joining the Transactions table. 2022-02-04 16:17:31 +00:00
CalDescent
23bafb6233 Removed unused methods 2022-02-04 14:00:44 +00:00
CalDescent
6dec65c5d9 Rewrite of fetchAllTransactionsInvolvingName() to avoid having to load all name transactions into memory. 2022-02-04 13:58:05 +00:00
CalDescent
4e59eb8958 Added unit test to simulate the false association between previous a UPDATE_NAME transaction, and the emoji name with a blank reducedName. 2022-02-04 12:51:22 +00:00
CalDescent
756d5e685a Added naming tests for blank new names 2022-02-04 12:50:05 +00:00
CalDescent
f52530b848 More thorough approach to fetchAllTransactionsInvolvingName(), to fix an issue found in unit testing. 2022-02-04 11:58:43 +00:00
CalDescent
c2bf37b878 Added transactionV5Timestamp featureTrigger to unit tests 2022-02-04 11:37:41 +00:00
CalDescent
98a2dd04b8 Fixed bug caused by improper handling of UPDATE_NAME transactions, similar to commit d16663f. 2022-02-04 10:28:26 +00:00
CalDescent
694ea689c8 Synchronize the loop and break out of it before fetching arbitrary data files. Hopeful fix for ConcurrentModificationException, and maybe a potential deadlock. 2022-02-03 21:14:41 +00:00
CalDescent
618aaaf243 Removed logs used for debugging only 2022-02-03 21:06:36 +00:00
CalDescent
9224ffbf73 Cache transaction list for 2 minutes, and synchronize, to prevent the balance and transactions APIs both requesting at once.
This ensures that only a single round of requests (per coin) is used for the wallettransactions and balance APIs. It also speeds up loading on subsequent requests. The 2 minute cache isn't much longer than the foreign block times, so shouldn't cause values to be too out of date.
2022-02-03 21:04:49 +00:00
CalDescent
892612c084 Calculate wallet balances from the transactions (ElectrumX) rather than using bitcoinj.
Hopeful fix for incorrect balances in wallets with large numbers of transactions. At the very least, this gives us control of the code that calculates the balance.
2022-02-03 20:22:25 +00:00
CalDescent
077165b807 Modified fetchArbitraryDataFileList() to support requesting only the missing hashes, but it is not yet used.
Once GetArbitraryDataFileListMessage is rolled out to the network we can uncomment this code. Needs testnet testing prior to that.
2022-02-03 20:01:44 +00:00
CalDescent
7994fc6407 Rework of onNetworkGetArbitraryDataFileListMessage() to support custom hashes to be optionally supplied.
Also simplified the existing logic, to make the code more readable.
2022-02-03 19:58:50 +00:00
CalDescent
d98df3e47d Added support for file hashes to optionally be included in GetArbitraryDataFileListMessage.
Needs testing on testnet, as the majority of nodes need to update before the hashes can be used.
2022-02-03 19:55:22 +00:00
CalDescent
1064b1a08b Removed duplicate metadata file checks. 2022-02-03 19:53:29 +00:00
CalDescent
0b7a7ed0f1 Added some more debug logging relating to file responses. 2022-02-03 19:52:28 +00:00
CalDescent
114b1aac76 Added arbitrary data file manager thread, which will ensure that all file list responses are tried until we receive the files.
Previously we would only try the first response and then discard the others due to being duplicates. They are now added to a queue and retried by the dedicated thread (up to the 60 second timeout).
2022-02-03 19:51:02 +00:00
CalDescent
6d06953a0e Increased RELAY_REQUEST_MAX_HOPS from 3 to 4, in an attempt to reach more peers. 2022-02-02 22:31:40 +00:00
CalDescent
0430fc8a47 Give up immediately after a synchronization if we are shutting down the synchronizer 2022-02-02 09:18:46 +00:00
CalDescent
7338f5f985 Attempt to acquire a blockchain lock (for up to 5 seconds) before shutting down the repository.
This should fix conflicts caused by the synchronizer and controller now being on separate threads. It may also reduce the chances of the database corrupting on shutdown, but this remains to be seen.
2022-02-02 09:17:24 +00:00
CalDescent
640bcdd504 Shutdown/interrupt the synchronizer as early as possible 2022-02-02 09:13:49 +00:00
CalDescent
c9d5d996e5 Increased accuracy of the block weights in the synchronizer logs, as the extra precision is now needed for debugging. 2022-02-01 22:06:01 +00:00
CalDescent
710befec0c Increased ARBITRARY_RELAY_TIMEOUT from 30 to 60 seconds, so that relay peers remember their mappings for longer. 2022-02-01 22:04:53 +00:00
CalDescent
8ccb158241 Reduce log spam when in DEBUG mode. 2022-02-01 22:04:12 +00:00
CalDescent
97199d9b91 Display the local and total chunk counts on the loading screen. 2022-02-01 22:03:52 +00:00
CalDescent
5a8b895475 Fixed bug in loading screen, which prevented the DOWNLOADING status from showing. 2022-02-01 22:03:17 +00:00
CalDescent
6c9600cda0 Added API key header to GET /arbitrary/resource/status/* endpoints 2022-02-01 22:02:41 +00:00
CalDescent
82fa6a4fd8 Include "localChunkCount" and "totalChunkCount" in the GET /arbitrary/resource/status/* API responses.
These values are left out of other API endpoints where multiple resources are returned, because calculating the chunk counts is too time consuming.
2022-02-01 22:02:14 +00:00
CalDescent
45f2d7ab70 Revert "We (currently) can't filter unconfirmed transactions by address, because only the public key is stored in the database until it is confirmed (at which point there is an entry in the TransactionParticipants table which contains the address). Given that this isn't a simple problem to solve, for now it makes sense to reject this combination if requested via the /transactions/search API."
This reverts commit 7aed0354f1.
2022-02-01 21:55:41 +00:00
CalDescent
33731b969a Direct peer connections now send a file list request to the peer, rather than individually requesting every chunk for a transaction. 2022-02-01 09:04:31 +00:00
CalDescent
40a8cdc71f Improved logging when fetching data files 2022-01-31 22:35:12 +00:00
CalDescent
cbe83987d8 Fixed occasional ConcurrentModificationException in the block archive reader. 2022-01-31 21:39:49 +00:00
CalDescent
01e4bf3a77 Try to speed up the shutdown process of the cleanup manager. 2022-01-31 21:39:20 +00:00
CalDescent
b198a8ea07 Fixed issue in ArbitraryDataFile.chunkExists() due to it not checking for the metadata file. 2022-01-31 21:38:51 +00:00
CalDescent
e2e87766fa Moved log from INFO to DEBUG, as now the synchronizer is on its own thread it can occur more often than before. 2022-01-30 20:15:43 +00:00
CalDescent
f005a0975d Shut down the synchronizer as soon as the controller is stopping, if we are able to. 2022-01-30 20:14:11 +00:00
CalDescent
5700369935 Prioritize syncing over transaction importing. 2022-01-30 20:09:35 +00:00
CalDescent
8a1fb6fe4e If a lock can't be obtained when synchronizing, automatically request the sync again. 2022-01-30 20:09:09 +00:00
CalDescent
5b788dad2f Relocated all synchronizer code from the controller to the synchronizer, and also moved synchonization onto its own thread. 2022-01-30 20:01:22 +00:00
CalDescent
fa2bd40d5f Reduce log spam 2022-01-30 19:51:49 +00:00
CalDescent
074bfadb28 Don't bother locking if there are no new transactions to process 2022-01-30 19:50:09 +00:00
CalDescent
bd60c793be Incoming transactions are now added to a queue, and then processed soon after.
This solves a problem where incoming transactions could rarely obtain a blockchain lock (due to multiple transactions arriving at once) and therefore most messages were thrown away. It was also causing constant blockchain locks to be acquired, which would often prevent the synchronizer from running.
2022-01-30 19:03:31 +00:00
CalDescent
90f3d2568a Log whenever the synchronizer can't obtain a blockchain lock, so that blockchain lock issues are more easily noticed. 2022-01-30 19:00:28 +00:00
CalDescent
c73cdefe6f transactionV5Timestamp moved to blockchain.json 2022-01-30 13:47:20 +00:00
CalDescent
c5093168b1 minAccountLevelToMint value moved to blockchain.json 2022-01-29 22:59:04 +00:00
CalDescent
a35e309a2f minAccountLevelForBlockSubmissions moved to blockchain.json 2022-01-29 22:57:22 +00:00
CalDescent
d4ff7bbe4d Fixed inaccurate README text that was accidentally merged from the data node repository. 2022-01-29 22:37:27 +00:00
CalDescent
d4d73fc5fc Merge pull request #67 from otaku/docker
feat: add Dockerfile
2022-01-29 20:15:06 +00:00
CalDescent
bb35030112 Merge pull request #65 from QuickMythril/reduce-doge-fee
reduce DOGE fees
2022-01-29 20:14:04 +00:00
CalDescent
7aed0354f1 We (currently) can't filter unconfirmed transactions by address, because only the public key is stored in the database until it is confirmed (at which point there is an entry in the TransactionParticipants table which contains the address). Given that this isn't a simple problem to solve, for now it makes sense to reject this combination if requested via the /transactions/search API. 2022-01-29 19:24:28 +00:00
CalDescent
c4f763960c Don't delete a resource's cache if a build is in progress.
Hopeful fix for "Unable to delete cache for resource: Unable to delete directory" error, and possibly some other file conflicts.
2022-01-29 19:18:06 +00:00
CalDescent
c5182a4589 Increased MAX_ACCOUNT_COUNT in GetOnlineAccountsMessage from 1000 to 5000 2022-01-29 12:39:43 +00:00
CalDescent
fc1a376fbd Added POST /transaction/fee API endpoint, to return the recommended fee for the supplied transaction data. 2022-01-29 10:38:15 +00:00
CalDescent
27387a134f Fixed typo 2022-01-28 19:58:32 +00:00
CalDescent
be7bb2df9e Added GET /transaction/unitfee API endpoint, to obtain the unit fee for a transaction type
Additional params:
- timestamp: to allow for hard forks. Default: the current time
- level: the account level, to allow for the future possibility of different fees per level. Not currently used.
2022-01-28 19:00:15 +00:00
CalDescent
72a291a54a Added initial support of different unit fees per transaction type.
Included a timestamp property, as this will be needed for each update (hard fork).
2022-01-28 18:50:03 +00:00
CalDescent
b1342d84fb Updated potentially misleading log message. 2022-01-28 11:02:37 +00:00
CalDescent
cdd57190ce Use getEffectiveMintingLevel() rather than getLevel() 2022-01-28 10:16:42 +00:00
CalDescent
d200a098cd Updated AdvancedInstaller project for v3.0.4 2022-01-27 23:18:34 +00:00
CalDescent
a0ed3f53a4 Merge branch 'master' of github.com:Qortal/qortal 2022-01-27 22:45:40 +00:00
CalDescent
e5c12b18af Bump version to 3.0.4 2022-01-27 19:58:39 +00:00
CalDescent
7808a1553e Fixed case sensitive ordering issues in other aspects of QDN. 2022-01-27 18:46:59 +00:00
CalDescent
a0ba016171 Fixed case sensitive ordering issue in websites list. 2022-01-27 18:42:52 +00:00
CalDescent
344704b6bf MIN_LEVEL_FOR_BLOCK_SUBMISSION temporarily increased to 6.
This is to hopefully improve network stability whilst a more advanced solution is being worked on. It also allows us to collect some data on how well the network behaves when there are less block candidates. It should have no effect on minting rewards (other than any side effects as a result of improved network stability).
2022-01-27 18:15:00 +00:00
CalDescent
3303e41a39 Fixed unhandled case in GET /arbitrary/{service}/{name}* endpoints 2022-01-27 18:12:21 +00:00
CalDescent
4e71ae0e59 Allow QDN data to be served without authentication by setting "qdnAuthBypassEnabled":true
This allows the GET /arbitrary/{service}/{name} and GET /{service}/{name}/{identifier} endpoints to operate without any authentication. Useful for those who are running public QDN nodes and need to serve data over http(s).
2022-01-27 18:10:24 +00:00
CalDescent
9daf7a6668 Synchronize lists, to prevent an occasional ConcurrentModificationException 2022-01-26 22:40:34 +00:00
Alexander Do
a2b2b63932 feat: add Dockerfile 2022-01-23 16:48:34 -08:00
CalDescent
af06774ba6 Clear the cache when deleting data, so that it disappears from the data management screen. 2022-01-23 22:53:35 +00:00
catbref
244d4f78e2 New network messages ONLINE_ACCOUNTS_V2 and GET_ONLINE_ACCOUNTS_V2.
Increased GetOnlineAccountsMessage.MAX_ACCOUNT_COUNT from 1000 to 5000.

The V2 versions are more efficiently encoded and also cache the payload bytes
which reduces CPU when sending to multiple peers.

Serialization / deserialization unit tests included.

Tentative V2 message activation set at core version 3.1.2
see Controller.ONLINE_ACCOUNTS_V2_PEER_VERSION
2022-01-23 16:24:10 +00:00
CalDescent
311fe98f44 Bump version to 3.0.3 2022-01-23 13:19:49 +00:00
CalDescent
6f7c8d96b9 Removed ApiService instance creation in ApplyUpdate as it wasn't really needed, and probably not sensible to instantiate it here. 2022-01-23 12:57:28 +00:00
CalDescent
ff6ec83b1c Removed localAuthBypassEnabled override in unit tests.
Hopefully this will allow us to proactively catch any missing API keys in the future.
2022-01-23 12:48:37 +00:00
CalDescent
ea10eec926 Fixed bug in auto update process - use the API key when stopping the node.
Luckily this code is included in the new JAR, not the old one, so we should be able to regain auto update ability by issuing a new update.
2022-01-23 12:38:19 +00:00
CalDescent
be561a1609 Add default "dataPath" to Windows installer builds, so that QDN data is located in AppData 2022-01-22 22:27:14 +00:00
CalDescent
6f724f648d Fixed testDirectoryDigest() which has been failing for a couple of versions (due to gitignore removing the cache file) 2022-01-22 20:48:58 +00:00
CalDescent
048776e090 Ignore failing test due to recent API update, which makes the test incompatible. To be fixed later. 2022-01-22 20:43:28 +00:00
CalDescent
dedf65bd4b Added initial protocol methods for metadata requests and forwarding. Not tested yet. 2022-01-22 20:24:37 +00:00
CalDescent
a7c02733ec Updated approve-auto-update.sh to use new service format 2022-01-22 19:40:13 +00:00
CalDescent
59346db427 Bump version to 3.0.2 2022-01-22 18:45:32 +00:00
CalDescent
25efee55b8 Added networking optimization, to avoid wasted processing on every read.
Thanks to @catbref for finding this.
2022-01-22 18:43:32 +00:00
CalDescent
a79ed02ccf Added initial (unfinished) category list, as well as the GET /arbitrary/categories API, and converted the category field from a string to an enum 2022-01-22 12:11:16 +00:00
CalDescent
79f87babdf Limit the metadata string lengths 2022-01-21 22:52:31 +00:00
CalDescent
f296d5138b Allow metadata to optionally be included with any arbitrary resource. 2022-01-21 21:14:28 +00:00
CalDescent
b30445c5f8 Increase MAX_ACCOUNT_COUNT to 5000 (around 0.5MB of data) to see if it helps with minting efficiency. 2022-01-21 16:11:21 +00:00
CalDescent
d105613e51 Improvements to some /blocks API endpoints 2022-01-21 16:07:18 +00:00
CalDescent
ef43e78d54 Reduced log spam 2022-01-21 15:20:16 +00:00
CalDescent
6f61fbb127 Reduced strictness of rate limiter, to allow two additional file list requests (15 and 30 seconds after initial attempt)
This should help catch any remaining chunks that were unable to be fetched in the first attempt due to network failures, bad peers, etc.
2022-01-21 15:20:04 +00:00
CalDescent
9f9b7cab99 Include "size" in /arbitrary/resource APIs
Note that this is not always accurate - it relates to the largest transaction size for this name, not necessarily the latest or the combined size of multiple transactions. This can be made accurate as soon as we have a "Resources" table to store this info. Trying to do it before then will be too inefficient in terms of queries.
2022-01-21 14:42:37 +00:00
CalDescent
f129e16878 Added retry mechanism to relay file deletions, just in case it fails on the first try.
A longer term solution to this hypothetical problem is to store relays in RAM or a temp folder only. Or maybe add an indicator file to instruct the cleanup manager to delete it. But this will require more development. 10 deletion attempts (each 1 second apart) should be enough for now.
2022-01-21 14:32:15 +00:00
CalDescent
8a42dce763 Use the first responding peer in the relay map.
This encourages shorter relays, since longer ones will take more time to respond, and also prevents a peer from intentionally taking a long time to respond so that it overwrites an existing entry.

Longer term we could consider keeping track of all respondents for each hash, if there are still issues with data retrieval. I suspect this won't be needed though, as the requesting peer has 16+ different peers connected, and therefore potentially 16 different mappings already.
2022-01-21 14:23:55 +00:00
CalDescent
6423d5e474 Optimized onNetworkGetArbitraryDataFileListMessage() to remove duplicate calls to Files.exists() 2022-01-21 14:12:59 +00:00
CalDescent
6e91157dcf Improved cache clearing process and logging. 2022-01-21 13:32:59 +00:00
CalDescent
85c61c1bc1 Added GET /arbitrary/resources/search API
Example usage

List all websites with a name containing the word "crow" (using default identifier):
http://localhost:12391/arbitrary/resources/search?service=WEBSITE&query=crow&default=true&limit=20

List all resources with name or identifier containing the word "crow":
http://localhost:12391/arbitrary/resources/search?query=crow&default=false&limit=20
2022-01-21 10:04:18 +00:00
CalDescent
54af36fb85 Remove duplicates in GET /arbitrary/hosted/resources response 2022-01-20 22:34:41 +00:00
CalDescent
fcdcc939e6 Sort hosted data in reverse order (newest first) 2022-01-20 22:29:19 +00:00
CalDescent
13450d5afa Added limit/offset to GET /arbitrary/hosted API endpoints 2022-01-20 22:28:28 +00:00
CalDescent
5e1e653095 Removed unnecessary database lookups in GET /hosted/resources API 2022-01-20 20:39:46 +00:00
CalDescent
e8fabcb449 Removed extra isDataLocal() check in GET /hosted/resources which was evading the cache. 2022-01-20 20:38:57 +00:00
CalDescent
a4ce41ed39 Updated qort and qdata to check for apikey.txt in $HOME/qortal, which is the most commonly installed location.
This allows the script to be run from any directory, as long as the core is installed at $HOME/qortal.
2022-01-20 20:31:16 +00:00
CalDescent
1b42062d57 Default minPeerVersion set to 3.0.1 2022-01-20 20:25:20 +00:00
CalDescent
c2a4b01a9c Allow local playback of media files 2022-01-20 20:25:06 +00:00
CalDescent
47e763b0cf Take a copy of the IP address history so it can be safely iterated. Without this, another thread could remove an element, resulting in an exception. 2022-01-20 20:24:51 +00:00
CalDescent
0278f6c9f2 Reduced more log spam 2022-01-20 20:24:26 +00:00
CalDescent
d96bc14516 Allow execution of inline scripts, at least for now. 2022-01-17 20:25:25 +00:00
CalDescent
318f433f22 Reduced log spam when checking for avatars. 2022-01-17 20:04:54 +00:00
CalDescent
cfc80cb9b0 Use a header instead of a meta tag for Content-Security-Policy, because we can't guarantee that we are parsing all HTML files.
Also use default-src instead of connect-src, as we want to block all external requests.
2022-01-17 20:04:35 +00:00
CalDescent
01c6149422 Restrict websites to same origin requests only, using a Content-Security-Policy meta tag. 2022-01-16 20:52:30 +00:00
CalDescent
6f80a6c08a Rework of file list requests and relays, allowing it to handle multiple chunk resources in a much more sensible way.
This could create a lot of additional relay traffic as a result, so needs lots of testing and possibly optimizing.
2022-01-16 20:39:37 +00:00
CalDescent
8fb2d38cd1 Revert "Revert log4j version for now. We need to put this back in the next update, once log4j2.properties files have transitioned to the new format."
This reverts commit 777bddd3d8.
2022-01-15 20:35:44 +00:00
CalDescent
5018d27c25 "Not started" renamed to "Published" 2022-01-15 20:21:52 +00:00
CalDescent
1d77101253 Use AES/CBC/PKCS5Padding for encryption, and fall back to just AES for legacy resource support.
Should fix "ECB mode cannot use IV" error due to mode and padding not being stated.
2022-01-15 20:14:32 +00:00
CalDescent
1ddd468c1f Added API key support to qdata script
As with the qort script, it currently needs to be run from either the qortal directory or the tools directory in order to pick up the API key
2022-01-14 11:46:26 +00:00
CalDescent
f05cd9ea51 Added API key support to qort script
Currently needs to be run from either the qortal directory or the tools directory in order to pick up the API key
2022-01-14 11:45:40 +00:00
CalDescent
70c00a4150 Updated AdvancedInstaller project for v3.0.1 2022-01-13 22:36:31 +00:00
CalDescent
d296029e8e Bump version to 3.0.1 2022-01-13 20:18:32 +00:00
CalDescent
e257fd8628 Updated stop.sh script to use the /admin/stop API endpoint if an apikey.txt file is available.
This brings the behaviour closer to the old version so should hopefully reduce the amount of newly introduced issues. If an API key is unavailable, it will fall back to using `kill -15 $pid` (i.e. a SIGTERM).
2022-01-13 19:18:39 +00:00
CalDescent
119c1b43be Use default values for method and compression if not specified.
Should fix issue with v4 transactions where these aren't used. Matches with the NOT NULL DEFAULT 0 which automatically transitions existing v4 ARBITRARY transactions to use the same defaults.
2022-01-13 19:09:00 +00:00
CalDescent
1277ce38de Bump version to 3.0.0 2022-01-12 21:11:02 +00:00
CalDescent
6761b91400 QDN go-live timestamp set to Fri, 14 Jan 2022 16:00:00 UTC 2022-01-12 20:53:57 +00:00
CalDescent
2a6244a5c2 Handle multiple qortal processes in stop.sh 2022-01-12 20:31:21 +00:00
CalDescent
777bddd3d8 Revert log4j version for now. We need to put this back in the next update, once log4j2.properties files have transitioned to the new format. 2022-01-12 20:28:23 +00:00
CalDescent
e2b13791bb Don't reload the log4j2.properties file as this seems to be buggy in a lot of cases. 2022-01-12 20:26:41 +00:00
CalDescent
f44c21ce59 Disallow any kind of website/app/data rendering when localAuthBypassEnabled is enabled.
This allows node operators to return their authentication to the legacy rules (local requests allowed), without introducing javascript vulnerabilities. The websites, apps, etc are just prevented from loading, to avoid the risk of any API calls from javascript.
2022-01-12 19:32:52 +00:00
CalDescent
ade977e416 Don't broadcast any arbitrary signatures if the list is empty (i.e. the node isn't yet hosting anything) 2022-01-12 19:23:36 +00:00
CalDescent
f09a131bd6 Added requestHops to log entry. 2022-01-12 19:21:11 +00:00
CalDescent
4815587de1 Use V2 of string serialization methods in ArbitrarySignaturesMessage, as it is designed to allow null values. 2022-01-12 19:20:49 +00:00
CalDescent
e0ebfb9b53 Reduced log spam. 2022-01-12 19:20:14 +00:00
CalDescent
90836afd91 External IP address updates now require 10 consecutive readings. 2022-01-12 19:19:56 +00:00
CalDescent
4e1b0a25bb Fixed arbitrary peer tests, which used a local address (but we now treat them as invalid). 2022-01-11 20:06:14 +00:00
CalDescent
89c3236bf5 Updated bundled log4j2.properties files 2022-01-11 20:02:42 +00:00
CalDescent
7658bc2025 Added X-API-KEY header field to API documentation endpoints that require it. 2022-01-11 19:13:56 +00:00
CalDescent
7cf60c7c35 Updated stop.sh so that it no longer uses the core API. 2022-01-11 19:12:56 +00:00
CalDescent
ccde725d3b Check for an empty string as well as null in a couple of places, so that deserializeSizedString() can be safely used. 2022-01-10 19:42:21 +00:00
CalDescent
e3b45cac0a Use an alternative version of Serialization.serializeSizedString() and Serialization.deserializeSizedString() for the new ARBITRARY transaction additions.
The modifications made to these methods were causing issues with other transaction types that were expecting blank strings instead of null. To keep risk to a minimum, I have split into two different sets of functions until there is more time to unify them.
2022-01-10 19:41:02 +00:00
CalDescent
8f8a500dcd Fixed some issues left over from the qortaldata project 2022-01-10 19:26:25 +00:00
CalDescent
f9749cd82c Merge remote-tracking branch 'qortal-data/master' into qdn 2022-01-09 21:10:48 +00:00
CalDescent
051052fdd2 Removed authentication for GET /peers/summary endpoint 2022-01-09 21:09:20 +00:00
CalDescent
940304b4c2 Added GET /admin/apikey/test endpoint, so that we have a dedicated place to test if authentication works. 2022-01-09 20:08:45 +00:00
CalDescent
b4d2fae27f Fixed a couple of FOLLOWED_AND_VIEWED references that were missed 2022-01-09 16:26:52 +00:00
CalDescent
11e194292c Removed API key requirement from GET /admin/status and GET /admin/mintingaccounts 2022-01-09 16:26:23 +00:00
CalDescent
5ba6f6f53e FOLLOWED_AND_VIEWED renamed to FOLLOWED_OR_VIEWED, since it's technically an OR not an AND. 2022-01-09 13:25:49 +00:00
CalDescent
f58a16905f Removed unused setting. 2022-01-09 13:19:07 +00:00
CalDescent
33e82b336b Limit arbitrary signature requests to 3 hops, just in case a bug caused any kind of circular broadcasting. 2022-01-09 11:22:27 +00:00
CalDescent
0ced712974 Merge remote-tracking branch 'qortal-data/master' into qdn 2022-01-08 12:29:48 +00:00
CalDescent
db8e35cc13 Allow a new API key to be generated if the existing apikey.txt file has been deleted 2022-01-08 12:27:24 +00:00
CalDescent
b6db5aa2d3 Use "apikey.txt" instead of "apikey" as the filename to store the api key, to make it easier for users to open. 2022-01-08 10:22:14 +00:00
CalDescent
396dc5c9b0 Always log "Synchronizing with peer..." as it may help give more clarity to those with sync issues. 2022-01-08 10:12:54 +00:00
CalDescent
67e424a32a Added GET /arbitrary/relaymode API endpoint, which returns whether relay mode is enabled in the settings or not. 2022-01-07 14:38:05 +00:00
CalDescent
d8cbec41d2 Various logging improvements and fixes. 2022-01-07 14:08:11 +00:00
CalDescent
374f6b8d52 Added restrictions when relaying file list requests
1) Each relay request expires after 5 seconds, after which nodes will stop relaying it, preventing any kind of infinite loop. So it has to reach the destination peer within 5 seconds. This should be fine, because the original peer's request would timeout anyway, so there's nothing to be gained by continuing to relay it.

2) Each relay request stops being forwarded after 3 "hops" - i.e. once it has been relayed through 3 different peers, it will no longer be transmitted any further. If we assume that each node has 16 connections, that allows it to reach a theoretical maximum of 4096 peers in 3 hops. In practice it will be less, and may not reach everyone due to peer "islands". But it will automatically retry a few times on a timer, so should hopefully find what it needs eventually. Plus, it still has the ability to make a direct connection to anyone hosting the data, as long as they are port forwarded.
2022-01-07 14:01:57 +00:00
CalDescent
20ec4cbd14 ARBITRARY_REQUEST_TIMEOUT increased from 6 to 10 seconds
This is likely longer than needed, but it's best to allow extra for now and then optimize the timeouts once we've had some experience with real world data.
2022-01-07 12:50:34 +00:00
CalDescent
1c80835f49 Default relayModeEnabled to true.
Even though a final decision is yet to be made, it makes sense to test with this scenario to ensure that everything works correctly.
2022-01-07 12:31:16 +00:00
CalDescent
5e0af26c27 Keep track of successes or failures for a particular arbitrary peer / signature combination.
This can help to inform decisions on data retention (although there is no deletion yet).
2022-01-05 21:23:29 +00:00
CalDescent
b42674ac06 Small code reorganization to improve logic when adding arbitrary peer data 2022-01-05 19:26:06 +00:00
CalDescent
3394543705 Don't save arbitrary peer data if it's a local address 2022-01-05 19:22:24 +00:00
CalDescent
75c51aa61b If a direct connection can't be made to a peer over the original or default port, match the peer's host with any entries in knownPeers and try connecting to each of those until one succeeds. 2022-01-05 19:03:41 +00:00
CalDescent
6041722250 Added missing import. 2022-01-05 18:35:03 +00:00
CalDescent
60d038b367 Return result of Network.connectPeer() back to caller. 2022-01-05 18:34:54 +00:00
CalDescent
b2c4bf96af Rebroadcast the entire list of hosted signatures each time our external IP address changes. 2022-01-03 19:52:42 +00:00
CalDescent
f007f9a86d Added optional "senderPeerAddress" string to HELLO messages, to allow external IP changes to be detected without using a centralized service. 2022-01-03 19:05:10 +00:00
CalDescent
b1c1634950 Updated log4j to 2.17.1
This involves modifying the log4j2.properties file on node startup to fix an incompatibility with ${dirname:-}. Thanks to AlphaX Projects for tracking down this incompatibility.
2022-01-02 20:50:38 +00:00
CalDescent
5157ccf7c0 Expire each authorization after 60 minutes. 2022-01-02 20:11:38 +00:00
CalDescent
c4a782301d Delete reader cache directories if a resource is unable to be built. 2022-01-02 18:31:17 +00:00
CalDescent
17fe94fa46 Added POST /render/authorize/{resourceId} endpoint
This allows an entire registered name to be preauthorized, therefore allowing for instance a website to automatically request other resources from the same author, such as videos.
2022-01-02 18:23:27 +00:00
CalDescent
75d9347d23 Show blank root page for the gateway.
This can ultimately be replaced with a website list / search engine.
2022-01-02 17:07:48 +00:00
CalDescent
ef784124f3 Fixed status bugs in loading screen. 2022-01-02 16:11:25 +00:00
CalDescent
bd1b631914 Use <base href="..."> in HTMLParser, rather than attempting to swap out every relative link
This delegates the task to the browser rather than doing it in java. It should also catch a few remaining types of links that we had missed - e.g. ones that originate from within js files.
2022-01-02 15:22:53 +00:00
CalDescent
edfc8cfdc4 Fixed bug which saved an incorrect peer address for a forwarded arbitrary signatures message. 2022-01-01 22:34:02 +00:00
CalDescent
fbe34015d4 Validate peer addresses before saving anything to the db. 2022-01-01 22:33:01 +00:00
CalDescent
391fa008d0 When making a direct connection to a peer, try using the default listen port along with the one specified in the ArbitraryPeers table.
This is a workaround to account for any ephemeral ports that may have made it into the dataset.
2022-01-01 21:37:23 +00:00
CalDescent
7df8381b8f Don't allow a row to be added to ArbitraryPeers (or the message to be rebroadcast) if an entry already exists with the same hash and host/ip.
This avoids duplicate entries from the same host/ip with differing ports. This can occur due to some requests using ephemeral port numbers. Ideally we would filter these out altogether, but this at least acts as a safety net to prevent a very cluttered db and associated "broadcast storm". The main tradeoff here is that multiple nodes on the same IP address will be recorded as a single entry. This doesn't seem like it will be a major limitation, because one of them will remain available.
2022-01-01 21:09:48 +00:00
CalDescent
c0234ae328 Added unit test to make sure the mempow nonce is being validated in ARBITRARY transactions. 2022-01-01 18:12:18 +00:00
CalDescent
5c64a85d7c Test a registered name with a space in it 2022-01-01 15:43:00 +00:00
CalDescent
7aa8f115ce Added "original copy indicator file", which prevents the node from deleting its own published content when storage space runs out.
Since some files won't have any mirrors, this prevents the cleanup manager from deleting the only copy in existence when freeing up space. This feature can be disabled by setting "originalCopyIndicatorFileEnabled": false in settings.json (or by deleting the ".original" files). The trade off is that the only copy in existence could be deleted if space gets low.

This will also allow for better reporting of own vs third party files in the local UI (not yet implemented).
2022-01-01 14:52:09 +00:00
CalDescent
cf2c8d6c67 Switched some IOExceptions to DataExceptions 2022-01-01 14:02:54 +00:00
CalDescent
37edebcad9 Fixed cleanup bug which could cause problems for ArbitraryDataWriter 2021-12-30 15:50:14 +00:00
CalDescent
4d4f661548 Renamed ArbitraryResourceSummary to ArbitraryResourceStatus, and added status titles & descriptions
This allows for consistent messaging about each status to be shown in different parts of the system. Previously these strings were hardcoded in the loading screen html so were inaccessible elsewhere.
2021-12-29 19:41:49 +00:00
CalDescent
46e4cb4f50 Removed custom version number 2021-12-29 12:47:49 +00:00
CalDescent
34e622cf0c Removed unused line. 2021-12-28 17:00:54 +00:00
CalDescent
7ccb99aa2c Improved clarity of file deletion message. 2021-12-28 17:00:41 +00:00
CalDescent
9e3847e56f Moved some of the less important arbitrary transaction related logs from INFO to DEBUG/TRACE.
Logs can be reinstated by adding these lines to log4j2.properties:

logger.arbitrary.name = org.qortal.arbitrary
logger.arbitrary.level = debug
logger.arbitrarycontroller.name = org.qortal.controller.arbitrary
logger.arbitrarycontroller.level = debug
2021-12-28 17:00:09 +00:00
CalDescent
90ced351f4 ARBITRARY transaction difficulty increased to 14, bringing it in line with MESSAGE transactions.
14 is currently the lowest accepted difficulty for on-chain transactions, so it's best that we don't go lower than that.
2021-12-28 14:25:17 +00:00
CalDescent
04295ea8c5 Set arbitrary transaction difficulty to 1 during unit tests, as they were taking too long. 2021-12-28 13:25:03 +00:00
CalDescent
2452d3c24b Moved proof of work difficulty definition from ArbitraryTransaction to ArbitraryDataManager 2021-12-28 13:01:26 +00:00
CalDescent
302428f1d1 Exclude RAW_DATA arbitrary transactions from various arbitrary data controller methods. 2021-12-28 12:01:03 +00:00
QuickMythril
cf603aa80e disable Open UI menu item
this doesn't work and should either be fixed or removed completely.
2021-12-25 14:23:49 -05:00
CalDescent
1f9f949a8c Added GET /peers/summary API which returns counts of the number of inbound and outbound connections that currently exist. 2021-12-24 15:25:58 +00:00
CalDescent
0bde1e97dc Added DELETE /resource/{service}/{name}/{identifier} API endpoint, to delete local data 2021-12-24 13:04:16 +00:00
CalDescent
42aca2e40f arbitraryDataCachedResources is now keyed by all ArbitraryDataResource elements, not just the resourceId 2021-12-24 12:12:16 +00:00
CalDescent
e1e44d35bb ArbitraryDataBuildQueueItem now extends ArbitraryDataResource
This is the start of a refactor to use ArbitraryDataResource objects rather than passing around separate resourceId, service and identifier, or other duplicate objects. Most of this will need to be done after the initial release due to time constraints.
2021-12-24 12:05:03 +00:00
QuickMythril
a790b2e529 reduce DOGE fees
https://github.com/dogecoin/dogecoin/blob/master/doc/fee-recommendation.md
2021-12-23 19:10:17 -05:00
CalDescent
357946388c New manager classes are no longer subclasses of Thread, as this part wasn't being used for anything. 2021-12-23 22:23:07 +00:00
CalDescent
b774583f28 Disabled all arbitrary data manager threads if QDN is disabled in the settings. 2021-12-23 22:19:14 +00:00
CalDescent
6436daca08 Split up ArbitraryDataManager into three separate manager classes, to make the code more readable and to keep each class within 500 lines.
This is still the least readable area of the data hosting code, but it should be a bit more maintainable now.
2021-12-23 22:15:28 +00:00
CalDescent
f153c7bb80 Added "qdnEnabled" setting (default: true) to allow users to opt out of QDN functionality. 2021-12-23 21:38:30 +00:00
CalDescent
1f8a618dcc Removed recently added logs. 2021-12-23 17:30:34 +00:00
CalDescent
2d853e5a2f Added support of patch creation from files without a newline at the end
The simplest solution was to only include a newline at the end of the patch file if the source file ended with a newline. This is used to inform the merge code as to whether to add the newline to the end of the resulting file. Without this, the checksums do not match (and therefore previously the complete file would have been included as a result).
2021-12-23 16:59:49 +00:00
CalDescent
361dc79ede Fixed bug caused by multiple concurrent builds of the same resource.
Several parts of the code request resources to be loaded/built, and these separate threads were tripping over each other and causing build failures. This has been avoided by making sure the resource isn't already building before requesting it.
2021-12-22 19:31:44 +00:00
CalDescent
19173321ea Small refactor to reduce code duplication. 2021-12-22 17:03:54 +00:00
CalDescent
87b724ec72 Updated various places that used File.separator when they could have used Paths.get() 2021-12-22 15:57:40 +00:00
CalDescent
67db0f950b Builds are now keyed by service, resourceId and identifier.
This allows for concurrent builds of multiple resources from the same registered name.
2021-12-22 15:44:00 +00:00
CalDescent
f85bbf12ca Applied reserved "default" keyword when publishing data too - it now maps to a blank identifier. 2021-12-22 13:59:08 +00:00
CalDescent
37e4f1e8d5 "default" is now a reserved keyword when used as an identifier.
Requesting a resource with the identifier "default" now maps to a blank string. This allows the /arbitrary/{service}/{name}/{identifier} endpoints to be used for default resources too, as they previously didn't support a blank string as the third parameter.
2021-12-22 13:33:08 +00:00
CalDescent
44d8bfd763 Added publish-auto-update-v5.pl - same as existing publish-auto-update.pl, but adapted to work with version 5 of ARBITRARY transactions.
The main differences are:
- Compute nonce instead of specifying a transaction fee
- Add blank/empty values for all the additional fields, as they are unused for auto updates
2021-12-22 13:00:29 +00:00
CalDescent
0cdbad6194 Bump version to 2.1.3-autoUpdateTest.0 - to check that auto updates still work with the QDN code. 2021-12-22 10:01:46 +00:00
CalDescent
4799a8a68e Don't allow files or file lists to be relayed if they relate to a blocked name.
This gives relay operators some control over the content they are relaying.
2021-12-22 09:21:08 +00:00
CalDescent
8caec81d1e Fixed content length bug 2021-12-21 16:15:20 +00:00
CalDescent
83d5bf45e5 Removed unnecessary call to isApiRestricted() 2021-12-21 16:04:32 +00:00
CalDescent
037eb8a163 Merge remote-tracking branch 'qortal/master' 2021-12-21 15:42:52 +00:00
CalDescent
3e5025b46e Domain map handling moved from RenderResource to a dedicated DomainMapResource
Including this as part of the rendering class was a bit of a hack; it's much cleaner to have a dedicated class.
2021-12-20 20:03:16 +00:00
CalDescent
35a5dc6219 Updated AdvancedInstaller project for v2.1.3 2021-12-19 22:53:04 +00:00
CalDescent
ace3ca0ad9 Bump version to 2.1.3 2021-12-19 21:10:24 +00:00
CalDescent
a8a498ddea Disable the names integrity check on startup by default.
This isn't really needed and is risky leaving it enabled, as there may be other unsupported cases.
2021-12-19 20:54:19 +00:00
CalDescent
d16663f0a9 Handle missing case in names integrity check caused by an UPDATE_NAME transaction with a blank "newName" value.
This is a valid transaction but not one that the integrity check was handling properly. Should fix NullPointerException on node startup.
2021-12-19 20:23:57 +00:00
CalDescent
623470209f Delete the combined file if has an incorrect hash after joining chunks together 2021-12-19 18:33:22 +00:00
CalDescent
553de5a873 Removed unnecessary javascript navigation code. 2021-12-19 17:25:58 +00:00
CalDescent
ccf8773b18 Added code that was left out from last commit. 2021-12-19 16:40:31 +00:00
CalDescent
cad25bf85d Add javascript to all HTML pages to allow for cross-origin navigation (back/forward buttons) from the UI.
Originally I had hoped to do this by using an intermediate iframe, to keep the message handlers separate from the user content, but this created CORS issues of its own. This approach is far simpler.
2021-12-19 12:21:29 +00:00
QuickMythril
9ce748452d Merge pull request #64 from Qortal/protoniuman-FR-patch-1
Protoniuman fr patch 1
2021-12-19 05:55:37 -05:00
Proto
9263d74b75 Update TransactionValidity_fr.properties 2021-12-19 11:16:20 +01:00
Proto
9601bddc84 Update SysTray_fr.properties 2021-12-19 11:05:34 +01:00
Proto
e281e19052 Add files via upload 2021-12-19 11:02:00 +01:00
Proto
0238b78f45 Update ApiError_fr.properties 2021-12-19 10:57:57 +01:00
QuickMythril
0ccee4326d Added French translations
credit: stchoupi
2021-12-18 15:57:49 -05:00
CalDescent
e9ab54f657 Fixed bug that prevented a resource from being overwritten if the existing data threw an exception when loading it. 2021-12-18 17:48:58 +00:00
CalDescent
0afb1a2d04 Make sure the blockchain is synced before running any data publishing code. 2021-12-18 16:07:29 +00:00
CalDescent
2d2b2964a5 When rebroadcasting an arbitrary signature list, don't send it back to the peer that we originally received it from. 2021-12-17 16:12:51 +00:00
CalDescent
10c4f7631b Added /zip versions of POST /arbitrary/* API endpoints.
These currently require the zip file's binary data to be encoded as base64, but it may not stay this way.
2021-12-17 14:28:07 +00:00
CalDescent
d921cffdaa Refactored ArbitraryResource to group by data type, to improve readability and code organization. 2021-12-17 12:50:00 +00:00
CalDescent
5369e21780 Allow the API whitelist to be easily disabled using the "apiWhitelistEnabled": false setting.
Now that we require API key authentication - and therefore security is greatly improved - many users will want to bypass the whitelist in order for the UI to communicate with their remote node. This gives an easy way to do this, without having to override the default whitelist. This boolean can now optionally be added to the default settings.json that is published with new releases, without removing the code's ability to update default whitelist values.
2021-12-17 10:22:08 +00:00
CalDescent
d34fb4494e Validate input data when uploading, to make sure it's not empty or missing. 2021-12-16 08:47:50 +00:00
CalDescent
1bd493ea37 Merge branch 'master' of github.com:Qortal/qortal
# Conflicts:
#	src/main/java/org/qortal/data/block/BlockData.java
2021-12-15 16:40:53 +00:00
CalDescent
391c3fe4c9 Added same functionality to GET /blocks/signature/{signature}
Also renamed query string parameter to "includeOnlineSignatures" to make it clearer.
2021-12-15 16:37:59 +00:00
CalDescent
3a7da9f13b Don't return online accounts signatures from GET /blocks/byheight/{height} unless requested using the includesignatures=true query string parameter.
This should fix issue where it would take up to 30 seconds to return for a recent block, and would consume masses of CPU due to having to base58 encode the online accounts signatures. Base58 is very slow and made this API endpoint almost unusable for recent blocks, due to them having untrimmed online accounts signatures.
2021-12-15 16:33:08 +00:00
CalDescent
3780767ccc Rotate hexagons by 90 degrees in the loading screen, to match the Qortal logo shape. 2021-12-15 13:27:42 +00:00
CalDescent
411279b3eb Removed all code relating to "fast sync" as it is not complete - we can reintroduce this later. 2021-12-15 13:27:12 +00:00
CalDescent
be3069e0e5 Only rebroadcast file list requests when in relay mode. 2021-12-15 12:29:05 +00:00
CalDescent
22cf870555 Updated some wording. 2021-12-15 12:16:11 +00:00
CalDescent
d6479c1390 ArbitraryDataFile.cleanupFilesystem() now uses generic FilesystemUtils.safeDeleteEmptyParentDirectories() code. 2021-12-15 12:15:58 +00:00
CalDescent
a4e82c79cc Completed implementation of relay mode
The procedure outlined in commit f4b06fb is now incorrect. Updated procedure:

- A node can opt into relay mode via the "relayModeEnabled":true setting.
- From this time onwards, they will ask their peers if they ever receive a file list request that they cannot serve by themselves.
- Whenever a peer responds with a file list, it is forwarded on to the originally requesting peer, complete with the peer address of the node that responded. Currently, only the first response is forwarded, but we may later decide to forward all responses.
- As well as forwarding, the relay peer keeps track of the peers that report to be holding hashes (these mappings are held for 30 seconds).
- The originally requesting peer can then make a request to the relay peer for the data file(s).
- The relay peer uses the mapping to forward the request on to another peer, and then forwards the response (i.e. the data file) back to the peer that originally requested the file.
2021-12-15 12:15:31 +00:00
CalDescent
bcc89adb5f Removed "peerAddress" from ArbitraryDataFileListMessage (introduced in recent commit)
It's best that the source peer's address isn't exposed to the requesting peer. The relay peer can keep track of this mapping itself.

The only real issue with this approach is that we can't use data from ArbitraryDataFileListMessage to update our ArbitraryPeers data, because we can't distinguish between relay peers and hosting peers. But this isn't something we currently do anyway, as we have the ARBITRARY_SIGNATURES message type to take care of updating ArbitraryPeers mappings.
2021-12-15 11:40:54 +00:00
CalDescent
a41c9e339a Fixed occasional NPE 2021-12-15 11:31:12 +00:00
CalDescent
feeca77436 Added directDataRetrievalEnabled setting (default true)
Setting this to false prevents new connections being made to peers that report to have the data that is needed. This is likely only useful for testing, as disabling it in production would reduce the success rate of data retrieval.
2021-12-15 11:29:49 +00:00
CalDescent
fcce12ba40 Added missing code from last commit. 2021-12-15 11:28:34 +00:00
CalDescent
f4b06fb834 Added relay mode for file list requests
This reuses most of the code already in place in the core related to forwarding.

- A node can opt into relay mode via the "relayModeEnabled": true setting
- From this time onwards, they will ask their peers if they ever receive a file list request that they cannot serve by themselves
- Whenever a peer responds with a file list, it is forwarded on to the originally requesting peer, complete with the peer address of the node that responded
- The original peer can then make a request for the data file(s) themselves using a similar approach, specifying the IP address of the ultimate peer so that the relay node knows who to ask. This part is not implemented yet.
2021-12-12 10:51:24 +00:00
CalDescent
e7fd803d19 Updated AdvancedInstaller project for v2.1.2 2021-12-11 22:16:43 +00:00
CalDescent
3b96747871 Bump version to 2.1.2 2021-12-11 21:50:54 +00:00
CalDescent
33088df07d Revert "Bump log4j-api version to 2.15.0"
This reverts commit 6a9904fd43.
2021-12-11 19:40:44 +00:00
CalDescent
a215714b6b Set log4j2.formatMsgNoLookups to true 2021-12-11 19:09:25 +00:00
CalDescent
6a9904fd43 Bump log4j-api version to 2.15.0
The main log4j version cannot be increased yet due to some incompatibilities with the Qortal code.
2021-12-10 17:54:11 +00:00
CalDescent
cc297ccfcd Removed the word "service" from all settings, to avoid confusion with arbitrary data services.
Updated setting names:

domainMapServiceEnabled -> domainMapEnabled
domainMapServicePort -> domainMapPort
gatewayServiceEnabled -> gatewayEnabled
gatewayServicePort -> gatewayPort
2021-12-10 12:29:33 +00:00
CalDescent
c7c88dec04 Attempt to differentiate between resources that are downloading, and ones where downloading hasn't been attempted yet. 2021-12-10 12:22:26 +00:00
CalDescent
e481a5926a Use "blocked" instead of "blacklisted", for consistency with the buttons and terminology in the UI 2021-12-10 11:34:10 +00:00
CalDescent
0464245218 Simplified lists feature to be referenced by a single name, instead of the confusing "category" and "resourceName" properties.
This makes them extremely generic, improves filenames, and makes it easier to create custom lists. It doesn't have backwards support, but the lists feature isn't working properly in core 2.1+ anyway.
2021-12-10 11:33:26 +00:00
CalDescent
0001f31c06 Merge remote-tracking branch 'qortal/master' 2021-12-10 09:35:44 +00:00
CalDescent
391d31759a Fixed small issue in GET /crosschain/trades and GET /crosschain/tradeoffers APIs where 0 was being treated as zero instead of unlimited.
In 2.1.1, unlimited results can be requested by removing the `limit` query string parameter completely, e.g:

http://127.0.0.1:12391/crosschain/trades?foreignBlockchain=LITECOIN&minimumTimestamp=1638835200000&reverse=false
2021-12-08 20:22:02 +00:00
CalDescent
ed2f2435d2 Updated AdvancedInstaller project for v2.1.1 2021-12-08 19:32:36 +00:00
CalDescent
6e6b2ccfa0 Bump version to 2.1.1 2021-12-08 18:39:30 +00:00
CalDescent
be9a73560d Added one more node 2021-12-08 17:23:42 +00:00
CalDescent
e82b5a4ecf Added 7 qortalnodes.live nodes to the default list. 2021-12-08 16:10:50 +00:00
CalDescent
a27d8ac828 Connect ACCTv3 trade bots to the ACCTv3 smart contracts 2021-12-08 12:52:22 +00:00
CalDescent
6267258189 Switch over to ACCTv3 for new listings 2021-12-08 12:22:26 +00:00
CalDescent
e7527f532e Added LitecoinACCTv3 and DogecoinACCTv3 to SupportedBlockchain 2021-12-08 12:21:52 +00:00
CalDescent
8b6e74d505 Added ACCTv3 trade bots. Identical to v1 and v2. 2021-12-08 12:18:22 +00:00
CalDescent
e6106c0c4e Added ACCTv3 tests, based on the same criteria as ACCTv1. Refunds are working as intended. 2021-12-08 12:10:36 +00:00
CalDescent
f52bafc014 Removed second "SLEEP_UNTIL_MESSAGE" function code call in LitecoinACCTv3 and DogecoinACCTv3.
It turns out that when you call SLEEP_UNTIL_MESSAGE, the AT resumes from that very same line on the next execution. The original code incorrectly assumed that it would execute from the restart position (SET_PCS).

So sleeping can be thought of as pausing one execution half way through, rather than ending it.

This caused a bug, because once the AT receives a transaction it wakes up and resumes from the SLEEP_UNTIL_MESSAGE line, which is after the refund check. Even when it loops back around again it lands on labelRedeemTxnLoop = codeByteBuffer.position(); which is again after the refund check.

For now, the simplest fix is to only sleep when listed. We could have alternatively moved the SLEEP_UNTIL_MESSAGE above GET_BLOCK_TIMESTAMP, but this would still require users to send a random transaction to the AT to trigger the refund. Given that the ATs are only "alive" for 30 minutes once the trade begins, it's simpler to just execute every block and therefore allow the refunds to happen automatically.
2021-12-08 12:08:06 +00:00
CalDescent
9e0630ea79 Added LitecoinACCTv3 and DogecoinACCTv3 - at the moment identical to ACCTv2 2021-12-08 11:55:25 +00:00
CalDescent
968bfb92d0 Fixed bugs in the GET /crosschain/tradeoffers API endpoint caused by the introduction of ACCTv2 2021-12-07 22:32:46 +00:00
CalDescent
284c9fcee2 Fixed bugs in the GET /crosschain/price API endpoint caused by the introduction of ACCTv2 2021-12-07 21:20:08 +00:00
CalDescent
5b0b939531 Fixed bugs in the GET /crosschain/trades API endpoint caused by the introduction of ACCTv2 2021-12-06 20:14:08 +00:00
CalDescent
2efac0c96b Modified zip implementation to preserve filenames of single file resources.
Also modified the directory structure of single file resources to make them consistent with multi file resources.

For multi file resources, the original folder is renamed to "data", resulting in a layout such as:
data/file1.txt
data/file2.txt
data/dir1/file3.txt

For single file resources, the file is now moved into a "data" folder, like so:
data/file.txt

This is slightly unconventional, but is appropriate within the context of QDN to keep everything consistent.
2021-12-06 19:53:20 +00:00
CalDescent
dc52fd1dcf Only return OFFERING trades in GET /crosschain/tradeoffers 2021-12-06 19:37:30 +00:00
CalDescent
19240a9caf Removed duplicated strings in zip tests 2021-12-05 18:13:02 +00:00
CalDescent
4eef28f93d Added unit tests for zipping / unzipping 2021-12-05 17:38:53 +00:00
CalDescent
c6e5c4e3b5 Tidy up of storage thresholds and comments. Also reduced deletion threshold from 100% to 98% to reduce the chances of filling up the disk and corrupting the db. 2021-12-05 16:36:54 +00:00
CalDescent
007f567c7a Improved directory cleanup process to avoid leaving empty parent directories lying around. 2021-12-05 16:24:47 +00:00
CalDescent
ffe178c64c Delete directories in the data folder that have no associated transaction.
This adds support for "unconfirmable" data uploads, which will be useful for Q-Chat. It also handles cases where a transaction is orphaned and then subsequently becomes invalid.
2021-12-05 16:24:21 +00:00
CalDescent
c3835cefb1 Fixed bug relating to storage manager instances. 2021-12-05 14:42:12 +00:00
CalDescent
2c382f3d3f Fixed logging error. 2021-12-05 13:30:22 +00:00
CalDescent
b592aa6a02 Added custom validation for websites.
A website must contain one of the following files in its root directory to be considered valid:

index.html
index.htm
default.html
default.htm
home.html
home.htm

This is the first page that is loaded when loading a Qortal-hosted website.
2021-12-05 13:30:10 +00:00
CalDescent
57e82b62a1 Increased the capabilities of the service validation functions. 2021-12-05 13:03:22 +00:00
CalDescent
13f3aca838 Added GET /addresses/online/levels API endpoint to return the number of minters at each level 2021-12-04 19:28:28 +00:00
CalDescent
94b17eaff3 Added unit test for random file deletion, and fixed some issues found via the test. 2021-12-04 16:36:20 +00:00
catbref
eb9b94b9c6 Add Qortal AT FunctionCodes for getting account level / blocks minted + tests 2021-12-04 16:36:05 +00:00
CalDescent
a3038da3d7 Moved some shared arbitrary test methods to a new ArbitraryUtils class. 2021-12-04 14:26:10 +00:00
CalDescent
6026b7800a Fixed some warnings. 2021-12-04 14:23:21 +00:00
CalDescent
36c5b71656 Delete data associated with a name at random, if a name is using more than its allocated limit.
This would happen if a name fills their limit, and then additional names are followed. Alternatively it could happen if the total storage capacity reduces due to disk space being used by other apps. Chunks are deleted at random to reduce the chance of the same chunk being deleted everywhere. Data loss is possible here for transactions that don't have many peers. We'll have to see in practice how much of a problem this is, but it's better than the scenario where one content creator consumes all space on their followers' nodes, leaving no space for other names that are subsequently followed.
2021-12-04 14:23:09 +00:00
CalDescent
a320bea68a Limit the amount of data that can be stored per name.
This is calculated by the total capacity divided by the number of names the node follows. The idea here is that a single content creator can't upload terabytes of data and consume all the space on their followers' nodes. They can only use a proportion, with equal space given to each followed name. And since the limit is dynamic, following more names reduces the allocation to existing names.
2021-12-04 13:33:45 +00:00
CalDescent
a87fe8b44d Combined path filtering into a single filter to avoid iterating through the list multiple times. 2021-12-04 12:06:15 +00:00
CalDescent
0a2b4dedc7 Don't sort the transactions in listAllHostedTransactions() as this is a waste of CPU. 2021-12-04 12:03:52 +00:00
CalDescent
f7ed3eefc8 Verify that each data file matches the size reported by transaction.
This discourages an incorrect file size being included with a transaction, as the system will reject it and won't even serve it to other peers.

FUTURE: we could introduce some kind of blacklist to track invalid files like this, and avoid repeated attempts to retrieve them. It is okay for now as the system will backoff after a few attempts.
2021-12-04 12:00:38 +00:00
CalDescent
8bb3a3f8a6 Added hosted transactions cache
This skips the need to check the filesystem if no new files have arrived since the last lookup.
2021-12-04 09:28:45 +00:00
CalDescent
89d08ca359 Renamed method from listAllHostedData() to listAllHostedTransactions() 2021-12-04 09:27:39 +00:00
CalDescent
b80aec37e0 Added GET /arbitrary/hosted/resources API to list all arbitrary resources that the node is hosting (at least partial) data for. 2021-12-03 21:43:18 +00:00
CalDescent
e34fd855a9 Added GET /arbitrary/hosted/transactions API to list all arbitrary transactions that the node is hosting (at least partial) data for.
This API call could get quite heavy when large amounts of files are hosted, but it's preferable to maintaining a list in the database. Ideally we need to keep the database generic so that it can be bootstrapped without interfering with the state. We can always add caching and rate limiting if needed.
2021-12-03 21:11:56 +00:00
CalDescent
fc12ea18b8 Fixed bug in getArbitraryResources() 2021-12-03 18:17:20 +00:00
CalDescent
f87df53791 Don't compute a nonce when creating arbitrary transactions. Instead this can be done by calling POST /arbitrary/compute, or more ideally by the client, such as the UI. 2021-12-03 17:33:07 +00:00
CalDescent
d6746362a4 Reduced log spam when a file can't be served. 2021-12-02 19:41:31 +00:00
CalDescent
2850bd0b46 Added new GET /arbitrary/resources/names endpoint to fetch resources grouped by name. 2021-12-02 19:41:07 +00:00
CalDescent
b762eff4eb Announce to the network when uploading a new file.
Previously only subsequent peers would announce, not the initial uploader. This made it very difficult to locate brand new files.
2021-12-01 20:42:51 +00:00
CalDescent
4b3b96447f Require an API key or prior authorization on GET /arbitrary/resource/status/* endpoints 2021-12-01 20:03:36 +00:00
CalDescent
13bcfbe3c5 Fixed issues preventing the loading screen from working when using the gateway. 2021-12-01 19:51:45 +00:00
CalDescent
8525fb89f8 Use a zero fee for ARBITRARY transactions, as we require a PoW nonce instead. 2021-12-01 18:12:33 +00:00
CalDescent
ed2d1c4932 Improved logging 2021-12-01 16:14:24 +00:00
CalDescent
5091f8457e Terminate metadata files with a newline 2021-12-01 16:13:59 +00:00
CalDescent
84b69fc58c Revert "Use response code 102 ("Processing") instead of 503 ("Service Unavailable") for the loading screen"
This reverts commit 8823f69256.
2021-12-01 13:56:47 +00:00
CalDescent
a2cac003a4 Major rework of chunk hashes
Chunk hashes are now stored off chain in a metadata file. The metadata file's hash is then included in the transaction.

The main benefits of this approach are:
1. We no longer need to limit the total file size, because adding more chunks doesn't increase the transaction size.
2. This increases the chain capacity by a huge amount - a 512MB file would have previously increased the transaction size by 16kB, whereas it now requires only an additional 32 bytes.
3. We no longer need to use variable difficulty; every transaction is the same size and so the difficulty can be constant no matter how large the files are.
4. Additional metadata (such as title, description, and tags) can ultimately be stored in the metadata file, as apposed to using a separate transaction & resource.
5. There is also scope for adding hashes of individual files into the metadata file, if we ever wanted to allow single files to be requested without having to download and build the entire resource. Although this is unlikely to be available in the short term.

The only real negative is that we now how to fetch the metadata file before we know anything about the chunks for a transaction. This seems to be quite a small trade off by comparison.

Since we're not live yet, there is no backwards support for on-chain hashes, so a new data testchain will be required. This hasn't been tested outside of unit tests yet, so there will likely be several fixes needed before it is stable.
2021-12-01 12:37:21 +00:00
CalDescent
7c16a90221 Moved relocation code from isDataLocal() to onImportAsUnconfirmed()
It's not good to be moving files around in a method that should really be read only. This also adds an intentional checkAndRelocateMiscFiles() call rather than relying on a call to isDataLocal() which may be removed at any time.
2021-12-01 12:13:23 +00:00
CalDescent
97cdd53861 Fixed bugs in safeDeleteEmptyParentDirectories() 2021-12-01 11:59:14 +00:00
CalDescent
b7ee00fb22 Fixed errors in Admin API tests due to failing authentication. 2021-11-27 20:08:59 +00:00
CalDescent
ef2ee20820 Merge remote-tracking branch 'qortal/master'
# Conflicts:
#	pom.xml
#	src/main/java/org/qortal/api/resource/ListsResource.java
#	src/main/java/org/qortal/list/ResourceList.java
#	src/main/java/org/qortal/list/ResourceListManager.java
#	src/main/java/org/qortal/transaction/ChatTransaction.java
2021-11-27 19:41:17 +00:00
CalDescent
4866e5050a If a single file resource is being published and a complete file patch has been chosen, make sure to use PUT instead of PATCH as there's nothing to be gained by adding another layer.
This would have been caught by the max differences check anyway, but it's a good check to have in place in case we recalibrate or remove the differences check in the future.
2021-11-27 19:35:22 +00:00
CalDescent
8e36c456e1 Wait for storage space to be calculated before running storage policy tests. 2021-11-27 18:06:48 +00:00
CalDescent
4b8bcd265b Make sure unit test use a different lists directory, and delete it before and after each test. 2021-11-27 18:05:25 +00:00
CalDescent
0db681eeda Fixed failing storage policy tests due to not calculating the available storage 2021-11-27 17:56:34 +00:00
CalDescent
8823f69256 Use response code 102 ("Processing") instead of 503 ("Service Unavailable") for the loading screen 2021-11-27 16:48:47 +00:00
CalDescent
f3e9dfe734 Return a 404 instead of a 500 if a resource is unavailable.
Could be improved in the future to return different codes depending on its status (e.g. doesn't exist = 404, 102 for loading, 500 for error, etc), but 404 makes the most sense until that has been developed
2021-11-27 16:43:29 +00:00
CalDescent
a7b31ab1f9 Small bug fixes 2021-11-27 16:40:12 +00:00
CalDescent
644ab27186 Updated wording 2021-11-27 14:22:09 +00:00
CalDescent
e90ecd2085 Adapted GET /arbitrary/resources endpoint to allow filtering by identifier
- If an identifier parameter is missing or empty, it will return an unfiltered list of all possible identifiers.
- If an identifier is specified, only resources with a matching identifier will be returned.
- If default is set to true, only resources without identifiers will be returned.
2021-11-27 14:21:36 +00:00
CalDescent
bc38184ebf Major rework of local data directory structure
Files are now keyed by signature, in the format:
data/si/gn/signature/hash

For times when there is no signature available (i.e. at the time of initial upload), files are keyed by hash, in the format:
data/_misc/ha/sh/hash

Files in the _misc folder are subsequently relocated to a path that is keyed by the resulting signature.

The end result is that chunks are now grouped on the filesystem by signature. This allows more transparency as to what is being hosted, and will also help simplify the reporting and management of local files.
2021-11-27 13:00:32 +00:00
CalDescent
d9de27e6f2 Updated AdvancedInstaller project for v2.1.0 2021-11-24 19:39:37 +00:00
CalDescent
6930bf0200 Bump version to 2.1.0 2021-11-24 18:36:40 +00:00
CalDescent
199833bdd4 Fixed issue with GET /crosschain/trades API endpoint where the minimumTimestamp parameter returned inconsistent results. 2021-11-24 15:21:08 +00:00
CalDescent
0dcd2e6e93 Fixed bug in GET ​/crosschain​/price​/{blockchain} inverse price API endpoint when the "inverse" parameter is null. 2021-11-24 15:09:45 +00:00
CalDescent
0dd43d5c9a Fixed bug in storage calculation 2021-11-24 14:22:57 +00:00
CalDescent
e879bd0fc5 Delete some random chunks when we reach the storage capacity
This should allow for a relatively even distribution of chunks, but there is a (currently unavoidable) risk of files with very few mirrors being deleted altogether.

Longer term this could be improved by checking that one of our peers has a file, before it's deleted locally
2021-11-24 14:15:22 +00:00
CalDescent
8bf7daff65 Track the storage capacity and the total data/temp directory sizes
Nodes will stop proactively storing new data when they reach 90% capacity.

A new "maxStorageCapacity" setting has been added to allow the user to optionally limit the allocated space for this node. Limits are approximate only, not exact.
2021-11-24 13:43:45 +00:00
CalDescent
ae0f01d326 Added storage policy unit tests 2021-11-24 11:02:54 +00:00
CalDescent
af8d0a3965 Separated computeNonce() from build() in the transaction builder.
This gives the option of the nonce to be computed elsewhere, such as in the UI, and also allows transaction unit tests to run much more quickly.
2021-11-24 11:02:17 +00:00
CalDescent
1b170c74c0 Modified storage code to support 2 new settings:
publicDataEnabled - whether to store decryptable data (default true)
privateDataEnabled - whether to store data without a decryption key (default false)
2021-11-24 09:38:18 +00:00
CalDescent
f6b9ff50c3 More loading screen improvements 2021-11-23 22:21:57 +00:00
CalDescent
9ef75ebcde Improved styling of loading panel 2021-11-23 21:15:45 +00:00
CalDescent
f76a618768 Display the latest status on the loading screen, updated via API calls on a timer 2021-11-23 20:53:09 +00:00
CalDescent
098d7baa4d Bump version to 2.1.0-prerelease.0 2021-11-23 19:03:27 +00:00
CalDescent
59a57d3d28 Increased frequency of automatic repository maintenance attempts
repositoryMaintenanceMinInterval reduced from 7 to 3 days
repositoryMaintenanceMaxInterval reduced from 30 to 14 days
2021-11-23 18:47:27 +00:00
CalDescent
cce95e09de Default min level for block submissions increased to 3
This doesn't affect minting rewards; it is simply a means of reducing block candidates. There should be no noticeable difference other than hopefully less re-orgs. We can ultimately do a hard fork and increase Blockchain.minAccountLevelToMint but this allows us to test the approach in a lower risk way.
2021-11-23 18:45:35 +00:00
CalDescent
ec48ebcd79 Improved resource statuses 2021-11-23 09:14:44 +00:00
CalDescent
908f80a15d Fixed bug when checking if all files exist locally in /arbitrary/status 2021-11-22 08:43:07 +00:00
CalDescent
02eab89d82 Fixed bug when trying to delete a file instead of a directory. 2021-11-21 19:24:20 +00:00
CalDescent
c588786a06 Added /base64 variation of POST /arbitrary/* APIs
This can be used to upload base64 encoded file data directly from the UI. Using base64 because base58 is unusably slow
2021-11-21 19:12:01 +00:00
CalDescent
b4f3105035 Added /render/authorize/{service}/{resourceId}* APIs
These allow the UI to pre-authorize a resource and therefore avoid having to pass a sensitive API key to a website or app.
2021-11-21 14:57:26 +00:00
CalDescent
d018f11877 Log the initial height of the block archiver on startup 2021-11-21 09:12:32 +00:00
CalDescent
d0000c6131 If "build=true" is specified in query string of GET /resource/status/{service}/{name}, build the resource before returning the status 2021-11-20 18:52:03 +00:00
CalDescent
c05ffefd7d Added a dynamic background to the loading screen. Still needs a lot of work. 2021-11-20 14:09:00 +00:00
CalDescent
530fc67a05 List available services in API docs when requesting arbitrary data 2021-11-20 11:46:03 +00:00
CalDescent
c79ec11b07 Fixed incorrect comment 2021-11-19 23:16:28 +00:00
CalDescent
668ef26056 Fixed major performance issue with GET /arbitrary/resources?includestatus=true
The missing data check was triggering decryptions, extractions, etc. Replaced with some code which checks for the presence of chunks on the local machine, without getting involved with any build process overhead.
2021-11-19 23:00:36 +00:00
CalDescent
75ec7723ef Improved accuracy of statuses
- "NOT_STARTED" is now "DOWNLOADED"
- "DOWNLOADING" is now "MISSING_DATA"
- Removed "DOWNLOAD_FAILED"

Some of these could be reintroduced once the system is able to support them.
2021-11-19 21:48:41 +00:00
CalDescent
73e609fa29 Replaced all IllegalStateException with DataException in arbitrary code
This was necessary to ensure that all exceptions are caught intentionally, as otherwise it creates endless amounts of edge cases.
2021-11-19 21:42:03 +00:00
CalDescent
8cb06bf451 Return statuses in GET /arbitrary/resources endpoint when the "includestatus" parameter is true. 2021-11-19 20:20:45 +00:00
CalDescent
1be8a059f4 Fixed bug caused by not catching a IllegalStateException
I may replace all IllegalStateExceptions with DataExceptions soon so we can ensure they are always caught.
2021-11-19 20:18:56 +00:00
CalDescent
7f41c7ab0e Added "BLACKLISTED" status for arbitrary data resources. 2021-11-19 20:18:00 +00:00
CalDescent
3860c5d8ec Fixed some failing tests. 2021-11-19 16:12:31 +00:00
CalDescent
a061a7cc4d Fixed various warnings raised by the IDE. 2021-11-19 16:11:37 +00:00
CalDescent
844501d6cd Added GET /arbitrary/resource/status/* API endpoints
These can be used to check the current status of a resource. The different statuses are:

NOT_STARTED,
DOWNLOADING
DOWNLOADED
BUILDING
READY
DOWNLOAD_FAILED
BUILD_FAILED
UNSUPPORTED

Not all statuses are returned yet. The build process needs more functionality to be able to support DOWNLOADED and DOWNLOAD_FAILED. Also, BUILDING and BUILD_FAILED are currently unable to distinguish between different resources with the same registered name, so need some attention.
2021-11-19 15:26:52 +00:00
CalDescent
020bd00b8f Removed incorrect @SecurityRequirement annotation 2021-11-19 13:40:02 +00:00
CalDescent
0706b0d287 Added some /site endpoints to the gateway, for backwards support of the demo sites 2021-11-19 13:26:47 +00:00
CalDescent
ce56cd2b16 Disallow local (loopback address) requests when using the gateway
This removes the possibility of some locally running javascript in a website or app requesting unvetted data via the local gateway.
2021-11-19 13:20:53 +00:00
CalDescent
b7a0a7eea4 Removed API authentication when using the gateway, as that would defeat the purpose of it. 2021-11-19 13:14:57 +00:00
CalDescent
824d14e793 Removed unnecessary check for isApiRestricted() when previewing.
The API key authentication will be enough to restrict requests.
2021-11-19 13:08:56 +00:00
CalDescent
83e0ed2b5d Reduced log spam 2021-11-19 12:59:25 +00:00
CalDescent
c8b70b51c3 Added gateway service, to allow websites to be served directly on a domain or IP.
This replaces the existing GET /site/{name} API endpoints.

Example settings:

"gatewayServiceEnabled": true,
"gatewayServicePort": 80

Websites can then be served using URL:

http://localhost/RegisteredName

Or, if node is behind public DNS:

http://example.com/RegisteredName

Or, if a custom port (such as 12393) is used:

http://localhost:12393/RegisteredName
http://example.com:12393/RegisteredName

This is currently for serving websites only, but can be adapted to serve other services if needed.
2021-11-19 12:59:15 +00:00
CalDescent
c0fedaa3a4 Attempt to request files directly from a peer if it isn't returned in the general network broadcast. 2021-11-19 12:05:40 +00:00
CalDescent
e74dcff010 Wait for 3 seconds between attempts to allow time for data to arrive. 2021-11-19 11:21:40 +00:00
CalDescent
3b5b45b463 Give up after 5 attempts to request data in the GET /arbitrary APIs 2021-11-19 11:21:14 +00:00
CalDescent
fead482b0d Fixed bugs introduced in preview functionality. 2021-11-17 19:32:35 +00:00
CalDescent
29bd8203b5 Removed service from POST /render/preview/{service} API as it isn't needed right now 2021-11-17 19:25:21 +00:00
CalDescent
08b79e45cf /site API endpoints replaced with more generic /render APIs so that they can be used for apps, blogs, etc
This involves passing a service along with the name, such as `GET /render/WEBSITE/Test`
2021-11-17 19:22:25 +00:00
CalDescent
3a05a0bcaa Added "LIST" service - to be used to optionally host an otherwise private list if the user wants to share it publicly. 2021-11-17 18:59:45 +00:00
CalDescent
d0aafaee60 Added POST /arbitrary/../string API endpoints to allow data to be passed to the core as a string.
This will be useful for metadata, playlists, etc, as well as some types of data published by Qortal apps.
2021-11-17 18:57:46 +00:00
CalDescent
332b874493 Removed /arbitrary PUT and PATCH API endpoints.
It's best to let the core decide which one to use now that it is able to.
2021-11-16 19:36:24 +00:00
CalDescent
6c995ed738 Validation removed from METADATA so that it is more generic - it's up to each application to decide how to structure its contents. Existing strict validation applied to a duplicate called QORTAL_METADATA. This will be the one used for website/app listings in the Qortal UI. 2021-11-16 19:32:14 +00:00
CalDescent
fb09d77cdc Rework of "Service" types to allow for validation
Each service supports basic validation params, plus has the option for an entirely custom validation function.

Initial validation settings:
- IMAGE must be less than 10MiB
- THUMBNAIL must be less than 500KiB
- METADATA must be less than 10KiB and must contain JSON keys "title", "description", and "tags"
2021-11-16 19:28:25 +00:00
CalDescent
9c952785e6 Allow the API key to be passed as an "apiKey" parameter in the query string or POST body as an alternate option to a header.
This is needed to avoid triggering a CORS preflight (which occurs when using an X-API-KEY header). The core isn't currently capable of responding to a preflight and the UI therefore blocks the entire request. See: https://stackoverflow.com/a/43881141
2021-11-14 20:24:02 +00:00
CalDescent
2f51c1bf47 Added all missing SecurityRequirement annotations 2021-11-14 19:56:26 +00:00
CalDescent
276a110e90 Fixed bug in API key comparison 2021-11-14 19:55:58 +00:00
CalDescent
b761674b2c Default temp path moved to a subfolder of the data path
This allows users to set only their data path, and for the temp folder to automatically follow it. The temp folder can be moved to a custom location by setting the "tempDataPath" setting.
2021-11-14 17:00:49 +00:00
CalDescent
0b20bf0145 Website serving now requires authentication for everything except the "domain map" server 2021-11-14 16:00:58 +00:00
CalDescent
1397cbeac2 General API key / security-related updates 2021-11-14 15:59:08 +00:00
CalDescent
06e122f303 Added 'localAuthBypassEnabled' setting to allow users to opt in to the old method of local authentication at their own risk. 2021-11-14 15:24:15 +00:00
CalDescent
f062acfd7c Rework of API keys
An API key is now _required_ for sensitive API calls that would previously have allowed local loopback authentication.

Previously, a request would have been considered authenticated if it originated from the same machine, however this creates a security issue when running third party code (particularly javascript) via the data network.

The solution is to now require an API key to authenticate sensitive API calls no matter where the request originates from.

It works as follows:

- When the core is first installed, it has no API key generated and will block sensitive calls until generated.
- A new POST /admin/apikey/generate API endpoint has been added, which can be used the generate an API key for a newly installed node. The UI will ultimately call this automatically.
- This API returns the generated key so that it can be stored by the requesting app (most likely the UI).
- From then on, the generate API requires authentication via the existing API key in order to regenerate a key. It can be used as a security measure if the existing key is compromised.
- The API key must be passed to all sensitive API endpoints from then on, even when calling it from the same local machine.
- If the core already has a legacy API key specified via the 'apiKey' setting, this will be automatically copied to the new format so that a new one doesn't need to be generated.
- The API key itself is stored in a flat file in the qortal directory (the path can be customized using the `apiKeyPath` setting). Deleting this file and restarting the core will allow a new one to be regenerated.
2021-11-14 15:14:37 +00:00
CalDescent
97ca414fc0 Revert "Added "apiKeyDisabled" setting to bypass API key / loopback checking for those who need it."
This reverts commit 8a7446fb40.
2021-11-13 19:19:54 +00:00
CalDescent
a9af5bcec4 Website serving code moved to a new class called ArbitraryDataRenderer
The process of serving resources to a browser will likely be needed for more than just websites (e.g. it will be needed for apps too) so it makes sense to abstract it to its own class.
2021-11-13 19:09:35 +00:00
CalDescent
7e30bf4197 Fixed website preview functionality which isn't compatible with asynchronous building.
The simplest solution was to build synchronously when previewing.
2021-11-13 17:40:09 +00:00
CalDescent
c724ea9f69 Removed various /arbitrary API endpoints that were only really useful at the start of the data storage project. 2021-11-13 17:11:40 +00:00
CalDescent
e6cc4a1180 Improved logging for times when data requests are rate limited. 2021-11-13 16:59:39 +00:00
CalDescent
3cce097b9d When a newer PUT exists for a resource, delete records of peers holding earlier transactions
This should help keep the peer lookup table size down, as there is no need to locate files for transactions that existed before the most recent PUT transaction.
2021-11-13 16:58:23 +00:00
CalDescent
53f9d6869d Improved logging when a resource has no identifier 2021-11-13 16:52:02 +00:00
CalDescent
61beee0f49 Tidied up unfinished arbitrary data payments code. 2021-11-13 15:00:52 +00:00
CalDescent
1f3d400ad6 Small refactor of previous commit's code to improve consistency. 2021-11-13 13:47:29 +00:00
CalDescent
f2ff2187d9 Case sensitivity preference can now be specified when checking if an item is in a list.
All registered name checks are now case insensitive, since the names themselves are case insensitive.
2021-11-13 13:37:16 +00:00
CalDescent
28ddc0055f Implemented reader cache clearing
Built resources are deleted when either:
- The resource reaches the expiry interval specified in the builtDataExpiryInterval setting (default 30 days)
- The resource is published by a name that is in the local blacklist

Resources only exist in the reader cache once they have been viewed, to remove the loading time on subsequent views. But some may prefer to reduce this expiry time (at the expense of longer load times and more CPU), as data is held unencrypted in the cache.
2021-11-13 13:35:40 +00:00
CalDescent
90b5b6bd8b Don't allow data to be fetched for viewing if the name is in the local blacklist.
We still allow it to be fetched even if it's outside of the storage policy, as the cleanup manager will delete the files very soon after, and they won't be allowed to be served to other peers due to other checks already in place.
2021-11-13 12:54:14 +00:00
CalDescent
53466797a5 Once we receive a file from a peer, add the mapping to the lookup table.
This allows other peers to find out where they can obtain these files if we were to stop hosting them later. Or even if we continue hosting copies, it still informs the network on other locations, for better decentralization.
2021-11-13 12:50:26 +00:00
CalDescent
f5235938b7 Rate limit any file list broadcasts
We don't want the network being spammed when a file isn't available by any reachable peers. This feature ensures retries are spaced out over longer timeframes. Basic logic:

- Wait 5 minutes in between failed attempts
- After 5 failed attempts (i.e. 25 mins) only try once per day from then on
- A core restart resets the counters

The stats gathered here can also be used to inform the core of when it should attempt a direct connection with a peer to obtain the data. That part isn't implemented yet.
2021-11-13 12:26:27 +00:00
CalDescent
054860b38d Rework of storage policy handling, as the previous implementation didn't handle viewed data properly.
This also adds a feature to allow data to be deleted and no longer served once a name has been blacklisted.
2021-11-13 11:35:29 +00:00
CalDescent
b60d02b8f4 Fixed preexisting list name issue when blocking chat transactions by address 2021-11-13 11:13:19 +00:00
CalDescent
0d69797851 Block chat transactions on the local node if its sender owns a name that is blacklisted by the user. 2021-11-13 11:12:52 +00:00
CalDescent
bfffff0750 /lists APIs now made fully generic
This allows for custom list creation without the need for creating API endpoints to go along with it. This should save time now that we are using lists more.
2021-11-13 11:00:01 +00:00
CalDescent
b7bcd8da7d Prevent arbitrary data transactions being created unless the network supports it (i.e. the hard fork has taken place). 2021-11-13 10:29:01 +00:00
CalDescent
d3862c97ba Added "APP" and "METADATA" service types
- "APP" will allow for user-created apps and the Qortal app store
- "METADATA" will be used to supply info about apps/websites/resources, such as title, description, tags, etc
2021-11-13 10:06:53 +00:00
CalDescent
c069c39ce1 Implemented automatic PUT/PATCH detection
When using POST /arbitrary/{service}/{name}... it will now automatically decide which method to use (PUT/PATCH) based on a few factors:

- If there are already 10 or more layers, use PUT to reset back to a single layer
- If the next layer's patch is more than 20% of the total resource file size, use PUT
- If the next layer modifies more than 50% of the total file count, use PUT
- Otherwise, use PATCH

The PUT method causes a new base layer to be created and all previous update history for that resource becomes obsolete. The PATCH method adds a small delta layer on top of the existing layer(s).

The idea is to wipe the slate clean with a new base layer once the patches start to get demanding for the network to apply. Nodes which view the content will ultimately have build timeouts to prevent someone from deploying a resource with hundreds of complex layers for example, so this approach is there to maximize the chances of the resource being buildable.

The constants above (10 layers, 20% total size, 50% file count) will most likely need tweaking once we have some real world data.
2021-11-13 09:56:13 +00:00
CalDescent
e994d501b0 Fixed errors in documentation 2021-11-12 18:22:10 +00:00
CalDescent
caf163f98c Include "tempDataPath" in test settings so that tests don't put files in the main temp directory. 2021-11-12 17:46:48 +00:00
CalDescent
1c408db907 Rework of arbitrary APIs and qdata to support identifiers
qdata has reached the stage of needing parameterized arguments, but this is low priority now that we have data functionality within the UI itself.
2021-11-12 17:42:21 +00:00
CalDescent
8d44e07c32 Fixes issues relating to reading resources containing a single file 2021-11-12 17:37:33 +00:00
CalDescent
d99fae4340 Added support for single file patching
This ensures that the folder structures align when comparing before and after versions.
2021-11-12 17:37:02 +00:00
CalDescent
d49caa29ce Pass Service enum to TransactionRepository.getSignaturesMatchingCriteria() instead of an Integer.
This fixes a bug when no service was specified in the /arbitrary/search API.
2021-11-12 14:22:22 +00:00
CalDescent
8bebe11b4e Allow single files to be uploaded without compression
We may choose to save on CPU by not compressing individual files, so this allows the network to support that. However it is still using compression by default, to reduce file sizes.
2021-11-12 13:44:28 +00:00
CalDescent
236a456cae Added support for single file uploads.
This process could potentially be simplified if we were to modify the structure of the actual zipped data (on the writer side), but this approach is more of a "catch-all" (on the reader side) to support multiple different zip structures, giving us more flexibility. We can still choose to modify the written zip structure if we choose to, which would then cause most of this new code to be skipped.

Note: the filename of a single file is not currently retained; it is renamed to "data" as part of the packaging process. Need to decide if this is okay before we go live.
2021-11-12 13:35:50 +00:00
CalDescent
7bc745fa8e Added "THUMBNAIL" and "PLAYLIST" service types, and fixed a duplicate issue in earlier commit.
Thumbnails will be used in order to show logos/screenshots in the list of websites or other resources. Playlists will allow for media apps to group videos/audio/images into collections, e.g. albums.
2021-11-12 09:02:44 +00:00
CalDescent
056fc8fbaf Treat a blank identifier as null 2021-11-12 08:59:43 +00:00
CalDescent
b6aa507b41 Added "AUDIO" and "BLOG" service types.
BLOG_POST and BLOG_COMMENT are using values 777 and 778 as these were the values used in Qora.
2021-11-11 09:16:16 +00:00
CalDescent
4b1a5a5e14 Connected the rest of the system up to the recently added "identifier" feature. 2021-11-11 09:12:54 +00:00
CalDescent
a364206159 Added "IMAGE", "VIDEO" and "DOCUMENT" service types. 2021-11-07 18:44:05 +00:00
CalDescent
b5feb5f733 Fixed test which was failing due to an earlier commit 2021-11-07 18:41:52 +00:00
CalDescent
991125034e Added "identifier" property to arbitrary transactions
Until now we have been limited to one data resource per name/service combination. This meant that each name could only have a single website, git repo, image, video, etc, and adding another would overwrite the previous data. The identifier property now allows an optional string to be supplied with each resource, therefore allowing an unlimited amount of resources per name/service combination.

Some examples of what this will allow us to do:

- Create a video library app which holds multiple videos per name
- Same as above but for photos
- Store multiple images against each name, such as an avatar, website thumbnails, video thumbnails, etc. This will be necessary for many "system level" features.
- Attach multiple websites to each name. The default website (with blank/null identifier) would remain the entry point, but other websites could be hosted essentially as subdomains, and then linked from the default site. This also provides a means to go beyond the 500MB website size limit.

Not all of these features will exist initially, but having this identifier included in the protocol layer allows them to be added at any time.
2021-11-07 18:39:43 +00:00
CalDescent
a0fe1a85f1 Removed website publishing API since we now do everything using POST /arbitrary/{service}/{name} 2021-11-07 18:23:14 +00:00
CalDescent
3a2e68c334 Improved directory structure of the "reader" cache 2021-11-07 17:16:42 +00:00
CalDescent
b6418cd912 Revert "Added an index to help speed up recent queries"
This reverts commit e652038018.
2021-11-06 12:52:20 +00:00
CalDescent
e652038018 Added an index to help speed up recent queries 2021-11-06 12:47:32 +00:00
CalDescent
b2e2af51ed Added API endpoint to list all arbitrary resources, grouped by name and service
This is used by the UI to list available websites (and ultimately other categories of hosted data)
2021-11-06 10:41:30 +00:00
CalDescent
9502444bbc Prevent any kind of trading unless the blockchain is fully synced 2021-11-05 16:31:54 +00:00
CalDescent
a0fe803c35 Added POST /arbitrary/{service}/{name} API endpoint
At the moment this just redirects to PUT, but will ultimately choose PUT or PATCH based on the differences in the data supplied.
2021-11-05 14:46:22 +00:00
CalDescent
ea2ca37abe Improved format of error messages 2021-11-05 14:38:10 +00:00
CalDescent
0601ffbb34 Added /transactions/convert method to use to convert full transaction bytes to ones used for signing
This is needed in order for the UI to sign arbitrary transactions, although it will work for all transaction types
2021-11-05 14:37:47 +00:00
CalDescent
09a7fcaba4 Added MissingDataException
This is generated whenever a data resource cannot be built because it is missing data for at least one layer. Using a custom exception type here enables a few new features:

1. A single build process is now able to request missing data from all the layers that need it. Previously it would only request from the first missing layer and would then give up. This resulted in the user/application having to issue the build command multiple times rather than just once, until all layers had been requested.

2. GET /arbitrary/{service}/{name} will now block the response and retry in the background until the data arrives. This allows it to be used synchronously. Note: we'll need to add a timeout.

3. Loading a website via GET /site/{name} will avoid adding to the failed builds queue when a MissingDataException is thrown, which allows it to be quickly retried. The interface already auto refreshes, allowing the site to load as soon as it's available.
2021-11-04 09:09:54 +00:00
CalDescent
ce15784851 Return a detailed error message in GET /arbitrary/{service}/{name} 2021-11-03 21:55:55 +00:00
CalDescent
b861b2dffb Fixed loading bug when a transaction's data size is smaller than the chunk size. 2021-11-03 21:54:57 +00:00
CalDescent
e50fd786da Don't respond with a file list for a transaction that is outside of our storage policy, even if we do have a copy of the file at the time of the request. 2021-11-03 21:44:31 +00:00
CalDescent
5e82de667e Don't broadcast that we are holding files for a transaction unless it's within the scope of our storage policy.
We may need to temporarily hold files for the purpose of viewing, but restrictions need to be in place to avoid these being served to peers of stored for longer than they are needed.
2021-11-03 21:40:15 +00:00
CalDescent
d7ddcda9da Refactor to simplify some duplicated code 2021-11-03 21:39:02 +00:00
CalDescent
6d031130b9 Invalidate the cache in ArbitraryTransaction.onImportAsUnconfirmed() if we have the local data
This ensures that the local copy of a resource updates as soon as the transaction is broadcast.
2021-11-03 21:33:29 +00:00
CalDescent
a61b0685f0 Arbitrary data manager now only prefetches data according to the storage policy.
- If storage policy includes "FOLLOWING", only process transactions relating to the followed names.
- If storage policy is "ALL", process all transactions.
- If storage policy is "NONE" or "VIEWED", don't process or prefetch any data.
2021-11-03 20:24:38 +00:00
CalDescent
abfeafc823 Refactored to move all build-related code to a new ArbitraryDataBuildManager class
This is will be used to coordinate all build processes and threads. This way it keeps it separate from the ArbitraryDataManager class, which was getting a bit cluttered.
2021-11-03 19:56:52 +00:00
CalDescent
3a51be3430 ArbitraryDataBuildManager renamed to ArbitraryDataBuilderThread, as we will likely want to run multiple instances of this when we scale up. 2021-11-03 19:38:17 +00:00
CalDescent
3b914d4a7f Improved trade bot backups so that the current order being bought is included.
This should fix any key recovery issues if the node crashes or otherwise fails when buying an offer.
2021-11-03 19:27:56 +00:00
CalDescent
ede4802ceb Converted ArbitraryTestTransaction to version 5
This fixes a failed serialization test when Transaction.getVersionByTimestamp() returns 5
2021-11-03 19:19:27 +00:00
CalDescent
fe79119809 Added PresenceTestTransaction, to allow SerializationTests.testTransactions() to be unblocked 2021-11-03 19:18:26 +00:00
Scare Crowe
319d96f94e Add CWD bootstrap node 2021-11-03 11:45:47 +05:00
CalDescent
6f07dc7852 Always overwrite existing data when building via the queue.
This fixes a significant bug that was interfering with updates.
2021-11-02 19:34:25 +00:00
CalDescent
16bcba6e2e When accessing a website or other data resource, request the chunks if we don't already have them.
This causes the build to fail on the first pass due to missing chunks, however it now fails with a message indicating that it should be retried shortly. The website loader is already set up in such a way that it will be automatically retried, during which time the loading screen is shown.

Also added code to remove the resource from the "failed builds list" once the chunks arrive, so that it is able to be rebuilt sooner than the FAILURE_TIMEOUT (currently 5 minutes).
2021-11-02 19:32:48 +00:00
CalDescent
1002acb021 Fixed error in log entry. 2021-11-02 09:15:55 +00:00
CalDescent
b771544c5d Added test to check website/data updates. 2021-11-02 09:09:54 +00:00
CalDescent
8c7f09c454 Fixed yet another case sensitivity issue. 2021-11-01 20:02:38 +00:00
CalDescent
618cffefb1 Made names case insensitive when using them as a search filter. 2021-11-01 19:44:43 +00:00
CalDescent
8fd37e857e Fixed case sensitivity issue when building from past transactions. 2021-11-01 19:41:50 +00:00
CalDescent
8218bfd24b Initial implementation of storage policies.
- Don't attempt to fetch data for transactions which fall outside of the storage policy
- Delete files relating to transactions that are no longer within the scope of the storage policy

Note: some additional work needs to be done to ensure that viewed files are deleted when using a storage policy that excludes "VIEWED" content.
2021-10-31 22:38:30 +00:00
CalDescent
cbb2dbffb9 The /arbitrary/search API endpoint now uses a string instead of an int for the "service", and shows a dropdown of possible values in the API documentation page. 2021-10-31 21:22:51 +00:00
CalDescent
528a838643 Fixed bug in previous commit 2021-10-31 21:20:59 +00:00
CalDescent
cbed6418e7 Added ability to filter arbitrary transactions by name when searching. 2021-10-31 21:07:14 +00:00
CalDescent
4882cc92a8 StoragePolicy enum moved to new ArbitraryDataStorageManager class 2021-10-31 20:32:32 +00:00
CalDescent
28fb11068e Added "storagePolicy" setting, including startup validation.
Possible values are:

FOLLOWED_AND_VIEWED (default)
FOLLOWED
VIEWED
ALL
NONE

Nothing is affected by this setting yet, but this will be added shortly.
2021-10-31 19:15:26 +00:00
CalDescent
394ced9fb9 "Lists" feature is now generic.
This means that no additional structural code is required to add new lists. The only non-generic aspect are the API endpoints - it's best to keep these specific until we have a need for user-created lists.
2021-10-31 18:45:40 +00:00
CalDescent
90465149e6 Added API endpoints to allow registered names to be followed/unfollowed.
These will ultimately be used by the UI to control which data the node should host.
2021-10-31 18:42:18 +00:00
CalDescent
c6d868d981 "Lists" feature is now generic.
This means that no additional structural code is required to add new lists. The only non-generic aspect are the API endpoints - it's best to keep these specific until we have a need for user-created lists.
2021-10-31 18:39:58 +00:00
CalDescent
bada4fd140 Use a buffered digest where possible instead of reading the whole file contents into memory. 2021-10-30 18:21:06 +01:00
CalDescent
60f96d15bd When specifying a domain without a subdomain, add a www. version automatically.
Longer term we will probably need a 301 redirect in these cases for SEO purposes, but this is a nice convenience feature for now.
2021-10-30 17:47:57 +01:00
CalDescent
0328007345 Domain mapping now uses registered name instead of transaction signature.
There's no real need to maintain support for signature mapping anymore. Using this new method means that the latest version of a site is always served via the traditional domain name, whereas using transaction signatures caused older versions to be shown.

Example settings.json configuration:

"domainMapServiceEnabled": true,
"domainMapServicePort": 80,
"domainMap": [
  {
    "domain": "webdemo.qortal.uk",
    "name": "QortalDemo"
  },
 {
    "domain": "www.reqorder.org",
    "name": "ReQorder"
  }
]
2021-10-30 17:29:23 +01:00
CalDescent
3ad0e92a0f Log empty responses in qdata 2021-10-29 18:58:59 +01:00
CalDescent
3934120541 Return the data directly in GET /arbitrary/{service}/{name}, instead of a path to the data 2021-10-29 18:52:05 +01:00
CalDescent
24ca126f5a Various qdata improvements 2021-10-29 17:58:47 +01:00
CalDescent
651ca71126 Added transaction validity log. 2021-10-29 17:50:03 +01:00
CalDescent
e7cb33d8e2 Synchronize peer data lookups.
Without this we could broadcast the same data multiple times, due to more than one thread processing the same message from different peers.
2021-10-29 17:46:58 +01:00
CalDescent
c63d238316 Log the error response in the qdata utility if broadcasting a transaction fails. 2021-10-29 17:26:40 +01:00
CalDescent
dcdc48d917 Private key moved from command line argument to QORTAL_PRIVKEY env variable, to improve security. 2021-10-29 17:24:22 +01:00
CalDescent
f4c1671079 Removed the need to include public key in recently added API endpoints, as it can be derived from the registered name's owner. 2021-10-29 17:08:26 +01:00
CalDescent
7aa2fbee1c Added "qdata" command line tool to host and retrieve data 2021-10-29 16:51:41 +01:00
CalDescent
f1939fdc2b Added generic PUT, PATCH, and GET /arbitrary API endpoints
These will likely avoid the need for some of the /site APIs.
2021-10-29 16:45:08 +01:00
CalDescent
c9356d0ff5 Re-broadcast the arbitrary signatures message if it contains new data, so that the message finds its way to all online peers. 2021-10-29 16:16:58 +01:00
CalDescent
6b5d938a40 Added saveChanges() missing from previous commit, and a discardChanges() just in case. 2021-10-29 16:16:05 +01:00
CalDescent
d82da160f3 Added DHT-style lookup table to track file locations
This maps ARBITRARY transactions to peer addresses, but also includes additional metadata/stats to track the success rate and reachability.

Once a node receives files for a transaction, it broadcasts this info to its peers so they can update their records.

TLDR: this allows us to locate peers that are hosting a copy of the file we need.
2021-10-29 13:35:17 +01:00
CalDescent
54c8aac20d Commented out unused forwarding code 2021-10-29 13:27:37 +01:00
CalDescent
314b6fc2f8 Include the initial peers when creating bootstraps 2021-10-27 08:46:52 +01:00
CalDescent
974df031a0 Added another bootstrap host 2021-10-26 21:41:22 +01:00
CalDescent
36d0292c6b Added "sleep until message" functionality to LTC ACCTv2. 2021-10-26 20:10:05 +01:00
CalDescent
7c16952c92 Added LitecoinACCTv2 and LitecoinACCTv2TradeBot 2021-10-26 19:56:33 +01:00
CalDescent
557807e3ba Initial attempt at adding "sleep until message" functionality to DOGE ACCTv2. 2021-10-26 10:59:23 +01:00
CalDescent
c1d5b2df29 Added DogecoinACCTv2 and DogecoinACCTv2TradeBot 2021-10-26 10:59:16 +01:00
CalDescent
05be5c1199 Version set to 2.0.0, given that this will ultimately be merged into the main qortal core. It makes sense to keep the version numbers in sync. 2021-10-25 19:06:48 +01:00
CalDescent
f19a65148a Revert "Default minPeerVersion set to 0.1.0. TODO: revert this if ever merged into the main repo."
This reverts commit 9b4d832d17.
2021-10-25 19:04:31 +01:00
CalDescent
a55fc4fff9 When validating an ARBITRARY transaction, ensure that the supplied name exists and is registered to the account that is signing the transaction.
This ensures that only the owner of a name is able to update data associated with that name.

Note that this doesn't take into account the ability for group members to update a resource, so this will need modifying when that feature is ultimately introduced (likely after v3.0)
2021-10-25 18:58:33 +01:00
CalDescent
35a7a70b93 Merge remote-tracking branch 'qortal/master' 2021-10-25 18:26:06 +01:00
CalDescent
3e0574e563 Added another missing block archive lookup relating to trade timestamps.
Note that this is unlikely to be the cause of some of the zero timestamps issue seen on a subset of nodes - there is still likely to be another problem that needs fixing.
2021-10-25 18:21:40 +01:00
CalDescent
69e557e70d Delete .sha256 file which was left lying around after running the bootstrap unit tests. 2021-10-25 18:20:58 +01:00
CalDescent
1b846be5fc Fixed bug in arbitrary transaction builder 2021-10-24 22:43:58 +01:00
CalDescent
707eb58068 Added testPatchBeforePut() unit test 2021-10-24 22:43:28 +01:00
CalDescent
8630f3be96 Added first end-to-end test of data storage 2021-10-24 19:08:09 +01:00
CalDescent
c90aeba286 Return ArbitraryTransactionData instead of a byte array, as it is more useful if the bytes are transformed separately. 2021-10-24 19:06:30 +01:00
CalDescent
5055cfc6cb Removed unused imports 2021-10-24 17:43:28 +01:00
CalDescent
c222c4eb29 Updated expected hash of demo data as it has been updated. 2021-10-24 17:41:51 +01:00
CalDescent
6c01955561 Refactor to move arbitrary transaction building to its own class. 2021-10-24 17:41:28 +01:00
CalDescent
305e0f1772 Disable validation of previous hash unless validateAllDataLayers is true
We may not need to validate this at all now that we have the ability to validate the current layer, but I'll leave it as it could be useful for debugging. It is disabled by default so not an issue.
2021-10-24 15:35:51 +01:00
CalDescent
52a94e3256 Added "validateAllDataLayers" setting (default false)
When true, the hashes of every layer are validated when building a data resource. When false, only the final layer's hash is validated.
2021-10-24 14:37:29 +01:00
CalDescent
a418fb18b6 Hash the current state when creating a patch
This is included in the patch metadata and then validated every time it is rebuilt.
2021-10-24 13:00:21 +01:00
CalDescent
9cd579d3db Another typo 2021-10-24 12:20:49 +01:00
CalDescent
e1a6ba7377 Fixed incorrect comment. 2021-10-24 12:03:22 +01:00
CalDescent
04aabe0921 Include the original file instead of a patch if the patch is larger than the original file.
This saves processing and disk space, as there is no point in applying a patch when the original file is smaller and can be included in its entirety.
2021-10-24 12:02:09 +01:00
CalDescent
8dd4d71d75 Significant rework of patches
- The "diff type" is now specified per file, allowing for different diff methods in each modified file.
- Patches will only be created when both the before and after files are less than 100kiB in size.
- Patches are validated after creation, and if invalid it will fall back to including the entire file.

This has identified a bug where patching fails for files without trailing newline characters, which still needs to be fixed. Until then, it will fall back to including the entire file in these cases.
2021-10-24 10:47:47 +01:00
QuickMythril
49dd63af1e updated BTC electrum servers 2021-10-23 00:46:02 -04:00
CalDescent
18c6f0ccc3 Merge pull request #60 from Tocoolmh58/master 2021-10-22 18:05:38 +01:00
QuickMythril
55c50a4b5b add API option to return inverse price (#61) 2021-10-22 18:04:53 +01:00
CalDescent
12b3267d5c Added arbitrary data merge tests. 2021-10-22 11:49:15 +01:00
CalDescent
d6d564c027 Fixed refresh interval of loading screen 2021-10-17 16:55:32 +01:00
CalDescent
1fbd5f7922 Fix for issue causing tradeTimestamp to report as 0 for trades in archived blocks. 2021-10-17 09:52:59 +01:00
CalDescent
f0e13fa492 Arbitrary transaction names are now case insensitive 2021-10-15 13:58:27 +01:00
CalDescent
c8d5ac9248 Fixed bug in ArbitraryTransactionTransformer.getDataLength() when missing a name. 2021-10-15 11:32:08 +01:00
CalDescent
aa4f77d4de Fixed merge issues relating to database updates. Existing data nodes will need to delete their db folder and resync. 2021-10-15 09:13:15 +01:00
CalDescent
f3ef112297 Merge remote-tracking branch 'qortal/master'
# Conflicts:
#	.gitignore
#	pom.xml
#	src/main/java/org/qortal/controller/Controller.java
#	src/main/java/org/qortal/gui/SysTray.java
#	src/main/java/org/qortal/settings/Settings.java
#	src/main/resources/i18n/ApiError_en.properties
#	src/test/java/org/qortal/test/CryptoTests.java
#	src/test/resources/test-settings-v2.json
2021-10-15 09:03:28 +01:00
CalDescent
bbb71083ef Updated AdvancedInstaller project for v2.0.0 2021-10-13 19:11:42 +01:00
CalDescent
e2134d76ec Bump version to 2.0.0 2021-10-13 18:16:50 +01:00
CalDescent
651372cd64 Bump version to 2.0.0-beta.7 2021-10-12 18:56:58 +01:00
CalDescent
581fe17b58 Added message to check the internet connection if the download cannot start. 2021-10-12 08:08:48 +01:00
CalDescent
af8608f302 Show full stack trace when bootstrapping fails for any reason. 2021-10-12 08:08:05 +01:00
CalDescent
290a19b6c6 Log the URL when downloading a bootstrap, to help with problem solving. 2021-10-12 08:01:47 +01:00
CalDescent
73eaa93be8 Added missing space in log entry. 2021-10-11 23:00:59 +01:00
CalDescent
7ab17383a6 Fix for NPE when serialized block bytes are unavailable. 2021-10-10 13:38:10 +01:00
CalDescent
b103c5b13f Bump version to 2.0.0-beta.6 2021-10-09 17:46:20 +01:00
CalDescent
b7d8a83017 Log "Downloading bootstrap..." as well as showing it in the splash screen. 2021-10-09 17:46:08 +01:00
CalDescent
e7bf4f455d Added missing repository.saveChanges() when reimporting data after creating a bootstrap. 2021-10-09 16:57:53 +01:00
CalDescent
a7f212c4f2 Create a .sha256 file to accompany each bootstrap
This can ultimately be validated after download, and can also be used to help coordinate updates on the various bootstrap hosts.
2021-10-09 16:57:19 +01:00
CalDescent
eb991c6026 Fixed issue causing bootstrap validation to be ignored before creation. 2021-10-09 16:29:40 +01:00
CalDescent
a78af8f248 Added SHA-256 file digest utility methods.
These read the file in small chunks, to reduce memory.
2021-10-09 16:22:21 +01:00
CalDescent
f34bdf0f58 Fixed issue causing minting accounts to be lost in some cases when auto bootstrapping. 2021-10-09 14:31:13 +01:00
CalDescent
ba272253a5 Bump version to 2.0.0-beta.5 2021-10-09 13:03:58 +01:00
CalDescent
9f488b7b77 Sleep for 5s before cleaning up temp path, in case this improves reliability on Windows. 2021-10-09 13:03:32 +01:00
CalDescent
3fb7df18a0 Delete temp directories at the beginning of the bootstrap process too, as Windows doesn't like deleting it at the end of the process. 2021-10-09 13:02:47 +01:00
CalDescent
00401080e0 Simplified cleanup process. Individual deletions aren't needed as they are all inside the main temp directory. 2021-10-09 13:02:00 +01:00
CalDescent
b265dc3bfb Don't log the complete stack trace for exceptions generated by bootstrap.checkRepositoryState(). The error message is enough in these cases. 2021-10-09 11:47:49 +01:00
CalDescent
63cabbe960 Log the full exception details and stack trace when creating bootstraps. 2021-10-09 11:39:08 +01:00
CalDescent
f6c1a7e6db Disregard exceptions in the bootstrap creation cleanup process because these don't affect the created bootstrap - instead just log the exception and full stack trace. 2021-10-09 11:38:13 +01:00
CalDescent
a3dcacade9 Now showing errors directly in the POST /bootstrap/create API response.
This avoids needing to check the log file each time.
2021-10-09 11:02:21 +01:00
CalDescent
17e65e422c Bump version to 2.0.0-beta.4 2021-10-08 19:11:25 +01:00
CalDescent
f53e2ffa47 Add initial peers on node startup if we don't have any in the repository.
This will be needed for future bootstraps, which don't contain any peers. It is also useful for those who have used the DELETE /peers/known API.
2021-10-08 19:10:02 +01:00
CalDescent
a1e4047695 Rework of bootstrap finalization process. 2021-10-08 18:06:41 +01:00
CalDescent
47ce884bbe Delete all known peers when creating a bootstrap 2021-10-08 15:24:10 +01:00
CalDescent
1b17c2613d Show "full node" or "top-only" in the "Downloading bootstrap" message. 2021-10-08 13:12:47 +01:00
CalDescent
dedc8d89c7 Handle case when attempting to load a block from the archive by reference, but the referenced block is in the main block repository, not the archive. This is the case with the genesis block.
Should fix issue where no block summaries were returned when syncing from block 1
2021-10-08 12:51:02 +01:00
CalDescent
d00fce86d2 Treat the genesis block as unpruned, as we leave this in the HSQLDB repository. 2021-10-08 12:42:23 +01:00
CalDescent
abab2d1cde Fixed issue preventing blocks from being served from the archive.
Now prefixing the byte buffer with the block height to mimic a cached block message.
2021-10-08 12:22:21 +01:00
CalDescent
33b715eb4e Merge branch 'networking' into v2.0-beta
# Conflicts:
#	src/main/java/org/qortal/settings/Settings.java
2021-10-07 18:53:49 +01:00
CalDescent
f6effbb6bb Removed unnecessary repository parameter from PruneManager.isBlockPruned() 2021-10-07 18:51:52 +01:00
CalDescent
dff9ec0704 Don't attempt to cache blocks from the archive, as they will never be recent 2021-10-07 18:50:59 +01:00
CalDescent
bfaf4c58e4 Make sure to check the archive when serving block summaries and signatures 2021-10-07 18:50:25 +01:00
CalDescent
ab7d24b637 Updated status wording 2021-10-07 09:02:28 +01:00
CalDescent
c256dae736 Ensure that the temp directory is always in the parent directory of the db folder. 2021-10-07 09:02:13 +01:00
CalDescent
5a55ef64c4 Bump version to 2.0.0-beta.3 2021-10-06 19:51:33 +01:00
CalDescent
045026431b Create a cleaner base directory path, without the "/./" 2021-10-06 19:50:32 +01:00
CalDescent
4dff91a0e5 Initial bootstrap import retry interval reduced from 5 minutes to 1 minute 2021-10-06 19:45:18 +01:00
CalDescent
7105872a37 Improved exception message 2021-10-06 19:44:30 +01:00
CalDescent
179bd8e018 Moved repository reopen to the finally {} block, so that we're never left without a repository instance. Should fix occasional "No repository available" error seen when retrying. 2021-10-06 19:44:04 +01:00
CalDescent
c82293342f Show full exception stack trace when a bootstrap import fails 2021-10-06 19:32:49 +01:00
CalDescent
81bf79e9d3 Bump version to 2.0.0-beta.2 2021-10-06 18:23:51 +01:00
CalDescent
8d6dffb3ff Added test for bootstrap random host selection. 2021-10-06 18:23:17 +01:00
CalDescent
2f6a8f793b Invert the colours in the splash screen 2021-10-06 18:22:52 +01:00
CalDescent
9bcd0bbfac Reduce log spam 2021-10-06 18:22:38 +01:00
CalDescent
cd359de7eb Scheduled maintenance now enabled by default, but uses a min and a max, to reduce the chances of multiple nodes running maintenance at the same time. Default to min: 7 days, max: 30 days. 2021-10-06 18:22:31 +01:00
Tocoolmh58
000f9ed459 Update ApiError_de.properties 2021-10-06 17:23:16 +02:00
Tocoolmh58
c5b2c0b4ec Create SysTray_de.properties 2021-10-06 17:01:52 +02:00
CalDescent
b7e9af100a Added scheduled repository maintenance feature. Currently disabled by default. 2021-10-06 08:52:27 +01:00
CalDescent
0d6409098f Added another bootstrap host 2021-10-05 22:08:18 +01:00
CalDescent
e07238ded8 Fixed variable name 2021-10-04 22:52:47 +01:00
CalDescent
27903f278d Add tmp folder to gitignore 2021-10-04 22:45:05 +01:00
CalDescent
ddf966d08c Show progress status when extracting files 2021-10-04 22:44:51 +01:00
CalDescent
65dca36ae1 Show progress status when downloading a bootstrap 2021-10-04 22:38:58 +01:00
CalDescent
289dae0780 Fixed issue causing the local repository data backup to be overwritten with an empty list. 2021-10-04 09:28:16 +01:00
CalDescent
71f802ef35 Exponentially backoff when bootstrapping fails, to reduce bandwidth
The retry interval starts at 5 minutes and doubles with each failure.
2021-10-04 09:25:23 +01:00
CalDescent
0135f25b9d Delete existing repository before extracting bootstrap
This limits the amount of additional space needed to the size of the compressed bootstrap (currently just under 4GB for full nodes, or 200MB for top-only nodes).
2021-10-04 09:15:54 +01:00
CalDescent
de3ebf664f Fixed issue with format specifier 2021-10-04 09:11:11 +01:00
CalDescent
850d879726 Use a "tmp" folder in the Qortal directory rather than a system generated temp folder.
This avoids the need to move files between partitions, and we also can't assume that the system partition has enough space to do the extraction.
2021-10-04 09:10:56 +01:00
CalDescent
5397e6c723 Bump version to 2.0.0-beta.1 2021-10-03 22:59:11 +01:00
CalDescent
889f6fc5fc Add a "testnet-" prefix in filenames when creating or importing bootstraps on testnet, so that the two databases can be kept separate. 2021-10-03 22:57:38 +01:00
CalDescent
41c2ed7c67 Fixed out of memory errors when copying AT states. 2021-10-03 22:51:15 +01:00
CalDescent
cdf47d4719 Reduce log spam. 2021-10-03 22:33:36 +01:00
CalDescent
210368bea0 Bump version to 2.0.0-beta.0 2021-10-03 19:43:28 +01:00
CalDescent
4f48751d0b Fixed issue caused when trying to update the splash frame status in a headless environment. 2021-10-03 19:43:10 +01:00
CalDescent
b6d3e82304 Update status when performing repository maintenance 2021-10-03 19:31:05 +01:00
CalDescent
3bb3528aa5 Merge branch 'master' into bootstrap 2021-10-03 18:44:13 +01:00
CalDescent
4f892835b8 Show maximum time estimations in archiving and pruning statuses 2021-10-03 18:41:47 +01:00
CalDescent
ac49221639 Show warning status on startup if the database is missing the AtStatesHeightIndex. 2021-10-03 18:34:21 +01:00
CalDescent
75ed5db3e4 Test multiple files when bulk archiving. 2021-10-03 17:33:02 +01:00
CalDescent
59c8e4e6a2 Fixed bug in earlier commit 2021-10-03 16:09:52 +01:00
CalDescent
52b322b756 Take a backup of local data before overwriting with a bootstrap.
Also moved the import phase to after the validation phase, so that the data returns after the bootstrap.
2021-10-03 16:09:40 +01:00
CalDescent
dc876d9c96 Force a bootstrap if the block archive isn't intact on launch
This allows the topOnly setting to be disabled and the node will automatically bootstrap to the archive version. A rebuild isn't attempted if bootstrapping is disabled, in order to reduce risk.
2021-10-03 16:00:11 +01:00
CalDescent
5b028428c4 Checkpoint immediately after starting/upgrading the repository
This should fix a longstanding issue where quitting the core before the first checkpoint (1-2 hours after first launch) causes the database to become corrupt.
2021-10-03 15:47:10 +01:00
CalDescent
f67a0469fc SplashFrame styling improvements 2021-10-03 15:33:53 +01:00
CalDescent
494cd0efff Added white background to splash frame - I think it looks nicer this way, and it may solve the X2Go issues too. 2021-10-03 15:27:19 +01:00
CalDescent
fc8e38e862 Show version number in the "Starting Qortal Core" status. 2021-10-03 15:22:21 +01:00
CalDescent
f09fb5a209 Run the bulk block archiver any time that it isn't reported to be up to date. This should allow it to re-run if the user quits the core before it completes. 2021-10-03 15:17:31 +01:00
CalDescent
b00c1c1575 Update splash frame statuses when reshaping, pruning, or building the block archive 2021-10-03 15:16:36 +01:00
CalDescent
7e5dd62a92 Show bootstrap statuses on splash frame. 2021-10-03 15:01:50 +01:00
CalDescent
35718f6215 Fixed issue with bootstrap retries. 2021-10-03 15:00:40 +01:00
CalDescent
a6d3891a95 Fixed bugs when importing bootstrap. 2021-10-03 09:48:14 +01:00
CalDescent
9591c4eb58 Added support for creating top-only bootstraps 2021-10-02 19:13:18 +01:00
CalDescent
8aaf720b0b Force archiveEnabled to false if we're in top only mode.
Most of the code handles this case anyway, but it's an easy place for bugs to be created. So it's safest to enforce it at the settings level.
2021-10-02 19:11:47 +01:00
CalDescent
63a35c97bc Fixed bug in path returned to POST /bootstrap/create API. 2021-10-02 19:10:23 +01:00
CalDescent
8eddaa3fac Small refactor to update wording. 2021-10-02 15:48:31 +01:00
CalDescent
1b3f37eb78 Delete the "archive" folder when in top-only mode
This allows a block archive node to be switched into top-only mode.
2021-10-02 15:37:06 +01:00
CalDescent
1f8fbfaa24 Missed a few topOnly references from the last commit. 2021-10-02 15:34:55 +01:00
CalDescent
ea92ccb4c1 "pruningEnabled" setting renamed to "topOnly"
Pruning is still a concept in the code, but since it relates to both archived and topOnly mode, it makes it cleaner to use "topOnly" to refer to the pruned db with no archive.
2021-10-02 15:13:23 +01:00
CalDescent
d25a77b633 Ignore bootstraps and other 7z archives in git. 2021-10-02 15:04:00 +01:00
CalDescent
51bb776e56 Select a random host when importing a bootstrap, and started adding support for multiple bootstrap types. 2021-10-02 15:03:31 +01:00
CalDescent
47b1b6daba Retry the entire bootstrap import process on failure, rather than just the download. 2021-10-02 14:43:26 +01:00
CalDescent
adeb654248 Rework of repository maintenance and backups
It will now attempt to wait until there are no other active transactions before starting, to avoid deadlocks. A timeout for this process is specified - generally 60 seconds - so that callers can give up or retry if something is holding a transaction open for too long. Right now we will give up in all places except for bootstrap creation, where it will keep retrying until successful.
2021-10-02 13:23:26 +01:00
CalDescent
c4d7335fdd Fixed more issues that could cause transactions to be held open. 2021-10-02 13:05:16 +01:00
CalDescent
ca7f42c409 Reduced unnecessary database queries in the block archiver. 2021-10-02 11:52:20 +01:00
CalDescent
ca02cd72ae Fixed issue in block archiver, which caused it to hold a transaction open for a very long time. This caused deadlocks when trying to create bootstraps or perform repository maintenance. 2021-10-02 11:51:53 +01:00
CalDescent
1ba542eb50 Simplified minting code in block archive tests. 2021-10-01 14:51:45 +01:00
CalDescent
53cd967541 Added defrag (repository maintenance) tests. 2021-10-01 14:51:28 +01:00
CalDescent
49749a0bc7 Added more precise checking of database states to the bulk pruning test. This highlighted a major bug in the bulk prune process whereby the recent AT states weren't being retained. 2021-10-01 11:03:56 +01:00
CalDescent
446f924380 Added bulk pruning test, which highlighted some bugs in both bulk and regular pruning. 2021-10-01 09:32:35 +01:00
CalDescent
5b231170cd Fixed small issues with block archive tests. 2021-10-01 08:10:28 +01:00
CalDescent
7375357b11 Added bootstrap tests
This involved adding a feature to the test suite in include the option of using a repository located on disk rather than in memory. Also moved the bootstrap compression/extraction working directories to temporary folders.
2021-10-01 07:44:33 +01:00
CalDescent
347d799d85 Reduce log spam. 2021-09-28 20:30:06 +01:00
CalDescent
0d17f02191 Pass a repository instance into the bulk archiving and pruning methods.
This is a better approach than opening a new session for each, and it makes it easier to write unit tests.
2021-09-28 20:29:53 +01:00
CalDescent
ce5bc80347 Increased threshold of BlockArchiveWriter.isArchiverUpToDate() from 90 to 95%.
In practice, the reading from a correctly archived chain with 550k blocks is currently around 99.5%, but it will be lower if starting with a chain that isn't fully synced.
2021-09-28 20:21:19 +01:00
CalDescent
0a4479fe9e Initial implementation of automatic bootstrapping
Currently supports block archive (i.e. full) bootstraps only. Still need to add support for top-only bootstraps.
2021-09-28 20:17:19 +01:00
CalDescent
de8e96cd75 Added Blockchain.validateAllBlocks() to check every block back to genesis.
This is extremely slow and shouldn't be needed in normal use cases. It currently checks that each block references the one before, but can ultimately be expanded to check more information about each block and its derived data.
2021-09-28 09:28:47 +01:00
CalDescent
e2a62f88a6 Modified repository backup and recovery to allow a custom filename to be specified. 2021-09-28 09:26:18 +01:00
CalDescent
8926d2a73c Rework of import/export process.
- Adds support for minting accounts as well as trade bot states
- Includes automatic import of both types on node startup, and automatic export on node shutdown
- Retains legacy trade bot states in a separate "TradeBotStatesArchive.json" file, whilst keeping the current active ones in "TradeBotStates.json". This prevents states being re-imported after they have been removed, but still keeps a copy of the data in case a key is ever needed.
- Uses indentation in the JSON files for easier readability.
2021-09-28 09:24:21 +01:00
CalDescent
114833cf8e Fixed NPE when attempting to lookup a block signature that doesn't exist. 2021-09-27 09:13:19 +01:00
CalDescent
32227436e0 Updated AdvancedInstaller project for v1.7.0 2021-09-27 08:43:44 +01:00
CalDescent
28ff5636af Bump version to 1.7.0 2021-09-26 22:20:21 +01:00
CalDescent
656896d16f Fixed issue causing base block prune/trim heights to not be updated on the final pass. 2021-09-24 21:36:05 +01:00
CalDescent
19bf8afece Fixed bug in pruning phase on node startup
This was causing very recent AT states to be deleted accidentally, because we weren't rebuilding the LatestATStates table before running the query. We should add unit tests to cover this process in case there are any other undiscovered problems.
2021-09-24 20:52:45 +01:00
CalDescent
841b6c4ddf Fixed another issue causing ATStatesHeightIndex to go missing after pruning. 2021-09-24 15:58:09 +01:00
CalDescent
4c171df848 Disable archiving and pruning if the AtStatesHeightIndex is missing, and log it so that the user knows they should bootstrap or resync. 2021-09-24 11:14:18 +01:00
CalDescent
1f79d88840 Fixed errors found in unit tests. 2021-09-24 09:35:36 +01:00
CalDescent
6ee7e9d731 Merge branch 'block-archive' of github.com:Qortal/qortal into block-archive
# Conflicts:
#	src/main/java/org/qortal/controller/Controller.java
2021-09-24 08:50:35 +01:00
CalDescent
4856223838 Fixed error in rebase. 2021-09-24 08:50:00 +01:00
CalDescent
74ea2a847d Added unit tests for trimming, pruning, and archiving. 2021-09-24 08:37:36 +01:00
CalDescent
9813dde3d9 Added importFromArchive() feature
This allows archived blocks to be imported back into HSQLDB in order to make them SQL-compatible again.
2021-09-24 08:37:36 +01:00
CalDescent
fea7b62b9c Fixed some bugs found in unit testing. 2021-09-24 08:37:36 +01:00
CalDescent
37e03bf2bb Removed BLOCK_LIMIT_REACHED result from the block archive writer.
This wasn't needed, and is now instead caught by the NOT_ENOUGH_BLOCKS result.
2021-09-24 08:37:36 +01:00
CalDescent
5656de79a2 Removed maxDuplicatedBlocksWhenArchiving setting as it's no longer needed. 2021-09-24 08:37:36 +01:00
CalDescent
70c6048cc1 Added block archive mode
This takes all trimmed blocks (which should now be all but the last 1450 or so) and moves them into flat files. Each file contains the serialized bytes of as many blocks that can fit within the file size target of 100MiB.

As a result, the HSQLDB size drops to less than 1GB, making it much faster and easier to maintain. It also significantly reduces the total size of each full node, because the data is stored in a highly optimized way.

HSQLDB then works similarly to the way it does in pruning mode - it holds all transactions, the latest state of every AT, as well as the full AT states data and hashes for the past 1450 blocks.

Each archive file contains headers and indexes in order to quickly locate blocks. When a peer requests a block that is within the archive, the serialized bytes are sent directly without the need to go via a BlockData object. Now that there are no slow queries or data serialization processes needed, it should greatly speed up the block serving.

The /block API endpoints have been modified in such a way that they will also check and retrieve blocks from the archive when needed.

A lightweight "BlockArchive" table is needed in HSQLDB to map block heights to signatures minters and timestamps. It made more sense to keep SQL support for these basic attributes of each block. These are located in a separate table from the full blocks, in order to create a clear distinction between HSQLDB blocks and archived blocks, and also to speed up query times in the Blocks table, which is the one we are using 99% of the time.

There is currently a restriction on the /admin/orphan API endpoint to prevent orphaning beyond the threshold of the block archive.
2021-09-24 08:37:36 +01:00
CalDescent
87595fd704 Synchronized LatestATStates, to make rebuildLatestAtStates() thread safe. 2021-09-24 08:37:15 +01:00
CalDescent
dc030a42bb Moved trimming and pruning classes into a single package (org.qortal.controller.repository) 2021-09-24 08:37:15 +01:00
CalDescent
89283ed179 Increased atStatesPruneBatchSize from 10 to 25. 2021-09-24 08:36:52 +01:00
CalDescent
64e8a05a9f Prune ATStatesData as well as the ATStates when switching to pruning mode. 2021-09-24 08:36:52 +01:00
CalDescent
676320586a Updated tests to use the renamed method. 2021-09-24 08:36:52 +01:00
CalDescent
734fa51806 Unified the code to build the LatestATStates table, as it's now used by more than one class.
Note - the rebuildLatestAtStates() must never be used by two different classes at the same time, or AT states could be incorrectly deleted. It is okay at the moment as we don't run the AT states trimmer and pruner in the same app session. However we should probably synchronize this method so that we don't accidentally call it from two places in the future.
2021-09-24 08:36:52 +01:00
CalDescent
f056ecc8d8 Added bulk pruning phase on node startup the first time that pruning mode is enabled.
When switching from a full node to a pruning node, we need to delete most of the database contents. If we do this entirely as a background process, it is very slow and can interfere with syncing. However, if we take the approach of transferring only the necessary rows to a new table and then deleting the original table, this makes the process much faster. It was taking several days to delete the AT states in the background, but only a couple of minutes to copy them to a new table.

The trade off is that we have to go through a form of "reshape" when starting the app for the first time after enabling pruning mode. But given that this is an opt-in mode, I don't think it will be a problem.

Once the pruning is complete, it automatically performs a CHECKPOINT DEFRAG in order to shrink the database file size down to a fraction of what it was before.

From this point, the original background process will run, but can be dialled right down so not to interfere with syncing.
2021-09-24 08:36:52 +01:00
CalDescent
1a722c1517 Break out of the AT pruning inner loops if we're stopping the app. 2021-09-24 08:36:51 +01:00
CalDescent
44607ba6a4 Fixed NPE introduced in earlier commit. 2021-09-24 08:36:51 +01:00
CalDescent
01d66212da Updated AT states pruner as it previously relied on blocks being present in the db to make decisions. As a side effect, this now prunes ATs up the the pruneBlockLimit too, rather than keeping the last 35 days or so. Will review this later but I don't think we will need the missing ones. 2021-09-24 08:36:51 +01:00
CalDescent
925e10b19b Rework of Blockchain.validate() to account for pruning mode. 2021-09-24 08:36:51 +01:00
CalDescent
1b4c75a76e Prune all blocks up until the blockPruneLimit
By default, this leaves only the last 1450 blocks in the database. Only applies when pruning mode is enabled.
2021-09-24 08:36:51 +01:00
CalDescent
3400e36ac4 Started work on pruning mode (top-only-sync)
Initially just deleting old and unused AT states, to get this table under control. I have had to delete them individually as the table can't handle complex queries due to its size.

Nodes in pruning mode will be unable to serve older blocks to peers.
2021-09-24 08:36:51 +01:00
CalDescent
78e2ae4f36 Allow trade bots in the REFUNDING state to be deleted, if the user chooses to via the DELETE /crosschain/tradebot API endpoint. 2021-09-23 17:53:57 +01:00
CalDescent
957944f6a5 Revert "original design"
This reverts commit 8c325f3a8a.
2021-09-23 17:44:57 +01:00
CalDescent
9eab500e2c atStatesMaxLifetime reduced from 14 days to 5 days
Whilst we would ultimately like to drop these to 24 hours only, for now we need some headroom to allow for orphaning in the event of a problem. Orphaning currently fails if there is no ATStatesData available (which is the case for trimmed blocks). This could ultimately be solved by retaining older unique states, which is essentially what the sleeping AT feature will do.
2021-09-23 08:42:15 +01:00
CalDescent
573f4675a1 Reduced online account signatures min and max lifetimes
onlineAccountSignaturesMinLifetime reduced from 720 hours to 12 hours
onlineAccountSignaturesMaxLifetime reduced from 888 hours to 24 hours

These were using up too much space in the database and so it makes sense to trim them more aggressively (assuming testing goes well). We will now stop validating online account signatures after 12 hours, which should be more than enough confirmations, and we will discard them after 24 hours.

Note: this will create some complexity once some of the network is running this code. It could cause out-of-sync nodes on old versions to start treating blocks as invalid from updated peers. It's likely not worth the complexity of a hard fork though, given that almost all nodes will be synced to the chain tip and will therefore be unaffected. And even with a hard fork, we'd still face this problem on out of date nodes.
2021-09-23 08:39:24 +01:00
CalDescent
e6bde3e1f4 Minimum order size set to 0.01 LTC, to avoid dust errors. 2021-09-23 08:36:55 +01:00
CalDescent
5869174021 Combined the three invalid name registration block patches into a single class. This should allow syncing from genesis again. 2021-09-23 08:28:51 +01:00
CalDescent
449761b6ca Rework of "Names" integrity check
Problem:
The "Names" table (the latest state of each name) drifts out of sync with the name-related transaction history on a subset of nodes for some unknown and seemingly difficult to find reason.

Solution:
Treat the "Names" table as a cache that can be rebuilt at any time. It now works like this:
- On node startup, rebuild the entire Names table by replaying the transaction history of all registered names. Includes registrations, updates, buys and sells.
- Add a "pre-process" stage to block/transaction processing. If the block contains a name related transaction, rebuild the Names cache for any names referenced by these transactions before validating anything.

The existing "integrity check" has been modified to just check basic attributes based on the latest transaction for a name. It will log if there are any inconsistencies found, but won't correct anything. This adds confidence that the rebuild has worked correctly.

There are also multiple unit tests to ensure that the rebuilds are coping with various different scenarios.
2021-09-22 08:15:23 +01:00
CalDescent
39d5ce19e2 Removed unused import. 2021-09-19 20:24:12 +01:00
CalDescent
3b156bc5c9 Added database integrity check for registered names
This ensures that all name-related transactions have resulted in correct entries in the Names table. A bug in the code has resulted in some nodes having missing data in their Names table. If this process finds a missing name, it will log it and add the name.

Missing names are added, but ownership issues are only logged. The known bug wasn't related to ownership, so the logging is only to alert us to any issues that may arise in the future.

In hindsight, the code could be rewritten to store all three transaction types in a single list, but this current approach has had a lot of testing, so it is best to stick with it for now.
2021-09-19 20:23:59 +01:00
CalDescent
a4f5124b61 Delete signatures from the invalidBlockSignatures array if we haven't seen them in over 1 hour
This is necessary because it's possible (in theory) for a block to be considered invalid due to an internal failure such as an SQLException. This gives them more chances to be considered valid again. 1 hour is more than enough time for the node to find an alternate valid chain if there is one available.
2021-09-19 19:46:48 +01:00
CalDescent
47a34c2f54 Validate blocks in syncToPeerChain() before orphaning
This prevents a valid block candidate being discarded in favour of an invalid one. We can't actually validate a block before orphaning (because it will fail due to various reasons such as already existing transactions, an existing block with the same height, etc) so we will instead just check the signature against the list of known invalid blocks.
2021-09-19 17:33:04 +01:00
CalDescent
8a7446fb40 Added "apiKeyDisabled" setting to bypass API key / loopback checking for those who need it.
This should only be used if all of the following conditions are true:
a) Your node is private and not shared with others
b) Port 12391 (API port) isn't forwarded
c) You have granted access to specific IP addresses using the "apiWhitelist" setting

The node will warn on startup if this setting is used without a sensible access control whitelist.
2021-09-19 09:34:48 +01:00
CalDescent
705e7d1cf1 Test name.register() and name.unregister() 2021-09-18 13:28:44 +01:00
CalDescent
44a90b4e12 Keep track of invalid block signatures and avoid peers that return them
Until now, a high weight invalid block can cause other valid, lower weight alternatives to be discarded. The solution to this problem is to track invalid blocks and quickly avoid them once discovered. This gives other valid alternative blocks the opportunity to become part of a valid chain, where they would otherwise have been discarded.

As with the block minter update, this will cause a fork when the highest weight block candidate is invalid. But it is likely that the fork would be short lived, assuming that the majority of nodes pick the valid chain.
2021-09-18 10:58:05 +01:00
CalDescent
54e5a65cf0 Allow an alternative block to be minted if the chain stalls due to an invalid block
If it has been more than 10 minutes since receiving the last valid block, but we have had at least one invalid block since then, this is indicative of a stuck chain due to no valid block candidates. In this case, we want to allow the block minter to mint an alternative candidate so that the chain can continue.

This would create a fork at the point of the invalid block, in which two chains (valid an invalid) would diverge. The valid chain could never rejoin the invalid one, however it's likely that the invalid chain would be discarded in favour of the valid one shortly after, on the assumption that the majority of nodes would have picked the valid one.
2021-09-18 10:41:58 +01:00
CalDescent
06a2c380bd Updated and added some naming tests. 2021-09-17 09:34:10 +01:00
CalDescent
33ac1fed2a Revert "Treat a REGISTER_NAME transaction as an UPDATE_NAME if the creator matches."
This reverts commit b800fb5846.
2021-09-16 19:27:17 +01:00
CalDescent
cc65a7cd11 Fixed bug which prevented the "reduced name" from being updated in UPDATE_NAME transactions.
Updating a name was incorrectly leaving the existing "reduced name" intact. Thanks to Qortal user @MyBestBet for reporting this bug.
2021-09-14 20:38:20 +01:00
CalDescent
d600a54034 Modified name update tests to check the reduced name. 2021-09-14 20:34:42 +01:00
CalDescent
ba06225b01 Merge branch 'master' into block-archive 2021-09-12 10:17:11 +01:00
CalDescent
ce60ab8e00 Updated naming unit tests
- Use the "{\"age\":30}" data to make the tests more similar to some real world data.
- Added tests to ensure that registering and orphaning works as expected.
2021-09-12 10:16:07 +01:00
CalDescent
14f6fd19ef Added unit tests for trimming, pruning, and archiving. 2021-09-12 10:13:52 +01:00
CalDescent
1d8351f921 Added importFromArchive() feature
This allows archived blocks to be imported back into HSQLDB in order to make them SQL-compatible again.
2021-09-12 10:10:25 +01:00
CalDescent
6a55b052f5 Fixed some bugs found in unit testing. 2021-09-12 09:57:12 +01:00
CalDescent
2a36b83dea Removed BLOCK_LIMIT_REACHED result from the block archive writer.
This wasn't needed, and is now instead caught by the NOT_ENOUGH_BLOCKS result.
2021-09-12 09:55:49 +01:00
CalDescent
14acc4feb9 Removed maxDuplicatedBlocksWhenArchiving setting as it's no longer needed. 2021-09-12 09:52:28 +01:00
CalDescent
0657ca2969 atStatesMaxLifetime increased to 5 days
For now, we need some headroom to allow for orphaning in the event of a problem. Orphaning currently fails if there is no ATStatesData available (which is the case for trimmed blocks). This could ultimately be solved by retaining older unique states.
2021-09-09 17:46:19 +01:00
CalDescent
e90c3a78d1 Updated default "data" field text in the API documentation, to match the value the UI uses. 2021-09-09 15:12:28 +01:00
CalDescent
63c9bc5c1c Revert "Workaround for block 535658 problem"
This reverts commit 278201e87c.
2021-09-09 12:55:21 +01:00
CalDescent
a6bbc81962 Revert "Merge pull request #58 from QuickMythril/536140-fix"
This reverts commit 6d1f7b36a7, reversing
changes made to 6b74ef77e6.

# Conflicts:
#	src/main/java/org/qortal/block/Block536140.java
2021-09-09 12:55:08 +01:00
CalDescent
b800fb5846 Treat a REGISTER_NAME transaction as an UPDATE_NAME if the creator matches.
Whilst not ideal, this is necessary to prevent the chain from getting stuck on future blocks due to duplicate name registrations. See Block535658.java for full details on this problem - this is simply a "catch-all" implementation of that class in order to futureproof this fix.

There is still a database inconsistency to be solved, as some nodes are failing to add a registered name to their Names table the first time around, but this will take some time. Once fixed, this commit could potentially be reverted.

Also added unit tests for both scenarios (same and different creator).

TLDR: this allows all past and future invalid blocks caused by NAME_ALREADY_REGISTERED (by the same creator) to now be valid.
2021-09-09 12:54:01 +01:00
CalDescent
172a629da3 Added comments 2021-09-05 23:32:11 +01:00
CalDescent
6d1f7b36a7 Merge pull request #58 from QuickMythril/536140-fix
Block 536140 fix (same situation as block 535658)
2021-09-05 23:16:08 +01:00
QuickMythril
673ee4aeed Update Block.java 2021-09-05 18:07:11 -04:00
QuickMythril
25b787f6f2 Add files via upload 2021-09-05 18:06:32 -04:00
CalDescent
6b74ef77e6 Increased log level of invalid transaction message. 2021-09-05 21:25:38 +01:00
CalDescent
278201e87c Workaround for block 535658 problem 2021-09-05 21:24:02 +01:00
CalDescent
703cdfe174 Added block archive mode
This takes all trimmed blocks (which should now be all but the last 1450 or so) and moves them into flat files. Each file contains the serialized bytes of as many blocks that can fit within the file size target of 100MiB.

As a result, the HSQLDB size drops to less than 1GB, making it much faster and easier to maintain. It also significantly reduces the total size of each full node, because the data is stored in a highly optimized way.

HSQLDB then works similarly to the way it does in pruning mode - it holds all transactions, the latest state of every AT, as well as the full AT states data and hashes for the past 1450 blocks.

Each archive file contains headers and indexes in order to quickly locate blocks. When a peer requests a block that is within the archive, the serialized bytes are sent directly without the need to go via a BlockData object. Now that there are no slow queries or data serialization processes needed, it should greatly speed up the block serving.

The /block API endpoints have been modified in such a way that they will also check and retrieve blocks from the archive when needed.

A lightweight "BlockArchive" table is needed in HSQLDB to map block heights to signatures minters and timestamps. It made more sense to keep SQL support for these basic attributes of each block. These are located in a separate table from the full blocks, in order to create a clear distinction between HSQLDB blocks and archived blocks, and also to speed up query times in the Blocks table, which is the one we are using 99% of the time.

There is currently a restriction on the /admin/orphan API endpoint to prevent orphaning beyond the threshold of the block archive.
2021-09-04 19:40:51 +01:00
CalDescent
02988989ad Reduced online account signatures min and max lifetimes
onlineAccountSignaturesMinLifetime reduced from 720 hours to 12 hours
onlineAccountSignaturesMaxLifetime reduced from 888 hours to 24 hours

These were using up too much space in the database and so it makes sense to trim them more aggressively (assuming testing goes well). We will now stop validating online account signatures after 12 hours, which should be more than enough confirmations, and we will discard them after 24 hours.

Note: this will create some complexity once some of the network is running this code. It could cause out-of-sync nodes on old versions to start treating blocks as invalid from updated peers. It's likely not worth the complexity of a hard fork though, given that almost all nodes will be synced to the chain tip and will therefore be unaffected. And even with a hard fork, we'd still face this problem on out of date nodes.
2021-09-03 10:11:02 +01:00
CalDescent
25c17d3704 atStatesMaxLifetime reduced from 14 days to 24 hours 2021-09-03 10:04:04 +01:00
CalDescent
9b4d832d17 Default minPeerVersion set to 0.1.0. TODO: revert this if ever merged into the main repo. 2021-09-01 09:11:50 +01:00
CalDescent
52ab19dec6 Added method and name to the /site/upload endpoint params. 2021-09-01 09:11:03 +01:00
CalDescent
9973fe4326 Synchronized LatestATStates, to make rebuildLatestAtStates() thread safe. 2021-08-28 11:00:49 +01:00
CalDescent
2479f2d65d Moved trimming and pruning classes into a single package (org.qortal.controller.repository) 2021-08-27 09:45:56 +01:00
CalDescent
9056cb7026 Increased atStatesPruneBatchSize from 10 to 25. 2021-08-27 09:45:56 +01:00
CalDescent
cd9d9b31ef Prune ATStatesData as well as the ATStates when switching to pruning mode. 2021-08-27 09:45:56 +01:00
CalDescent
ff841c28e3 Updated tests to use the renamed method. 2021-08-27 09:45:56 +01:00
CalDescent
ca1379d9f8 Unified the code to build the LatestATStates table, as it's now used by more than one class.
Note - the rebuildLatestAtStates() must never be used by two different classes at the same time, or AT states could be incorrectly deleted. It is okay at the moment as we don't run the AT states trimmer and pruner in the same app session. However we should probably synchronize this method so that we don't accidentally call it from two places in the future.
2021-08-27 09:45:56 +01:00
CalDescent
5127f94423 Added bulk pruning phase on node startup the first time that pruning mode is enabled.
When switching from a full node to a pruning node, we need to delete most of the database contents. If we do this entirely as a background process, it is very slow and can interfere with syncing. However, if we take the approach of transferring only the necessary rows to a new table and then deleting the original table, this makes the process much faster. It was taking several days to delete the AT states in the background, but only a couple of minutes to copy them to a new table.

The trade off is that we have to go through a form of "reshape" when starting the app for the first time after enabling pruning mode. But given that this is an opt-in mode, I don't think it will be a problem.

Once the pruning is complete, it automatically performs a CHECKPOINT DEFRAG in order to shrink the database file size down to a fraction of what it was before.

From this point, the original background process will run, but can be dialled right down so not to interfere with syncing.
2021-08-27 09:45:56 +01:00
CalDescent
f5910ab950 Break out of the AT pruning inner loops if we're stopping the app. 2021-08-27 09:45:56 +01:00
CalDescent
22efaccd4a Fixed NPE introduced in earlier commit. 2021-08-27 09:45:56 +01:00
CalDescent
c8466a2e7a Updated AT states pruner as it previously relied on blocks being present in the db to make decisions. As a side effect, this now prunes ATs up the the pruneBlockLimit too, rather than keeping the last 35 days or so. Will review this later but I don't think we will need the missing ones. 2021-08-27 09:45:56 +01:00
CalDescent
209a9fa8c3 Rework of Blockchain.validate() to account for pruning mode. 2021-08-27 09:45:56 +01:00
CalDescent
bc1af12655 Prune all blocks up until the blockPruneLimit
By default, this leaves only the last 1450 blocks in the database. Only applies when pruning mode is enabled.
2021-08-27 09:45:55 +01:00
CalDescent
e7e4cb7579 Started work on pruning mode (top-only-sync)
Initially just deleting old and unused AT states, to get this table under control. I have had to delete them individually as the table can't handle complex queries due to its size.

Nodes in pruning mode will be unable to serve older blocks to peers.
2021-08-27 09:45:55 +01:00
CalDescent
1b39db664c Added missing ATStatesHeightIndex to the reshape code.
This was accidentally missed out of the original code. Some pre-updated nodes on the network will be missing this index, but we can use the upcoming "auto-bootstrap" feature to get those back.
2021-08-27 08:54:46 +01:00
CalDescent
7397b9fa87 Added more detail to exception message. 2021-08-21 08:35:12 +01:00
CalDescent
5bed5fb8fd Removed unnecessary code in ArbitraryResource.uploadFileAtPath()
This is now handled by ArbitraryDataWriter instead
2021-08-21 08:34:46 +01:00
CalDescent
fd795b4361 Don't attempt to cleanup the filesystem if a build is in progress.
This isn't essential but it helps to reduce unnecessary load and processing which would be better spent on building.
2021-08-20 20:19:51 +01:00
CalDescent
b2c0915a71 Removed accidentally duplicated code
This was causing two instances of the build manager to run.
2021-08-20 20:17:02 +01:00
CalDescent
095083bcfb Use lowercase directory names for consistency 2021-08-20 19:54:09 +01:00
CalDescent
4ba72f7eeb Regularly clean up old and unused files/folders in the temp directory
Also added code to purge built resource caches, but it is currently disabled. This will become more useful once we implement local storage limits.
2021-08-20 19:27:42 +01:00
CalDescent
6cb39795a9 Removed requirement to have connected peers in order to cleanup directories. 2021-08-20 17:00:53 +01:00
CalDescent
00ba16f536 Fixed incorrect ArbitraryTransactionTransformer layout. 2021-08-20 16:59:05 +01:00
CalDescent
988a839623 Improved response value of ArbitraryDataFile.deleteAllChunks() as it was inaccurate. 2021-08-20 16:58:43 +01:00
CalDescent
8fa61e628c Delete files related to transactions that have a more recent PUT
A PUT creates a new base layer meaning anything before that point is no longer needed. These files are now deleted automatically by the cleanup manager. This involved relocating a lot of the cleanup manager methods into a shared utility, so that they could be used by the arbitrary data manager. Without this, they would be fetched from the network again as soon as they were deleted.
2021-08-20 16:56:49 +01:00
CalDescent
8f3620e07b Fixed bug introduced in commit 51b1256 2021-08-20 13:44:30 +01:00
CalDescent
190f70f332 Removed unused, buggy code in HSQLDBArbitraryRepository.save()
It's safer to throw an exception and point the user towards ArbitraryDataWriter, rather than maintaining unused code.
2021-08-20 13:01:55 +01:00
CalDescent
6730683919 Added arbitrary data cleanup manager
This deletes redundant copies of data, and also converts complete files to chunks where needed. The idea being that nodes only hold chunks, since they currently are much more likely to serve a chunk to another peer than they are to serve a complete file.

It doesn't yet cleanup files that are unassociated with transactions, nor does it delete anything from the _temp folder.
2021-08-20 12:55:42 +01:00
CalDescent
51b12567e8 Switched ArbitraryDataFile.filePath from String to Path 2021-08-20 09:17:36 +01:00
CalDescent
d01cdeded8 Added pagination to ArbitraryDataManager
This improves scalability but isn't sufficient for a long term solution. TODO: It probably makes sense to add an additional query for recent transactions only, so that they are fetched quickly.
2021-08-20 07:58:55 +01:00
CalDescent
d22a03f1a5 Fixed misleading exception string. 2021-08-20 07:51:58 +01:00
CalDescent
ab0aeec434 Default method of serving websites switched from signature to name.
Before:
GET /site/:signature
GET /site/name/:name

After:
GET /site/signature/:signature
GET /site/:name
2021-08-20 07:51:27 +01:00
CalDescent
47ff51ce4e Use a random last reference on the very first transaction for an account
This is needed because we want to allow brand new accounts to publish data without a fee. A similar approach to CrossChainResource.buildAtMessage(). We already require PoW on all arbitrary transactions, so no additional logic beyond this should be needed.
2021-08-20 07:49:05 +01:00
CalDescent
1d62ef357d Check for null last references at the beginning of the process when creating an arbitrary tx. 2021-08-19 09:10:13 +01:00
CalDescent
59ef66f46d Handle shutdowns when zipping a large number of files. 2021-08-19 08:48:39 +01:00
CalDescent
77479215a6 Throw an exception if a patch file doesn't exist. 2021-08-18 20:32:47 +01:00
CalDescent
11da1f72b1 Added convenience method to make the code more readable. 2021-08-18 20:30:00 +01:00
CalDescent
6375b9d14d Fixed bug in getLatestTransaction() filtering. 2021-08-18 20:27:18 +01:00
CalDescent
42bc12f56d Call cleanupQueues() from the Controller 2021-08-18 16:25:47 +01:00
CalDescent
8515112811 Fixed unused variable issue 2021-08-18 16:24:37 +01:00
CalDescent
bedb87674b FAILURE_TIMEOUT set to 5 minutes 2021-08-18 16:24:13 +01:00
CalDescent
029c038a49 Catch runtime exceptions (e.g. IllegalStateException) when using the arbitrary data reader/writer. 2021-08-18 16:23:46 +01:00
CalDescent
95e905a5ae If a build fails, prevent any rebuilds for 5 minutes.
This prevents a resource with build problems from getting into a loop due to the browser requesting a rebuild as soon as it fails.
2021-08-18 16:04:57 +01:00
CalDescent
8a8ec32f2c Added java-diff-utils to pom.xml
This was accidentally missing from commit cb6fc46.
2021-08-18 09:59:56 +01:00
CalDescent
cd958398af Exclude all data* paths from gitignore. 2021-08-18 09:59:12 +01:00
CalDescent
2eedafd506 Log the error if a build fails. 2021-08-18 09:58:59 +01:00
CalDescent
9a88c0d579 Apply the unified-diff patch when combining layers. 2021-08-18 09:58:30 +01:00
CalDescent
cb6fc466d1 Create a "unified-diff" patch using java-diff-utils instead of including the entire modified file. I still need to test how well this works with binary files. 2021-08-18 09:57:44 +01:00
CalDescent
a6154cbb43 Don't allow a new layer to be created if it matches the existing state. 2021-08-18 07:51:50 +01:00
CalDescent
9c20967d24 Catch NPE seen a couple of times in Systray.setTrayIcon() 2021-08-18 07:51:26 +01:00
CalDescent
79bbadad2f Initial implementation of arbitrary data build queue
This adds the loadAsynchronously() method to ArbitraryDataReader, in addition to the existing loadSynchronously() method.

When requesting a website in a browser, previously the building of the resource's layers would be done synchronously in the API handler. This understandably caused many issues, so the building is now done asynchronously by a dedicated thread. A loading screen is shown in its place which auto refreshes every second until the build has completed.
2021-08-18 07:50:45 +01:00
CalDescent
c3b44cee94 Fixed bugs when computing directory digest. The order was being lost when adding to the second List, which was producing different hashes on different systems. It also doesn't make sense to convert paths to lowercase, given that we want our validation to be case sensitive. 2021-08-18 07:38:08 +01:00
CalDescent
1968496ce1 Invalidate the cache if hash validation fails, so that it can be rebuilt next time. 2021-08-17 09:30:27 +01:00
CalDescent
e1feb46de9 Put test data in a separate folder to real data. 2021-08-17 09:08:03 +01:00
CalDescent
f51a082049 Validate the previous state's hash each time a new layer is applied.
It's possible that this concept will struggle in the real world if operating systems, virus scanners, etc start interfering with our file stucture. Right now it is using a zero tolerance approach when checking the validity of each layer. We may choose to loosen this slightly if we encounter problems, e.g. by excluding hidden files. But for now it is best to be as strict as possible.
2021-08-17 09:07:46 +01:00
CalDescent
0f1927b4b1 Take a hash of the previous state's directory structure and file contents, and then include that hash in the patch metadata (when creating a new patch). This allows the integrity of the layers to be validated as each one is applied. 2021-08-17 08:43:08 +01:00
CalDescent
9baccc0784 Improved HTTP response generation when serving websites. 2021-08-16 22:40:55 +01:00
CalDescent
95c9cc7f99 Added ArbitraryDataCache
This decides whether to build a new state or use an existing cached state when serving a data resource. It will cache a built resource until a new transaction (i.e. layer) arrives. This drastically reduces load, and still allows for almost instant propagation of new layers.
2021-08-16 22:40:05 +01:00
CalDescent
0ed8e04233 Arbitrary data metadata classes moved to their own package. 2021-08-15 21:54:21 +01:00
CalDescent
b46c328811 Fixed bug in FilesystemUtils.copyAndReplaceDirectory() 2021-08-15 21:50:19 +01:00
CalDescent
c7d88ed95b Added "cache" file to the .qortal metadata folder.
This is used to store the transaction signature and build timestamp for each built data resource. It involved a refactor of the ArbitraryDataMetadata class to introduce a subclass for each file ("patch" and "cache"). This allows more files to be easily added later.
2021-08-15 21:49:45 +01:00
CalDescent
94da1a30dc When merging two states, validate that the transaction signature we are using for the "before" layer matches the "previous transaction signature" that is baked into the "after" layer.
This defends against a missing or out-of-order transaction. If this ever fails validation, we may need to rethink the way we are requesting transactions. But in theory this shouldn't happen, given that the "last reference" field of a transaction ensures that out-of-order transactions are invalid already.
2021-08-15 20:27:16 +01:00
CalDescent
219a5db60c Fixed circular bug in ArbitraryDataMetadata.getPreviousSignature() 2021-08-15 20:21:47 +01:00
CalDescent
a5cfedcae9 Exclude the .qortal metadata directory in all diffs. Also improved logging. 2021-08-15 20:02:14 +01:00
CalDescent
9b8a632f37 Include the .qortal folder in the merge output, since it will be needed for validation. We may choose to exclude it from the final output path, but for now it is left there to make debugging easier. 2021-08-15 19:55:56 +01:00
CalDescent
be0426d9a2 Improved logging 2021-08-15 19:19:55 +01:00
CalDescent
8fac0a02e5 Use the /.qortal/patch file to apply differences when merging together two layers, rather than walking through the file tree like we did before. 2021-08-15 18:49:30 +01:00
CalDescent
fa696a2901 Log exceptions when publishing data updates. 2021-08-15 18:37:27 +01:00
CalDescent
f5615b1c54 Don't exclude hidden files in the zips, as the .qortal folder needs to be included. 2021-08-15 17:45:37 +01:00
CalDescent
9850c294d1 Fixed various issues relating to syncing data for transactions without any chunks. 2021-08-15 11:28:26 +01:00
CalDescent
8929f32068 No longer creating the ".removed" files. 2021-08-15 10:43:00 +01:00
CalDescent
3019bb5c97 Enforce minBlockchainPeers in ArbitraryDataManager, as there is no point in trying to request data files when we don't have the minimum number of peers. 2021-08-15 10:39:00 +01:00
CalDescent
95044d27ce Fixed NPE caused by having an arbitrary transaction with no chunks (which is expected if the total data size is less than the chunk size). 2021-08-15 10:32:37 +01:00
CalDescent
16ac92b2ef Write patch metadata to a file inside a hidden ".qortal" folder which is included with each patch. This can be used in place of the existing ".removed" placeholder files to track removals. 2021-08-15 10:13:32 +01:00
CalDescent
f095964f7b Fixed edge case. 2021-08-14 18:28:22 +01:00
CalDescent
cfba793fcf Fixed bugs in ArbitraryDataFile, introduced when switching to the new temporary path. 2021-08-14 18:10:59 +01:00
CalDescent
c50a11e58a Cleanup intermediate paths in ArbitraryDataWriter. 2021-08-14 17:32:25 +01:00
CalDescent
a7282a5794 Renamed encrypted and zipped files. 2021-08-14 17:31:58 +01:00
CalDescent
efaf313422 When creating an ArbitraryDataFile from a path, make sure to move the file to the correct location within the data directory.
This bug was introduced now that the temp directory is contained within the data directory. Without this, it would leave it in the temp folder and then fail at a later stage.
2021-08-14 16:58:10 +01:00
CalDescent
d9f5753f58 Ensure parent directories exist when creating a diff. 2021-08-14 16:55:19 +01:00
CalDescent
179318f1d8 Ensure a patch exists after creating it. 2021-08-14 16:54:35 +01:00
CalDescent
dd33d24346 Fixed bug with identifier in ArbitraryDataWriter 2021-08-14 16:53:57 +01:00
CalDescent
fa684eabab Removed ArbitraryDataPatches class
This was essentially just a wrapper for a single method.
2021-08-14 15:51:16 +01:00
CalDescent
4b5bd6eed7 Cleanup temporary files used when building data directories. Also added a safety net to only delete files that are without our data or temp directories. 2021-08-14 15:43:08 +01:00
CalDescent
63f5946527 Switched all system-generated temp paths to a user specified "tempDataPath".
This ensures that the temporary files are being kept with the rest of the data, rather than somewhere inappropriate such as on flash storage. It also allows the user to locate them somewhere else, such as on a dedicated drive.
2021-08-14 14:30:06 +01:00
CalDescent
8dac3ebf96 More miscellaneous refactoring 2021-08-14 14:05:33 +01:00
CalDescent
09e783fbf6 Renamed a few variables and methods that weren't caught in previous refactors. 2021-08-14 14:01:30 +01:00
CalDescent
b18b686545 Renamed DATA_FILE to ARBITRARY_DATA_FILE and GET_DATA_FILE to GET_ARBITRARY_DATA_FILE.
This will break existing data nodes, but this is okay, as nothing is in production yet.
2021-08-14 13:57:44 +01:00
CalDescent
e99ea41117 Renamed DataFileMessage to ArbitraryDataFileMessage and GetDataFileMessage to GetArbitraryDataFileMessage 2021-08-14 13:52:41 +01:00
CalDescent
172a37ec8c Renamed org.qortal.storage package to org.qortal.arbitrary 2021-08-14 13:49:19 +01:00
CalDescent
da6b341b63 Renamed DataFile to ArbitraryDataFile, and DataFileChunk to ArbitraryDataFileChunk 2021-08-14 13:25:45 +01:00
CalDescent
16d93b1775 Renamed newly added classes to ArbitraryData*.java instead of DataFile*.java 2021-08-14 13:18:02 +01:00
CalDescent
e15cf063c6 Initial implementation of data patches/updates
This adds support for the PATCH method in addition to the existing PUT method.

Currently, a patch includes only files that have been added or modified, as well as placeholder files to indicate those that have been removed.

This is not production ready, as I am hoping to create patches on a more granular level - i.e. just the modified bytes of each file. It would also make sense to track deletions using a metadata/manifest file in a hidden folder.

It also adds early support of accessing files using a name rather than a signature or hash.
2021-08-14 13:11:36 +01:00
CalDescent
5ac9e3e47a Fixed recently introduced bugs in arbitrary transaction transformation. 2021-08-13 19:14:44 +01:00
CalDescent
743a61bf49 Fixed bug which overwrote the data file, causing it to be invalid. 2021-08-13 06:55:44 +01:00
CalDescent
c790ea07dd Started abstracting the file processing code away from the API handlers, and making it more modular. 2021-08-12 20:53:42 +01:00
CalDescent
b4f980b349 Restrict lists API endpoints to local/apiKey requests only. 2021-08-12 19:52:49 +01:00
CalDescent
673f23b6a0 Improvement to commit f71516f
Now only skipping the HTLC redemption if the AT is finished and the balance has been redeemed by the buyer. This allows HTLCs to be refunded for ATs that have been refunded or cancelled.
2021-08-12 08:29:52 +01:00
QuickMythril
8c325f3a8a original design 2021-08-11 18:18:10 -04:00
CalDescent
f71516f36f Skip finished ATs in the refund API endpoints. 2021-08-11 21:26:29 +01:00
CalDescent
1752386a6c Fixed logging errors in previous commit. 2021-08-11 20:33:54 +01:00
CalDescent
112675c782 Better handling of RPC errors.
Previously, if an error was returned from an Electrum server (such as "server busy") it would throw a NetworkException that would be caught outside of the server loop and cause the entire request to fail.

Instead of throwing an exception, I am now logging the error and returning null, in the same way we do for IOException and NoSuchElementException further up in the same method.

This allows the caller - most likely connectedRpc() - to move on to the next server in the list and try again.

This should fix an issue seen where a "server busy" response from a single server was essentially breaking our implementation, as we would give up altogether instead of trying another server.
2021-08-11 19:22:53 +01:00
CalDescent
3b6ba7641d Updated icon for Qortal.exe 2021-08-10 19:35:03 +01:00
CalDescent
477a35a685 Fixed response schema for GET /lists/blacklist/addresses endpoint 2021-08-10 08:43:47 +01:00
CalDescent
2a0a39a95a Avoid creation of lists directory until the first item is added to a list. 2021-08-09 23:35:32 +01:00
CalDescent
dfc77db51d Merge pull request #57 from ScythianQortal/translations
Added Hungarian translations and reorganised existing translations
2021-08-09 10:05:13 +01:00
CalDescent
c9596fd8c4 Catch exceptions thrown during GUI initialization.
This is a workaround for an UnsupportedOperationException thrown when using X2Go, due to PERPIXEL_TRANSLUCENT translucency being unsupported in splashDialog.setBackground(). We could choose to use a different version of the splash screen with an opaque background in these cases, but it is low priority.
2021-08-09 10:02:24 +01:00
CalDescent
78373f3746 HTLC redeem/refund APIs switched from GET to POST. 2021-08-08 10:29:15 +01:00
CalDescent
ebc3db8aed Default file path for repository data imports set to "qortal-backup/TradeBotStates.json". This allows the trade bot backup to be imported in a single click, and can now be potentially added as a button in the UI. 2021-08-08 10:20:44 +01:00
CalDescent
756601c1ce Initialize to an empty list.
This fixes various bugs caused by the list being null when no blacklist JSON file was available.
2021-08-08 08:41:13 +01:00
CalDescent
8bb5077e76 Catch occasional NPE when setting tray icon. 2021-08-08 08:29:29 +01:00
CalDescent
5b85f01427 Added defensiveness to list management methods in ResourceList.java 2021-08-08 08:28:34 +01:00
CalDescent
a7d594e566 Log the AT states reshape progress, as it seems to be taking a very long time. 2021-08-07 19:18:20 +01:00
CalDescent
481e6671c2 Added GET /lists/blacklist/addresses API endpoint
This returns a JSON array containing the blacklisted addresses.
2021-08-07 16:27:16 +01:00
Scythian
b890e02a6a Added new TransactionValidity keys
Added ADDRESS_ABOVE_RATE_LIMIT and DUPLICATE_MESSAGE ValidationResults to localeLang translation keys
2021-08-07 15:09:48 +01:00
Scythian
4772840b4c Reorganised translations
Updated the "localeLang" files with new keys and removed old unused keys for English, German, Dutch, Italian, Finnish, Hungarian, Russian and Chinese translations
2021-08-07 14:05:10 +01:00
CalDescent
cd7adc997b Prevent duplicate entries in a list. 2021-08-07 11:32:49 +01:00
CalDescent
9fdc901b7a Added POST /lists/blacklist/addresses and DELETE /lists/blacklist/addresses API endpoints.
These are the same as the /lists/blacklist/address/{address} endpoints but allow a JSON array of addresses to be specified in the request body. They currently return true if
2021-08-07 11:31:45 +01:00
Scythian
76ec3473d6 Updated TransactionValidity keys
Added ADDRESS_IN_BLACKLIST ValidationResult to TransactionValidity translation keys
2021-08-07 10:47:48 +01:00
CalDescent
b29ae67501 Apply the address blacklist to chat transactions.
This is based on code originally written by @DrewMPeacock
2021-08-07 10:31:56 +01:00
CalDescent
24f1fb566d Initial implementation of resource lists
The ResourceList class creates or updates a list for the purpose of tracking resources on the Qortal network. This can be used for local blocking, or even for curating and sharing content lists. Lists are backed off to JSON files (in the lists folder) to ease sharing between nodes and users.

This first implementation allows access to an address blacklist only, but has been written in such a way that other lists can be easily added. This might be needed in the future, e.g. to blacklist a group, a poll, or some hosted data. It could also be used by community members to curate lists of favourite or problematic content, which could then be shared or even subscribed to on the chain by other users.
2021-08-07 10:20:14 +01:00
CalDescent
a253294890 Ensure frozen ATs are still executed every block.
We currently want to execute frozen ATs, to maintain backwards support. We could optionally choose to stop executing them later, via a hard fork.
2021-08-06 20:01:59 +01:00
Scythian
0b53de1bb6 Added Hungarian translations 2021-08-06 19:28:56 +01:00
Scythian
746c68c9f6 Reorganised translations
Added new keys and removed old unused keys
Localised the "Build version" string in the SysTray
2021-08-06 19:27:28 +01:00
CalDescent
ec008b4a16 Merge branch 'AT-sleep-until-message' 2021-08-04 19:00:24 +01:00
CalDescent
1d65e34fe5 Revert "Added DogecoinACCTv2 and DogecoinACCTv2TradeBot"
This reverts commit 797dff4752.
2021-08-04 18:59:36 +01:00
CalDescent
8ae78703ca Revert "Initial attempt at adding "sleep until message" functionality to DOGE ACCTv2."
This reverts commit a1c61a1146.
2021-08-04 18:59:30 +01:00
CalDescent
bd4b9a9fd3 Modified .gitignore to allow multiple testnets to exist by adding a number or other suffix. 2021-08-04 18:56:16 +01:00
CalDescent
f09677d376 Added inputs, outputs and feeAmount to /crosschain//walletbalance endpoints
The inputs and outputs contain a simpler version than the ones in the raw transaction, consisting of `address`, `amount`, and `addressInWallet`. The latter of the three is to know whether the address is one that is derived from the supplied xpub master public key.
2021-08-04 18:54:36 +01:00
CalDescent
f669e3f6c4 Fixed Dogecoin tests. 2021-08-04 18:48:59 +01:00
CalDescent
961c5ea962 Treat zero as null in sleepUntilHeight AT data. This is needed because we are unable to call setSleepUntilHeight() with a null value due to the datatype used in the CIYAM AT library. An alternate option would be to fork the AT library and use an Integer or Long rather than an int, but since we don't have a block zero, this is still a valid thing to check even when using that approach. 2021-08-04 09:22:17 +01:00
CalDescent
a1c61a1146 Initial attempt at adding "sleep until message" functionality to DOGE ACCTv2. 2021-08-02 20:08:53 +01:00
CalDescent
797dff4752 Added DogecoinACCTv2 and DogecoinACCTv2TradeBot 2021-08-02 20:07:34 +01:00
CalDescent
711ad638b8 Renamed Chinese translation files.
zh_SC renamed to zh_CN, and zh_TC renamed to zh_TW. This is necessary for the localization library to locate the files correctly.
2021-08-02 09:24:38 +01:00
CalDescent
4956c3328c Updated AdvancedInstaller project for v1.6.0 2021-08-01 19:14:55 +01:00
CalDescent
96a82381d1 Bump version to 1.6.0 2021-08-01 18:19:59 +01:00
CalDescent
68190c8c76 Fixed a build error relating to using an int rather than Integer in the CIYAM AT library. Solved for now by using 0 instead of null, but will review this again before release. 2021-08-01 18:07:19 +01:00
CalDescent
dde47bc1fc Fixed build errors by adding sleepUntilMessageTimestamp to recent method additions. 2021-08-01 18:05:58 +01:00
CalDescent
744deaed8d Fixed merge issue due to differing db schemas. 2021-08-01 10:39:34 +01:00
CalDescent
a62910c8b6 Merge branch 'master' into AT-sleep-until-message 2021-08-01 10:30:52 +01:00
CalDescent
e259a09b89 Fixed merge issues. 2021-08-01 09:54:15 +01:00
CalDescent
6472d8438a Merge remote-tracking branch 'qortal/master'
# Conflicts:
#	pom.xml
#	src/main/java/org/qortal/api/ApiError.java
#	src/main/java/org/qortal/settings/Settings.java
#	src/main/resources/i18n/ApiError_en.properties
2021-07-31 22:57:07 +01:00
CalDescent
dc8a402a4a Revert "Removed all cross-chain code. It's not needed for data nodes."
This reverts commit cc59510cd0.

# Conflicts:
#	src/main/java/org/qortal/controller/Controller.java
2021-07-31 22:55:27 +01:00
CalDescent
f72374488e Revert "Removed block 212937"
This reverts commit b917da765c.
2021-07-31 22:54:19 +01:00
CalDescent
3c6d9a4b8e Merge branch 'new-coins' 2021-07-31 21:27:40 +01:00
CalDescent
3073388403 Fix for missing transactions in getWalletTransactions() response
The previous criteria was to stop searching for more leaf keys as soon as we found a batch of keys with no transactions, but it seems that there are occasions when subsequent batches do actually contain transactions. The solution/workaround is to require 5 consecutive empty batches before giving up. There may be ways to improve this further by copying approaches from other BIP32 implementations, but this is a quick fix that should solve the problem for now.
2021-07-31 18:10:24 +01:00
CalDescent
67f856c997 Increased minimum order size to 3 DOGE, as some transactions for 2.1 DOGE were being rejected due to the "dust" error. 2021-07-31 12:20:08 +01:00
CalDescent
742fd0b444 Added /crosschain/htlc/refundAll API endpoint
This is the equivalent of the redeemAll API but can be used from the buyer's side to get their LTC or DOGE back in the event of a problem.
2021-07-31 12:19:12 +01:00
CalDescent
e1d69c0eae Removed /refund/{ataddress}/{receivingAddress} API
This was causing confusion and isn't generally needed.
2021-07-31 11:55:48 +01:00
CalDescent
49d4190615 Support multiple blockchains in the /crosschain/htlc APIs.
This involved a small refactor of the ACCT code to expose findSecretA() in a more generic way. Bitcoin is disabled for refunding and redeeming as it uses a legacy approach that we no longer support. The {blockchain} URL parameter has also been removed from the redeem and refund APIs, because it can be obtained from the ACCT via the code hash in the AT.
2021-07-31 11:11:30 +01:00
CalDescent
64d39765ca Merge pull request #56 from DrewMPeacock/update-splash
Update task tray icon files. Now with transparency™
2021-07-30 10:51:44 +01:00
Sir.Galahad
aca8f64415 Update task tray icon files. Now with transparency™ 2021-07-30 03:43:31 -06:00
CalDescent
855b600268 testBlockHeightSpeed() reduced from 30k to 10k blocks.
This was intermittently failing on github and I suspect it may have been hitting memory limits.
2021-07-30 09:02:15 +01:00
CalDescent
476d613e20 Merge branch 'master' of github.com:Qortal/qortal 2021-07-30 09:00:54 +01:00
CalDescent
fb8a4d0a41 Merge pull request #55 from DrewMPeacock/update-splash
Update Qortal node graphics.
2021-07-30 08:30:15 +01:00
CalDescent
130f3f6d41 Added code to parse variable length block headers, necessary for DOGE. 2021-07-29 09:00:55 +01:00
CalDescent
ed997af043 Limit order size to a minimum of 2 DOGE.
The "dust" threshold is around 1 DOGE - meaning orders below this size cannot be refunded or redeemed. The simplest solution is to prevent orders of this size being placed to begin with.
2021-07-29 08:47:45 +01:00
Sir.Galahad
3c47f6917a Merge remote-tracking branch 'qm/status-on-icon' into update-splash 2021-07-27 19:46:34 -06:00
Sir.Galahad
e32a486493 Update splash screen image. 2021-07-27 19:42:00 -06:00
QuickMythril
208da935a1 updating icons
design credit: Haoshiro
2021-07-27 07:20:15 -04:00
QuickMythril
1dda9a875e updating icons
design credit: Haoshiro
2021-07-27 07:19:58 -04:00
CalDescent
b26175b7c6 Set DOGE fees to sensible values for mainnet. These can probably be optimized. 2021-07-26 19:03:33 +01:00
CalDescent
ffc6befb38 Fixed some missing DOGECOIN references 2021-07-26 09:21:56 +01:00
CalDescent
9df7c96d08 Added Dogecoin TradeBot and ACCT 2021-07-25 18:45:10 +01:00
CalDescent
32fa66f0a2 Merge pull request #54 from DrewMPeacock/master
Include AT address in /trades API call.
2021-07-23 08:50:27 +01:00
Sir.Galahad
7153ed022c Include AT address in /trades API call.
Include Seller Qortal address in /trades API call.
Include Buyer Qortal Receiving address in /trades API call.
2021-07-22 22:08:13 -06:00
CalDescent
50e4e71abb Added DOGE wallet. 2021-07-18 16:55:39 +01:00
CalDescent
d6e65a3d63 Removed version number from qortal.exe and qortal.zip
Recently we have stopped including the version number in the zip and exe files uploaded to github, as this allows us to use the "https://github.com/Qortal/*/releases/latest/download/*" redirect for all 3 files when linking from the qortal.org website. Previously, it could only be done for the JAR since this was the only file that didn't contain a version number. This avoids having to update the website every time we distribute a new release.

Note that this currently requires that the Qortal-x.y.z.exe file created by AdvancedInstaller is renamed to qortal.exe before running ./build-release.sh. If you forget to rename, the script will exit with a warning that the file couldn't be found.
2021-07-18 10:36:04 +01:00
CalDescent
7a77b12834 Repalce &amp; with & in HTML documents.
This is not a long term solution, but it's a quick fix to solve URL problems when converting to a static site via httrack.
2021-07-17 19:18:22 +01:00
CalDescent
fe387931a4 Increased buffer when serving assets from 1kiB to 10kiB
This speeds up image loading. Ultimately we will stop serving these via the application memory altogether.
2021-07-17 15:48:27 +01:00
CalDescent
dc83e32173 Fixed site preview functionality. 2021-07-17 15:23:24 +01:00
CalDescent
f5b29bad33 Encrypt websites with AES.
This ensures that nodes are storing unreadable files, outside of the context of Qortal. For public data, the decryption keys themselves are on-chain, included in the "secret" field of arbitrary transactions. When we introduce the concept of private data, we can simply exclude the secret key from the transaction so that only the owner can decrypt it.

When encrypting the file, I have added the 16 byte initialization vector as a prefix to the cyphertext, and it is then automatically extracted back out when decrypting. This gives us the option to encrypt more than one file with the same key, if we ever need it. Right now, we are using a unique key per file, so it's not actually needed, but it's good to have support.
2021-07-17 14:51:39 +01:00
CalDescent
f599aa4852 Modified serializeSizedString() and deserializeSizedString() to cope with null strings.
This affects various other parts of the system, not just arbitrary transactions.
2021-07-17 14:43:02 +01:00
CalDescent
6f05de2fcc Fixed newly introduced issues with arbitrary transaction transformation 2021-07-17 14:42:00 +01:00
CalDescent
02016c77f1 Fixed issue in HSQLDBSaver, introduced in recent commit. 2021-07-17 14:40:29 +01:00
CalDescent
93eede7c6b Added missing break statement which caused a database version mismatch. 2021-07-15 19:56:09 +01:00
CalDescent
944e396823 Added AES utility class from baeldung and updated copyright notice for ZipUtils which was based on code from the same author. This code still needs reviewing and modifying but it's a good starting point for AES encryption and decryption. 2021-07-15 19:50:42 +01:00
CalDescent
8a654834ac Small reorganization. 2021-07-15 19:47:53 +01:00
CalDescent
bb76fa80cd Another significant upgrade of arbitrary transactions
Adds "name", "method", "secret", and "compression" properties. These are the foundations needed in order to handle updates, encryption, and name registration. Compression has been added so that we have the option of switching to different algorithms whilst maintaining support for existing transactions.
2021-07-15 09:27:49 +01:00
CalDescent
53f44a4029 Added support for subdirectories in the HTML parser. 2021-07-14 18:03:51 +01:00
CalDescent
182dcc7e5f MAX_FILE_SIZE reduced to 500MiB to match the difficulty calculation. 2021-07-14 18:03:35 +01:00
CalDescent
2d272e0207 Added some service constants.
These combine some Qora services (SERVICE_NAME_STORAGE, SERVICE_BLOG_POST, and SERVICE_BLOG_COMMENT) with existing Qortal services (SERVICE_AUTO_UPDATE), and some new additions (SERVICE_ARBITRARY_DATA, SERVICE_WEBSITE, and SERVICE_GIT_REPOSITORY)
2021-07-14 17:50:55 +01:00
CalDescent
9384a50879 Derive PoW difficulty from the file size. Exact values TBC. 2021-07-13 22:18:21 +01:00
CalDescent
00d4f35f2c Track the request time in arbitraryDataFileRequests and automatically remove those that have timed out. 2021-07-11 10:28:14 +01:00
CalDescent
483557163e Refactor: moved arbitrary data code from Controller to ArbitraryDataManager 2021-07-11 10:17:38 +01:00
CalDescent
2679252b04 Rework of arbitrary data requests
Previously we would ask all connected peers for the file itself, but this caused the network to be swamped when multiple peers responded with the same file.

This new approach instead asks all connected peers to send back a list of hashes for all files they have relating to a transaction signature. The requesting node then uses these lists to make separate requests for each missing file.
2021-07-11 10:00:09 +01:00
CalDescent
5a95c827b4 When serving a website, delete the unzipped directory if the index file is not found.
This is a quick solution to rebuild directory structures with missing files. This whole area of the code needs some reworking, as serving the site from a temporary folder is not a robust long term solution.
2021-07-10 15:51:46 +01:00
CalDescent
f938d8c878 More refactoring 2021-07-10 14:47:51 +01:00
CalDescent
bfc0122c1b Added warning to README 2021-07-10 13:53:07 +01:00
CalDescent
79691541ae Merge pull request #51 from JaymenChou/patch-2
Update and rename SysTray_zh.properties to SysTray_zh_SC.properties
2021-07-08 23:06:19 +01:00
CalDescent
05d0542875 Merge pull request #50 from JaymenChou/patch-1
Create SysTray_zh_TC.properties
2021-07-08 23:06:03 +01:00
CalDescent
1d22b39a1d Merge pull request #52 from marcomoesman/master
Create Dutch (nl_NL) translations
2021-07-08 23:05:40 +01:00
CalDescent
549b68cf71 Merge pull request #53 from DrewMPeacock/master
Removes the flawed BIP-39 implementation in the core.
2021-07-08 23:05:15 +01:00
CalDescent
55f87de2e0 Updated AdvancedInstaller project for v1.5.6 2021-07-08 19:19:35 +01:00
CalDescent
b8424e20aa Bump version to 1.5.6 2021-07-08 18:24:11 +01:00
Sir.Galahad
bbe3a30e77 Removes the flawed BIP-39 implementation in the core.
Removes API calls for /mnemonic.

Readout for VanityGen.java now excludes a BIP-39 seed-phrase and only gives a raw private key.
2021-07-08 02:24:31 -06:00
CalDescent
cdc5348a06 Added "domain map" server
Domain names can be mapped to arbitrary transaction signatures via the node's settings, and then served over port 80 or 443. This allows Qortal hosted sites to be accessible via a traditional domain name.

Example configuration to map two domains:

"domainMapServiceEnabled": true,
"domainMapServicePort": 80,
"domainMap": [
  {
    "domain": "example.com",
    "signature": "tEsw4kUn4ZJfPha7CotUL6BHkFPs79BwKXdY6yrf28YTpDn4KSY6ZKX3nwZCkqDF9RyXbgaVnB1rTEExY3h9CQA"
  },
  {
    "domain": "demo.qortal.org",
    "signature": "ZdBWWPMhR7AZwSu5xZm89mQEacekqkNfAimSCqFP6rQGKaGnXR9G4SWYpY5awFGfhmNBWzvRnXkWZKCsj6EMgc8"
  }
]

Each domain needs to be pointed to the Qortal data node via an A record or CNAME. You can add redundant nodes by adding multiple A records for the same domain (this is known as DNS Failover).

Note that running a webserver on port 80 (or anything less than 1024) requires running the data node as root. There are workarounds to this, such as disabling privileged ports, or using a reverse proxy. I will investigate this more as time goes on, but this is okay for a proof of concept.
2021-07-06 18:50:40 +01:00
CalDescent
e64a3978e6 Moved HTML parsing to new class. 2021-07-06 08:23:28 +01:00
CalDescent
f2feb12708 /site API endpoint now tales a signature rather than a file hash
This allows it to verify that the data in a transaction, after which it will then build the complete file from its chunks if needed.
2021-07-05 09:07:06 +01:00
CalDescent
5319c5f832 Reworked existing unused ArbitraryDataManager.
It's now capable of syncing chunks as well as complete files. This isn't production ready as it currently requests/receives the same file from multiple peers at once, which slows down the sync and wastes lots of bandwidth. Ideally we would find an appropriate peer first and then sync the file from them.
2021-07-05 09:05:45 +01:00
CalDescent
7531fe14fe Fixed major performance issue in DataFile.toString() 2021-07-05 08:23:29 +01:00
CalDescent
0086c6373b Significant refactor of DataFile and DataFileChunk
This introduces the hash58 property, which stores the base58 hash of the file passed in at initialization. It leaves digest() and digest58() for when we need to compute a new hash from the file itself.
2021-07-05 07:26:20 +01:00
CalDescent
10dc19652e Use "qortaldata-" version prefix 2021-07-04 16:52:20 +01:00
CalDescent
2f2c4964c5 Transaction version temporarily bumped to 5. TODO: We will need to set a hard fork timestamp if this is ever merged back into the main Qortal core repo. 2021-07-04 16:43:12 +01:00
CalDescent
cb4203b6db Use public key as parameter instead of address, since we can obtain the address from the public key in all cases. 2021-07-04 14:53:54 +01:00
CalDescent
bb5b62466e Fixed bug introduced in recent commit. 2021-07-04 13:41:37 +01:00
CalDescent
6407b5452b Delete our copies of data if any exception is thrown. 2021-07-04 13:39:00 +01:00
CalDescent
a742fecf9c API refactors to avoid generic unhandled states. 2021-07-04 13:38:20 +01:00
CalDescent
60415b9222 /arbitrary/upload/creator/{address} now returns an unsigned ARBITRARY transaction, currently with pre-computed nonce (same as the /site equivalent) 2021-07-04 10:25:15 +01:00
CalDescent
ffb39ef074 /data API endpoints moved to /arbitrary 2021-07-04 09:49:01 +01:00
CalDescent
d73f5ed2b5 /site/upload/creator/{address} now returns an unsigned ARBITRARY transaction, currently with pre-computed nonce 2021-07-04 09:45:52 +01:00
CalDescent
7af973b60d Removed TODO comments that are now done. 2021-07-04 09:41:53 +01:00
CalDescent
49eddc9da5 Allow zero fee transactions if the fee is zero in blockchain.json
Until now it wasn't possible to set up a chain with zero transaction fees due to a hardcoded zero check in Payment.isValid(), and a divide by zero error in Transaction.hasMinimumFeePerByte()
2021-07-04 09:31:51 +01:00
CalDescent
4b1de108d1 Fixed bug in expected chunk count. 2021-07-03 18:42:42 +01:00
CalDescent
e46c735efa Fixed recently introduced bugs with file management. 2021-07-03 18:05:17 +01:00
CalDescent
56da7deb4c DataFile updates to simplify integration with arbitrary transactions. 2021-07-03 17:41:52 +01:00
CalDescent
5f4649ee2b Major upgrade of arbitrary transactions
- Adds support for files up 500MiB per transaction (at 2MiB chunk sizes). Previously, the max data size was 4000 bytes.
- Adds a nonce, giving us the option to remove the transaction fees altogether on the data chain.

These features become enabled in version 5 of arbitrary transactions.
2021-07-03 17:40:02 +01:00
CalDescent
7cc2c4f621 Progress on website API endpoints. 2021-07-02 08:57:55 +01:00
CalDescent
cc449f9304 Added DataFile.chunkHashes() method which appends all hashes into a single byte array (32 bytes / 256 bits each). 2021-07-02 08:54:05 +01:00
CalDescent
28425efbe7 Added jsoup dependency - this was missing from an earlier commit. 2021-07-02 08:52:38 +01:00
CalDescent
8c3a22aa5c Improved link replacement criteria. 2021-07-01 11:21:40 +01:00
CalDescent
f3e5933599 Fixed naming error in joinFiles API. 2021-07-01 11:18:46 +01:00
Marco Moesman
39d8750ef9 Merge branch 'Qortal:master' into master 2021-07-01 10:24:03 +02:00
CalDescent
52b0c244a8 Extend CHECKPOINT_LOCK to HSQLDBSaver.execute()
This is used when saving new data to the db, so also needs to be blocked if we are checkpointing or deciding whether to checkpoint.
2021-06-28 19:24:53 +01:00
CalDescent
ee95a00ce2 Hopeful fix for "Performing repository CHECKPOINT..." deadlock.
This is probably our number one reliability issue at the moment, and has been a problem for a very long time.

The existing CHECKPOINT_LOCK would prevent new connections being created when we are checkpointing or about to checkpoint. However, in many cases we obtain the db connection early on and then don't perform any queries until later. An example would be in synchronization, where the connection is obtained at the start of the process and then retained throughout the sync round. My suspicion is that we were encountering this series of events:
1. Open connection to database
2. Call maybeCheckpoint() and confirm there are no active transactions
3. An existing connection starts a new transaction
4. Checkpointing is performed, but deadlocks due to the in-progress transaction

This potential fix includes preparedStatement.execute() in the CHECKPOINT_LOCK, to block any new transactions being started when we are locked for checkpointing. It is fairly high risk so we need to build some confidence in this before releasing it.
2021-06-28 09:13:36 +01:00
QuickMythril
11566ec923 set icon on status change 2021-06-27 03:45:15 -04:00
QuickMythril
a78ff08202 add setTrayIcon function 2021-06-27 03:44:29 -04:00
QuickMythril
ceb3969c8b load icons into gui 2021-06-27 03:44:25 -04:00
QuickMythril
6f048ef40e add status icons 2021-06-27 03:41:49 -04:00
CalDescent
aac4fe37e8 Fixed API response description. 2021-06-26 10:28:22 +01:00
CalDescent
ebfa941a4f Fixed some file separators. 2021-06-26 10:27:32 +01:00
CalDescent
47c70eea9e Use system temp directory instead of making a "temp" subfolder when zipping files. 2021-06-26 10:11:34 +01:00
CalDescent
fe7c40cb7c Reduced log spam. 2021-06-25 19:31:28 +01:00
CalDescent
8973626a4b Fixed issue with temp directory on Linux. 2021-06-25 19:31:13 +01:00
CalDescent
ace5d999e2 Log a comma separated list of hashes after splitting a file into chunks, so they can easily be requested from another node using the //data/files/frompeer/{peer} API endpoint. Again temporary until the sync happens automatically. 2021-06-25 19:30:45 +01:00
CalDescent
52829a244b More work on APIs to request files from peers. I won't spend too long making these perfect as they are mostly a temporary measure. 2021-06-25 19:28:39 +01:00
CalDescent
71c247fe56 Added POST /data/file/{hash}/build API, used to join multiple chunks together. 2021-06-25 19:28:01 +01:00
CalDescent
b34066f579 More work on HTML parsing.
The style tag parsing ideally needs rewriting using an actual CSS parser, but we can get away with this hacky approach in the short term.
2021-06-25 08:34:44 +01:00
CalDescent
b286c15c51 Optimized website serving, and added code to return the correct content types.
This is probably the most efficient way to process the data on the fly, but it's still not very scalable. A better approach would be to pre-process the HTML when building the file structure, and then serve them completely statically (i.e. using a standard webserver rather than via application memory). But it makes sense to keep it this way for development and maybe early beta testing.
2021-06-25 08:32:22 +01:00
Marco Moesman
aff4f6c859 Create TransactionValidity_nl.properties 2021-06-24 19:13:55 +02:00
Marco Moesman
1f8f73fa30 Create ApiError_nl.properties 2021-06-24 18:44:53 +02:00
Marco Moesman
620d6624a9 Create SysTray_nl.properties 2021-06-24 18:30:50 +02:00
JaymenChou
287f42ae64 Update and rename SysTray_zh.properties to SysTray_zh_SC.properties
Rename to zh_SC for better distinguish between zh_SC (Simple Chinese)and zh_TC(Traditional Chinese)
Rephrase some of the words for better understanding.
2021-06-24 11:42:48 +08:00
JaymenChou
d976c97d13 Create SysTray_zh_TC.properties 2021-06-24 11:31:46 +08:00
CalDescent
6d549b0754 Updated AdvancedInstaller project for v1.5.5 2021-06-23 19:57:03 +01:00
CalDescent
02dd64558f Bump version to 1.5.5 2021-06-23 18:21:55 +01:00
CalDescent
ea5e2f5580 Added POST /site/preview API
This can be used to preview a site before signing a transaction and announcing it to the network. The response will need reworking to return JSON (along with most of the other new APIs)
2021-06-23 09:28:38 +01:00
CalDescent
b65c7a75fe Handle relative links when parsing HTML. 2021-06-23 09:26:41 +01:00
CalDescent
39f5dce51c Moved "directory" data uploads to new POST /site/upload API.
Directory uploads don't make much sense outside of website hosting, so it's best to make this API specific to that purpose.
2021-06-23 09:10:06 +01:00
CalDescent
f77ec1faf6 A very crude proof of concept which serves a website from a zipped (and in future, chunked) data blob. This forms the beginnings of the "website hosting" layer on top of the data storage. It needs a significant rework - most importantly so that we aren't serving every asset from memory, and also so that the correct content-type headers are returned, etc. 2021-06-23 08:43:21 +01:00
CalDescent
cd3a1e0159 Increased max file size to 1GiB. Will review this again later. 2021-06-23 08:33:53 +01:00
CalDescent
3f20fadb81 When zipping files, rename the outer folder to "data" instead of using the original folder name. This means that the data can be accessed deterministically without the need to first lookup the folder name. 2021-06-23 08:09:59 +01:00
CalDescent
1c6428dd3b Added equivalent split and join test but this time using a 5.5MiB source file. 2021-06-23 08:07:35 +01:00
CalDescent
aca620241a More work on data file split/join, and added a test. 2021-06-22 08:58:16 +01:00
CalDescent
808b36e088 Specify chunk size when splitting. 2021-06-22 07:44:34 +01:00
CalDescent
1613375cc0 Added more validation of files received in GET /data/file/frompeer 2021-06-21 19:03:34 +01:00
CalDescent
787ef957d2 Added support for uploading an entire directory via POST /data/upload/path
If a directory is specified instead of a file, the directory is automatically zipped before being split into chunks.
2021-06-21 19:02:49 +01:00
CalDescent
b915d0aed5 Only create the output file directories when we are actually writing a file there. This should prevent empty directories being created when initializing a nonexistent DataFile using a hash. 2021-06-21 08:40:52 +01:00
CalDescent
16dc5b5327 Include the data length in DataFileMessage, which is more similar to the approach used by ArbitraryDataMessage. These two message types are very similar, except arbitrary code currently has a requirement of one piece of data per signature, whereas DataFile code is independent and can support multiple files per transaction. Maybe the two can be combined at some point, but for now I'll keep them separate. 2021-06-20 19:49:10 +01:00
CalDescent
d25e98d9c4 Include peer connection ID in recently created log message. 2021-06-20 10:26:14 +01:00
CalDescent
227cdc1ec8 Log each sync attempt when our blockchain isn't up to date
Without this it can appear as though nothing is happening for a while after the app launches.
2021-06-20 07:25:25 +01:00
CalDescent
c2d0c63db0 Improved error logging. 2021-06-19 20:31:04 +01:00
CalDescent
f5c9807a48 Use contains() rather than equals() when matching a peer in /data/file/frompeer, so that the port can be optionally left out. 2021-06-19 20:29:05 +01:00
CalDescent
7e9b1d5e16 Rework of DataFile.base58Digest()
This fixes an NPE when trying to send a file that doesn't exist. It also removes the caching, which we can add again later if it turns out to be needed.
2021-06-19 20:25:25 +01:00
CalDescent
5070c4eea9 Better handling of data file responses in the /data/file/frompeer API endpoint. 2021-06-19 20:23:33 +01:00
CalDescent
33d9c51b6f Validate supplied base58 string in /data/file/frompeer API endpoint 2021-06-19 19:26:13 +01:00
CalDescent
d0f9d478c2 Fixed bug which prevented the DATA_FILE message ID from making it through to the reply queue. 2021-06-19 19:05:11 +01:00
CalDescent
f296ec46c8 Removed unused headers. 2021-06-19 17:32:55 +01:00
CalDescent
64d19e480b Chunk size set to 1MB for now, as it seems that our networking code has problems when transferring 2MB chunks. We can increase this later once that problem has been fixed. 2021-06-19 17:32:48 +01:00
CalDescent
2c585a9328 Upper connection time limit reduced from 60 mins to 20 mins.
Now that we aren't disconnecting mid sync, we can get away with more frequent disconnections. This brings the average connection length to around 9 mins.
2021-06-19 17:25:50 +01:00
CalDescent
45b0d9e19b Added more initial peers, submitted by CWDSYSTEMS 2021-06-19 16:19:53 +01:00
CalDescent
026a4b896c Added BTC and LTC electrum nodes submitted by CWDSYSTEMS 2021-06-19 14:50:18 +01:00
CalDescent
78237fcd11 Don't intentionally disconnect peers if we are currently syncing with them. 2021-06-19 13:12:16 +01:00
CalDescent
73cc3dcb92 Force a disconnect of each peer when the connection age reaches the maximum allowed time.
Connection limits are defined in settings (denominated in seconds):
"minPeerConnectionTime": 120,
"maxPeerConnectionTime": 3600

Peers will disconnect after a randomly chosen amount of time between the min and the max. The default range is 2 minutes to 1 hour, as above.

This encourages nodes to connect to a wider range of peers across the course of each day, rather than staying connected to an "island" of peers for an extended period of time. Hopefully this will reduce the amount of parallel chains that can form due to permanently connected clusters of peers.

We may find that we need to reduce the defaults to get optimal results, however it is best to do this incrementally, with the option for reducing further via each node's settings. Being too aggressive here could cause some of the earlier problems (e.g. 20% missing blocks minted) to reappear. We can re-evaluate this in the next version. Note that if defaults are reduced significantly, we may need to add code to prevent this from happening mid-sync. With higher defaults, this is less of an issue.

Thanks to @szisti for supplying some base code for this commit, and also to @CWDSYSTEMS for diagnosing the original problem.
2021-06-19 13:03:46 +01:00
CalDescent
4cff03e7fe Include "size" value in the "Synchronized with peer" logs.
This indicates the size of the re-org/rollback that was required in order to perform this sync operation. It is only included if it's greater than 0 blocks.
2021-06-19 09:04:14 +01:00
CalDescent
8e35f131d5 Removed debugging log that wasn't intended to be committed. 2021-06-18 08:33:51 +01:00
CalDescent
aafb9d7e4f Include running total in "Sent X bytes" log entry. 2021-06-18 08:05:35 +01:00
CalDescent
652f30bdbd Added DATA_FILE and GET_DATA_FILE message types. 2021-06-18 08:02:57 +01:00
CalDescent
f4ba7b2a0c Added onNetworkGetDataFileMessage() handler 2021-06-18 08:02:13 +01:00
CalDescent
592490d709 Added GET /data/file/frompeer API endpoint
This requests a file from the supplied peer address, and stores a copy locally if successful. Still a work in progress.
2021-06-18 07:59:46 +01:00
CalDescent
5ac676d201 Added DataFileMessage and GetDataFileMessage, used for requesting and sending files between peers. 2021-06-17 19:05:00 +01:00
CalDescent
abfe0a925a More DataFile methods and improvements 2021-06-17 18:20:42 +01:00
CalDescent
fa11f4f45b Moved DataFileChunk(byte[] fileContent) constructor to the superclass so it can be used by regular data files too. 2021-06-17 18:20:11 +01:00
CalDescent
1e8dbfe4b7 Delete chunk if it fails the hash validation in the constructor. 2021-06-17 18:18:19 +01:00
CalDescent
f82f2bd287 Added DELETE /data/file API endpoint
This deletes a file referenced by a user supplied SHA256 digest string (which we will use as the file's "ID" in the Qortal data system). In the future this could be extended to delete all associated chunks, but first we need to build out the data chain so we have a way to look up chunks associated with a file hash.
2021-06-16 20:03:15 +01:00
CalDescent
76742c3869 Removed .dat extension, and use an extra level in the directory structure (data/aB/cD/aBcDeF... etc) 2021-06-16 19:49:12 +01:00
CalDescent
9407e7e418 Data storage location moved to settings ("dataPath") 2021-06-16 19:22:50 +01:00
CalDescent
120552b36e Added resource file missing from last commit. 2021-06-16 19:13:24 +01:00
CalDescent
9fb58c7ae3 Added the beginnings of an upload API, with some basic data file management. 2021-06-16 19:10:35 +01:00
CalDescent
b917da765c Removed block 212937 2021-06-15 09:29:07 +01:00
CalDescent
cc59510cd0 Removed all cross-chain code. It's not needed for data nodes. 2021-06-15 09:27:05 +01:00
CalDescent
86aab7023c Initial settings for data node development. Most to be decided later. 2021-06-14 19:40:37 +01:00
CalDescent
e3b0a41ba9 Merge branch 'sync-multiple-blocks' 2021-06-14 19:01:17 +01:00
CalDescent
802c55dfc8 Merge branch 'networking' 2021-06-14 19:00:17 +01:00
CalDescent
280f7814aa Merge branch 'master' of github.com:Qortal/qortal 2021-06-14 18:49:55 +01:00
CalDescent
3174681bd8 Removed /src/main/resources/log*.properties from .gitignore
We must be careful not to add files to the resources folder accidentally, given that a bundled log4j2.properties file is used in preference to the user's copy. By keeping this out of gitignore, it becomes more obvious if a file is added, and it can then be caught and removed before a release.
2021-06-14 18:49:24 +01:00
CalDescent
853f80b928 Updates to build-zip.sh and build-release.sh
There were necessary for these scripts to function in my build environment (Mac OSX). This may give errors when running in other environments, but we can deal with that in future, when others need to use these scripts.
2021-06-14 18:46:21 +01:00
CalDescent
8bdad377d7 Removed outdated qortal.jar from "WindowsInstaller/Install Files" directory.
Including an older JAR in the source code only leads to confusion, because a zip of the source code is automatically included with each github release. From what I can see, there is no need for it to be here. Added to .gitignore so we have the option of keeping a local copy.
2021-06-14 18:41:27 +01:00
CalDescent
9e1c2a5bd1 Updated AdvancedInstaller project for v1.5.4 2021-06-14 18:39:46 +01:00
CalDescent
b1777b6011 Bump version to 1.5.4 2021-06-13 19:22:16 +01:00
CalDescent
904be3005f Enable fast sync by default. 2021-06-12 13:07:25 +01:00
CalDescent
95eaf4c887 Merge branch 'master' into sync-multiple-blocks 2021-06-12 11:15:28 +01:00
CalDescent
e3923b7b22 Fixed issue causing frequent disconnects (found by szisti)
When sending or requesting more than 1000 online accounts, peers would be disconnected with an EOF or connection reset error due to an intentional null response. This response has been removed and it will instead now only send the first 1000 accounts, which prevents the disconnections from occurring.

In theory, these accounts should be in a different order on each node, so the 1000 limit should still result in a fairly even propagation of accounts. However, we may want to consider increasing this limit, to maximise the propagation speed.

Thanks to szisti for tracking this one down.
2021-06-11 19:10:04 +01:00
CalDescent
a43993e3ec Use placeholder build timestamp and build version when building without mvn package (e.g. from within an IDE)
This fixes an exception thrown when running directly in IntelliJ, as previously we relied on mvn package to parse the commit hash and timestamp.
2021-06-11 19:01:35 +01:00
CalDescent
bc6b3fb5f4 Include timestamps in block-timings.sh 2021-06-09 13:04:49 +01:00
CalDescent
df47f5d47b Merge branch 'master' into sync-multiple-blocks 2021-06-06 10:35:47 +01:00
CalDescent
319e64bacc Defend against an edge case NPE in the chat messages websocket. 2021-06-06 10:34:20 +01:00
CalDescent
ecf044bed1 Removed qortal-backup folder from git 2021-06-06 10:29:58 +01:00
CalDescent
76e1de38e8 Workaround for issue where sometimes an AT stays in "TRADING" mode even after it is marked as finished. This caused Bob's tradebot to enter BOB_REFUNDED mode instead of redeeming the LTC. This workaround treats "TRADING" as "REDEEMED" as long as the AT is finished. It will still enter the BOB_REFUNDED state if the AT's trade state is "REFUNDED" or "CANCELLED", to prevent it trying to redeem LTC without the secret. Longer term we need to prevent the AT itself from getting in this state to begin with, but this should at least solve the LTC redemption problem that occurs as a result. 2021-06-05 12:31:49 +01:00
CalDescent
1648a74ed7 Removed code which auto deletes trade bot data if it can't locate the AT after 24 hours. It's not a good idea to ever delete trade bot data, since it can contain private keys necessary to redeem or refund LTC. We have seen at least one instance of this where the trade bot data was deleted for an active trade. We still have the auto backup in these cases, so the keys are recoverable, but it's safest to avoid any auto deletions. 2021-06-05 11:25:05 +01:00
CalDescent
c63a7884cb Limit to 10 untrimmed blocks per response, as they are larger than the trimmed ones. 2021-06-02 09:10:25 +01:00
CalDescent
cffbd41f26 Reduce memory allocations in onNetworkGetBlocksMessage 2021-06-02 09:09:06 +01:00
CalDescent
c443187d0b Reduce log spam by logging the total number of expired unconfirmed transactions that are deleted, rather than each one individually. The individual deletion logs have been moved from INFO to DEBUG. 2021-06-01 20:06:37 +01:00
CalDescent
8c305d8390 Reduced log levels of recent synchronizer / controller log additions from INFO to DEBUG.
These are no longer necessary now that we have achieved good stability in the network.
2021-06-01 08:39:38 +01:00
CalDescent
0345c5c03b Updated AdvancedInstaller project for v1.5.3 2021-05-31 21:27:42 +01:00
CalDescent
cc6ac4c9d9 Bump version to 1.5.3 2021-05-31 17:36:21 +01:00
CalDescent
2ceba45782 Fast sync default blocks per request increased to 100. 2021-05-30 14:57:58 +01:00
CalDescent
ed423ed041 Increased MAX_DATA_SIZE and SYNC_BATCH_SIZE, to increase the effectiveness of the batch sync. 2021-05-30 14:54:13 +01:00
CalDescent
f58a52eaa4 Further work to increase the response timeout when requesting multiple blocks. 2021-05-30 13:10:38 +01:00
CalDescent
688404011b Relocate FETCH_BLOCKS_TIMEOUT to Peer.java and use a static import. 2021-05-30 10:08:50 +01:00
CalDescent
8881e0fb75 Merge branch 'master' into sync-multiple-blocks 2021-05-30 09:57:56 +01:00
CalDescent
61de7e144e Merge branch 'networking' into sync-multiple-blocks
# Conflicts:
#	src/main/java/org/qortal/network/Peer.java
2021-05-30 09:57:35 +01:00
CalDescent
815934ff5c Added GET /crosschain/htlc/redeemAll/LITECOIN API
This loops through all sell orders and attempts to redeem the LTC from each one. It will return true if at least one was redeemed, or false if none are available to be redeemed. Details are logged to the log.txt file rather than returned in the API response.
2021-05-29 19:43:08 +01:00
CalDescent
c3ff9e49e8 Merge pull request #40 from szisti/fixedNetwork
Support for configuration based fixed network
2021-05-29 09:51:56 +01:00
Istvan Szabo
d52875aa8f Added logs to intentional disconnects 2021-05-28 16:06:27 +01:00
Istvan Szabo
9027cd290c Filter out on demand connections when using fixed network 2021-05-28 14:47:30 +01:00
Istvan Szabo
58a7203ede Support for configuration based fixed network 2021-05-28 14:47:30 +01:00
CalDescent
5a84016a91 Merge pull request #39 from szisti/networking
Code formatting, connection age, and logging changes for networking
2021-05-28 10:09:07 +01:00
Istvan Szabo
bb0269f484 Converted time format 2021-05-28 08:53:01 +01:00
Istvan Szabo
1adc9349fc Added connection age to connected peers dto 2021-05-28 08:04:57 +01:00
Istvan Szabo
06215c83f2 Reduced log levels 2021-05-27 10:48:17 +01:00
Istvan Szabo
8a828137ee Removed code coverage report as it seems to conflict with tests randomly 2021-05-27 09:52:33 +01:00
Istvan Szabo
de4b1c8f09 Removed missed functional change 2021-05-27 09:15:32 +01:00
Istvan Szabo
265d40f04a Code formatting and logging changes for networking 2021-05-27 09:03:18 +01:00
szisti
b64e52c0c0 Automated testing (#38)
* added basic workflow

* Testing workflow

* renamed workflow file

* Disabled extremely slow test

* Disabled currently failing tests

* Added jacoco and updated workflow

* We cannot run gui tests headless

* Fixed jacoco configuration

* Updated job name in the workflow

* Adjusting workflow

* Testing maven caching

* Added logging for one of the jacoco related issues

* Updated coverage logging

Co-authored-by: Istvan Szabo <istvan.szabo@betvictor.com>
2021-05-26 11:27:46 +01:00
CalDescent
ac02e5c0a6 Merge pull request #37 from szisti/fix-cross-transaction-display
Fix cross chain transaction display
2021-05-26 08:39:51 +01:00
Istvan Szabo
427a415fbf Adjusted bitcoiny to convert transaction info into the new DTO 2021-05-25 23:57:54 +01:00
Istvan Szabo
9a3414aaa7 Added new DTO to store the data 2021-05-25 23:55:12 +01:00
CalDescent
7f5486dade Merge branch 'master' into sync-multiple-blocks 2021-05-25 07:37:01 +01:00
CalDescent
c8897ecf9b Rewrite of HSQLDBATRepository.getBlockATStatesAtHeight() SQL query
The previous query was taking almost half a second to run each time, whereas the new version runs 10-100x faster. This was the main bottleneck with block serialization and should therefore allow for much faster syncing once rolled out to the network. Tested several thousand blocks to ensure that the results returned by the new query match those returned by the old one.
2021-05-24 19:52:20 +01:00
CalDescent
2c8b94d469 Always use the org.qortal.utils.Base58 implementation
A couple of classes were using the bitcoinj alternative, which is twice as slow. This mostly affected the API on port 12392, as byte arrays were automatically encoded as base58 strings via the Base58TypeAdapter / JAXB package-info.
2021-05-24 19:38:01 +01:00
CalDescent
36c1cfae51 Log the P2SH address when redeeming or refunding LTC via the API. 2021-05-24 19:00:04 +01:00
CalDescent
41ad78750e Don't allow QORT addresses to be used as the receiving address when redeeming LTC
This is probably more validation than is actually needed, but given that we use the same field for LTC and QORT receiving addresses in the database, it is best to be extra careful.
2021-05-24 18:59:41 +01:00
CalDescent
3eaa4d5b38 Added /crosschain/htlc/refund/LITECOIN/{ataddress}/{receivingAddress} API
This is the same as the /crosschain/htlc/refund/LITECOIN/{ataddress} API, but allows a custom destination address to be specified.
2021-05-23 18:52:03 +01:00
CalDescent
35176f9550 Added other files to .gitignore 2021-05-23 16:57:09 +01:00
CalDescent
eb2c7268ea Removed .DS_Store files. 2021-05-23 15:31:26 +01:00
CalDescent
80311355ae Added /blocks/signature/{signature}/data API
This returns serialized, base58 encoded data for the entire block. It is the same format as the data sent between nodes when synchronizing, with base58 encoding added so that it can be outputted cleanly in the API response.
2021-05-23 13:10:47 +01:00
CalDescent
39d1590ace Improved descriptions of the new API endpoints. 2021-05-22 14:16:14 +01:00
CalDescent
0b36b650a4 Added /redeem/LITECOIN/{ataddress} API
This is the equivalent of the refund API but can be used by the seller to redeem LTC from a stuck transaction, by supplying the associated AT address, There are no lockTime requirements; it is redeemable as soon as the buyer has redeemed the QORT and sent the secret to the seller.
2021-05-22 13:59:00 +01:00
CalDescent
39575e8542 Added /refund/LITECOIN/{ataddress} API
This is designed to be called by the buyer, and will force refund their P2SH transaction associated with the supplied AT. The tradebot responsible for this trade must be present in the user's db for this API access the necessary data. It must be called after lockTime has passed, which for LTC is currently 60 minutes from the time that the P2SH was funded. Trying to refund before this time will result in a FOREIGN_BLOCKCHAIN_TOO_SOON error.
2021-05-22 10:09:28 +01:00
CalDescent
326ef498b0 Added /crosschain/htlc/redeem/LITECOIN/{ataddress}/{tradePrivateKey}/{secret}/{receivingAddress} API
This can currently be used by either the buyer or the seller, but it requires the seller's trade private key & receiving address to be specified, along with the buyer's secret. Currently hardcoded to LITECOIN but I will aim to make this generic as we start adding more coins.
2021-05-22 09:51:57 +01:00
CalDescent
5148bad82e /crosschain/htlc APIs now take base58 encoded params instead of hex.
This makes them more compatible with the output of the /crosschain/tradebot and /crosschain/trade/{ataddress} APIs which is likely where most people will be retrieving data from, rather than the database itself.
2021-05-20 09:20:14 +01:00
CalDescent
518f02472f Added POST /crosschain/LitecoinACCTv1/redeemmessage API
This is similar to the BTC equivalent, but removes secretB as an input parameter. It also signs and broadcasts the transaction, because the wallet isn't needed for this. These transactions have to be signed using the tradePrivateKey from the tradebot data rather than any of the wallet's keys.

There are two other LitecoinACCTv1 APIs still to implement, but I will leave these until they are needed.
2021-05-20 07:59:19 +01:00
CalDescent
ee5a132eb2 Updated AdvancedInstaller project for v1.5.2 2021-05-17 20:31:28 +01:00
CalDescent
654dc5bff3 Bump version to 1.5.2 2021-05-17 17:02:38 +01:00
CalDescent
27aeb4f05f Merge branch 'master' into sync-multiple-blocks
# Conflicts:
#	src/main/java/org/qortal/controller/Synchronizer.java
2021-05-16 11:21:34 +01:00
CalDescent
13dcf7f72a Added/updated some comments relating to a possible future optimization. 2021-05-16 11:03:11 +01:00
CalDescent
65c26f17df Reduced "Error while trying to find common block with peer" log from INFO to DEBUG when determining which peer to sync with. When performing the actual synchronization, use INFO logging as this is a more serious error. 2021-05-16 10:45:40 +01:00
CalDescent
3bedba71d5 Reduced frequency and level of some synchronizer logs. 2021-05-16 10:36:41 +01:00
CalDescent
1ba64d9745 Bumped bitcoinj version from 0.15.6 to 0.15.10 2021-05-16 10:00:28 +01:00
CalDescent
84bf570243 Added optional "maxtrades" parameter to /crosschain/price/{blockchain} API
This specifies the maximum number of trades to be used when calculating the price. Default: 10
2021-05-16 09:51:11 +01:00
CalDescent
28d50bccf9 Exclude peers if we don't have a complete set of their block summaries.
This tightens up the decision making by adding two requirements:

1. The peer must return the same number of summaries to the ones requested.
2. The peer must return a summary that matches its latest reported signature.

This ensures we are always making sync decisions based on accurate data, and removes peers that are currently mid re-org. This is probably more validation than is actually necessary, but it's best to be really thorough here so it is as optimized as possible.
2021-05-16 09:15:37 +01:00
CalDescent
66711c2e9d Require a complete sync in syncToPeerChain()
We have gone backwards and forwards on this one a lot recently, but now that stability has returned, it is best to tighten this up. Previously it was loosened to help reduce network load, but that is no longer a problem. With this stricter approach, it should prevent a node ending up in an incomplete state after syncing, which is the main cause of the shorter re-orgs we are seeing.
2021-05-16 08:45:23 +01:00
CalDescent
92d8c37d7d Added AT count to block debug logs. 2021-05-15 12:54:46 +01:00
CalDescent
5824f75669 Rework of the repository export and import functions.
The existing HSQL export/import (PERFORM EXPORT SCRIPT and PERFORM IMPORT SCRIPT) have been replaced with a custom JSON import and export. Whilst this is less generic, it has some significant advantages:

- When exporting data, it is now able to combine the exported data with any data that already exists in the backup file. This prevents a backup after a bootstrap from overwriting data from before the bootstrap, and removes the need for all of the "archive" files that we currently create.
- Adds support for partial imports, and updates. Previously an import would fail if any of the data being imported already existed in the db. It will now add new rows and update existing ones.
- The format and contents of the exported trade bot data now matches the output of the /crosschain/tradebot API.
- Data is retrieved without the need for a database lock, and therefore the export process is much faster and less invasive. This should prevent the lockups and other problems seen when using the trade portal.

For now, there are a couple of trade-offs to using this new approach:
- The minting key import/export has been temporarily removed until there is more time to transition it to this new format.
- Existing .script backups can no longer be imported using versions higher than 1.5.1.

Both of these can be solved by temporarily running version 1.5.1, performing the necessary imports/exports, then returning to the latest version. Longer term the minting keys export/import will be reimplemented using the JSON format.
2021-05-15 12:19:15 +01:00
CalDescent
deb8adafc9 Added org.json dependency.
The com.googlecode.json-simple dependency we use in other parts of the project isn't ideal for some of the more complex parsing.
2021-05-15 09:15:29 +01:00
CalDescent
d2649b237c Moved chain weight calculation log from DEBUG to TRACE. 2021-05-11 19:01:23 +01:00
CalDescent
255233fe38 Reduced log spam. 2021-05-10 09:10:51 +01:00
CalDescent
6532c258f6 Reduced log spam. 2021-05-10 09:10:14 +01:00
CalDescent
4ac3984b7c Merge branch 'master' into sync-multiple-blocks
# Conflicts:
#	src/main/java/org/qortal/settings/Settings.java
2021-05-10 09:09:41 +01:00
CalDescent
83e2b10904 Merge branch 'ignore-old-versions' 2021-05-10 09:01:04 +01:00
CalDescent
26c1793d85 Added "allowConnectionsWithOlderPeerVersions" setting (default: true)
This controls whether to allow connections with peers below minPeerVersion.

If true, we won't sync with them but they can still sync with us, and will show in the peers list. This is the default, which allows older nodes to continue functioning, but prevents them from interfering with the sync behaviour of updated nodes.

If false, sync will be blocked both ways, and they will not appear in the peers list at all.
2021-05-10 09:00:42 +01:00
CalDescent
23a9eea26b Merge branch 'ignore-old-versions' 2021-05-09 23:02:35 +01:00
CalDescent
af9b536dd9 Moved version check above getMinBlockchainPeers() check, so that nodes with old versions aren't counted. 2021-05-09 23:00:51 +01:00
CalDescent
e4874f86f9 Merge branch 'block-timings' of github.com:Qortal/qortal into block-timings
# Conflicts:
#	src/main/java/org/qortal/api/model/BlockMintingInfo.java
#	src/main/java/org/qortal/api/resource/BlocksResource.java
#	tools/block-timings.sh
2021-05-09 19:25:33 +01:00
CalDescent
e300a957e4 Added online accounts count to /blocks/byheight/{height}/mintinginfo API and block-timings.sh script. 2021-05-09 19:25:05 +01:00
CalDescent
1c38afcd25 Slight reordering of vars. 2021-05-09 19:24:25 +01:00
CalDescent
a06faa7685 Updated usage info to reflect the fact that the "count" parameter is optional.
Usage:

block-timings.sh <startheight> [count] [target] [deviation] [power]
2021-05-09 19:24:25 +01:00
CalDescent
019ab2b21d Added tools/block-timings-sh which can be used to test out new block timings (specified in blockchain.json).
The script will fetch a set of blocks and then backtest the specified blockTimings settings (target, deviation, and power) against those real life blocks. This allows configurations to be fine tuned to tighten up block times, and to adjust the timestamp variance between levels.

Usage:
block-timings.sh <startheight> <count> [target] [deviation] [power]

startheight: a block height, preferably within the untrimmed range, to avoid data gaps
count: the number of blocks to request and analyse after the start height. Default: 100
target: the target block time in milliseconds. Originates from blockchain.json. Default: 60000
deviation: the allowed block time deviation in milliseconds. Originates from blockchain.json. Default: 30000
power: used when transforming key distance to a time offset. Originates from blockchain.json. Default: 0.2
2021-05-09 19:24:25 +01:00
CalDescent
f6ba5f5d51 Added /blocks/byheight/{height}/mintinginfo API, which returns info on the minter level, key distance, and block timings. 2021-05-09 19:24:25 +01:00
CalDescent
c4cbb64643 Added "minPeerVersion" setting, and avoid syncing with peers on lower versions. 2021-05-09 17:38:07 +01:00
CalDescent
8260cec713 Added "maximumCount" parameter to HSQLDBATRepository.getMatchingFinalATStatesQuorum() and use it to limit the number of ATs being returned in the query.
Initially set to 10 when used by the /crosschain/price/{blockchain} API, so that the price is based on the last 10 trades rather than every trade that has ever taken place.
2021-05-09 15:56:15 +01:00
CalDescent
428af3c0e8 Only use fast sync on trimmed blocks, as others are too large.
This could probably be improved to make sure that all blocks in the next request are within the trimmed time range, but it's enough for now.
2021-05-09 13:57:47 +01:00
CalDescent
68544715bf Skip Block.logDebugInfo() altogether if the log level is more specific than DEBUG, to avoid wasting resources. 2021-05-09 09:05:51 +01:00
CalDescent
d2ea5633fb Fixed divide by zero exception.
Block.calcKeyDistance() cannot be called on some trimmed blocks, because the minter level is unable to be inferred in some cases. This generally hasn't been an issue, but the new Block.logDebugInfo() method is invoking it for all blocks. For now I am adding defensiveness to the debug method, but longer term we might want to add defensiveness to Block.calcKeyDistance() itself, if we ever encounter this issue again. I will leave it alone for now, to reduce risk.
2021-05-09 09:05:42 +01:00
CalDescent
f4520e2752 Skip Block.logDebugInfo() altogether if the log level is more specific than DEBUG, to avoid wasting resources. 2021-05-09 09:00:53 +01:00
CalDescent
475802afbc Fixed divide by zero exception.
Block.calcKeyDistance() cannot be called on some trimmed blocks, because the minter level is unable to be inferred in some cases. This generally hasn't been an issue, but the new Block.logDebugInfo() method is invoking it for all blocks. For now I am adding defensiveness to the debug method, but longer term we might want to add defensiveness to Block.calcKeyDistance() itself, if we ever encounter this issue again. I will leave it alone for now, to reduce risk.
2021-05-09 08:25:24 +01:00
CalDescent
3aa9b5f0b6 Increased timeout when syncing multiple blocks from 4s to 10s. 2021-05-08 22:36:41 +01:00
CalDescent
6c5dbf7bd0 Preliminary version for fast sync set to 1.6.0, because 1.5.x is already released. 2021-05-08 16:36:42 +01:00
CalDescent
3b3dc5032b Merge branch 'master' into sync-multiple-blocks
# Conflicts:
#	pom.xml
#	src/main/java/org/qortal/controller/Synchronizer.java

Removed all fast sync code from Controller.syncToPeerChain(), so it is now the same as `master`.
2021-05-08 16:35:09 +01:00
Tom
a170668d9d Updated AdvancedInstaller project for v1.5.1 2021-05-07 09:58:15 +01:00
Tom
f8dac39076 Updated AdvancedInstaller project for v1.5.0
This includes updating AdoptOpenJDK to version 11.0.11.9, because 11.0.6.10 is no longer recommended or available in their archive. It also looks like I am using a newer version of AdvancedInstaller itself.
2021-05-07 09:40:38 +01:00
CalDescent
fe4ae61552 Added "maxRetries" setting.
This controls the maximum number of retry attempts if a peer fails to respond with the requested data.
2021-05-06 17:49:45 +01:00
CalDescent
0c3597f757 Bump version to 1.5.1 2021-05-05 18:41:05 +01:00
CalDescent
6109bdeafe Set go-live timestamp for same-length chain weight consensus: 1620579600000 2021-05-05 18:40:07 +01:00
CalDescent
6e9a61c4e5 Fixed logging issue where it would underreport the number of common blocks found when loading some from the cache. 2021-05-02 20:51:53 +01:00
CalDescent
8e244fd956 Fixed yet another bug with minChainLength. 2021-05-02 20:45:20 +01:00
CalDescent
2eb6771963 Adapted logging in comparePeers() to report correct values for both chain weight algorithms. 2021-05-02 20:26:51 +01:00
CalDescent
db77108054 Log the number of blocks used in Block.calcChainWeight()
This makes it easier to check that the new consensus code is being used, and that it is working correctly.
2021-05-02 19:59:32 +01:00
CalDescent
241e2bef85 Merge branch 'master' into chain-weight-consensus
# Conflicts:
#	src/main/java/org/qortal/block/BlockChain.java
#	src/main/resources/blockchain.json
#	src/test/resources/test-chain-v2-founder-rewards.json
#	src/test/resources/test-chain-v2-leftover-reward.json
#	src/test/resources/test-chain-v2-minting.json
#	src/test/resources/test-chain-v2-qora-holder-extremes.json
#	src/test/resources/test-chain-v2-qora-holder.json
#	src/test/resources/test-chain-v2-reward-scaling.json
#	src/test/resources/test-chain-v2.json
2021-05-02 18:18:20 +01:00
CalDescent
fac02dbc7d Fixed bug in maxHeight parameter passed to Block.calcChainWeight()
Like the others, this one is only relevant after switching to same-length chain weight comparisons.
2021-05-02 15:56:13 +01:00
CalDescent
9ebcd55ff5 Fixed calculation error in existing chain weight code, which would have caused the last block to be missed out of the comparison after switching to same-length chain comparisons. 2021-05-01 13:34:13 +01:00
CalDescent
50244c1c40 Fixed bug which would cause other peers to not be compared against each other, if we had no blocks ourselves.
Again, this wouldn't have affected anything in 1.5.0 or before, but it will become more significant if we switch to same-length chain weight comparisons.
2021-05-01 13:32:16 +01:00
CalDescent
b4395fdad1 Fixed bug which could cause minChainLength to report a higher value.
This wouldn't have affected anything in 1.5.0, but it will become more significant if we switch to same-length chain weight comparisons.
2021-05-01 10:57:24 +01:00
CalDescent
1da8994be7 Log the block timestamp, minter level, online accounts, key distance, and weight, when orphaning or processing.
This gives an insight into the contents of each chain when doing a re-org. To enable this logging, add the following to log4j2.properties:

logger.block.name = org.qortal.block.Block
logger.block.level = debug
2021-05-01 10:24:50 +01:00
QuickMythril
55ff1e2bb1 updated and tested BTC electrum servers (#36)
* updated electrum servers

mainnet list: https://1209k.com/bitcoin-eye/ele.php?chain=btc
testnet list: https://1209k.com/bitcoin-eye/ele.php?chain=tbtc

* removed servers

tested each mainnet server individually and removed those that did not respond
2021-05-01 09:18:46 +01:00
CalDescent
5fd8528c49 Small refactor for code readability, and added some defensiveness to avoid possible NPEs. 2021-04-29 09:04:59 +01:00
CalDescent
26d8ed783a Same as commit c0c5bf1, but for blocks as well as block summaries. 2021-04-29 08:55:16 +01:00
CalDescent
c0c5bf1591 Apply blocks in syncToPeerChain() if the latest received block is newer than our latest, and we started from an out of date chain.
This solves a common problem that is mostly seen when starting a node that has been switched off for some time, or when starting from a bootstrap. In these cases, it can be difficult get synced to the latest if you are starting from a small fork. This is because it required that the node was brought up to date via a single peer, and there wasn't much room for error if it failed to retrieve a block a couple of times. This generally caused the blocks to be thrown away and it would try the same process over and over.

The solution is to apply new blocks if the most recently received block is newer than our current latest block. This gets the node back on to the main fork where it can then sync using the regular applyNewBlocks() method.
2021-04-28 22:03:13 +01:00
CalDescent
c17a481b74 Bump version to 1.5.0 2021-04-26 18:34:01 +01:00
CalDescent
a9a0e69ec0 Set go-live block height for share bin fix: block 399000 2021-04-26 17:19:39 +01:00
CalDescent
ea1fed2fd3 Merge branch 'block-reward-distribution-fix' 2021-04-26 17:16:14 +01:00
CalDescent
b37f2c7d7f MAXIMUM_RETRIES set to 2, as 3 retries may have been slightly too many. 2021-04-26 17:08:21 +01:00
CalDescent
0c0c5ff077 Invalidate our block summaries cache for a peer if it fails to respond with signatures when synchronizing. 2021-04-25 12:50:40 +01:00
CalDescent
e12b99d17e Invalidate our common block cache for a peer if we can't find a common block when synchronizing. 2021-04-25 09:37:32 +01:00
CalDescent
d599146c3a Cache peer block summaries to avoid duplicate requests when comparing peers. 2021-04-24 22:10:40 +01:00
CalDescent
476731a2c3 In syncToPeerChain(), only apply a partial set of peer's blocks if they are recent.
If a peer fails to reply with all requested blocks, we will now only apply the blocks we have received so far if at least one of them is recent. This should prevent or greatly reduce the scenario where our chain is taken from a recent to an outdated state due to only partially syncing with a peer. It is best to keep our chain "recent" if possible, as this ensures that the peer selection code always runs, and therefore avoids unnecessarily syncing to a random peer on an inferior chain.
2021-04-24 20:12:11 +01:00
CalDescent
1e491dd8fb MAXIMUM_RETRIES increased from 1 to 3.
Now that we are spending a lot of time to carefully select a peer to sync with, it makes sense to retry a couple more times before giving up and starting the peer selection process all over again.
2021-04-24 19:45:53 +01:00
CalDescent
ba6397b963 Improved logging, to give a clearer picture of the peer selection decisions. 2021-04-24 19:23:09 +01:00
CalDescent
3146da6aec Don't add to the inferior chain signatures list when comparing peers against each other.
In these comparisons it's easy to incorrectly identify a bad chain, as we aren't comparing the same number of blocks. It's quite common for one peer to fail to return all blocks and be marked as an inferior chain, yet we have other "good" peers on that exact same chain. In those cases we would have stopped talking to the good peers again until they received another block.

Instead of complicating the logic and keeping track of the various good chain tip signatures, it is simpler to just remove the inferior peers from this round of syncing, and re-test them in the next round, in case they are in fact superior or equal.
2021-04-24 16:43:29 +01:00
CalDescent
5643e57ede Fixed string formatting error. 2021-04-24 16:21:04 +01:00
CalDescent
f532dbe7b4 Optimized code in Synchronizer.uniqueCommonBlocks() 2021-04-24 15:22:29 +01:00
CalDescent
ec2af62b4d Fix for bug which failed to remove peers without block summaries.
The iterator was removing the peer from the "peersSharingCommonBlock" array, when it should have been removing it from the "peers" array. The result was that the bad peer would end up in the final list of good peers, and we could then sync with it when we shouldn't have.
2021-04-24 15:21:30 +01:00
CalDescent
423142d730 Tidied up RECOVERY_MODE_TIMEOUT constant, and made checkRecoveryModeForPeers() private. 2021-04-24 10:35:01 +01:00
CalDescent
bdddb526da Added recovery mode, which is designed to automatically bring back a stalled network.
The existing system was unable to resume without manual intervention if it stalled for more than 7.5 minutes. After this time, no peers would have "recent' blocks, which are prerequisites for synchronization and minting.

This new code monitors for such a situation, and enters "recovery mode" if there are no peers with recent blocks for at least 10 minutes. It also requires that there is at least one connected peer, to reduce false positives due to bad network connectivity.

Once in recovery mode, peers with no recent blocks are added back into the pool of available peers to sync with, and restrictions on minting are lifted. This should allow for peers to collaborate to bring the chain back to a "recent" block height. Once we have a peer with a recent block, the node will exit recovery mode and sync as normal.

Previously, lifting minting restrictions could have increased the risk of extra forks, however it is much less risky now that nodes no longer mint multiple blocks in a row.

In all cases, minBlockchainPeers is used, so a minimum number of connected peers is required for syncing and minting in recovery mode, too.
2021-04-23 09:21:15 +01:00
CalDescent
dbf1ed40b3 Log the parent block's signature when minting a new block, to help us keep track of the chain it's being minted on. 2021-04-19 09:33:24 +01:00
CalDescent
02ace06526 Revert "When syncing to a peer on a different fork, ensure that all blocks are obtained before applying them."
This reverts commit c919797553.
2021-04-18 13:03:04 +01:00
CalDescent
2d2bfc0a4c Log the number of common blocks found in each search. 2021-04-18 13:02:38 +01:00
CalDescent
3c22a12cbb Experimental idea to prevent a single node signing more than one block in a row.
This could drastically reduce the number of forks being created. Currently, if a node is having problems syncing, it will continue adding to its own fork, which adds confusion to the network. With this new idea, the node would be prevented from adding to its own chain and is instead forced to wait until it has retrieved the next block from the network.

We will need to test this on the testnet very carefully. My worry is that, because all minters submit blocks, it could create a situation where the first block is submitted by everyone, and the second block is submitted by no-one, until a different candidate for the first block has been obtained from a peer. This may not be a problem at all, and could actually improve stability in a huge way, but at the same time it has the potential to introduce serious network problems if we are not careful.
2021-04-18 10:26:36 +01:00
CalDescent
3071ef2f36 Removed redundant uiLocalServers 2021-04-17 20:55:30 +01:00
CalDescent
3022cb22d6 Merge branch 'master' into prioritize-peers 2021-04-17 20:51:35 +01:00
CalDescent
e9b4a3f6b3 Automatically backup trade bot data when starting a new trade (from either side). 2021-04-17 20:45:35 +01:00
CalDescent
4312ebfcc3 Adapted the HSQLDBRepository.exportNodeLocalData() method
It now has a new parameter - keepArchivedCopy - which when set to true will cause it to rename an existing TradeBotStates.script to TradeBotStates-archive-<timestamp>.script before creating a new backup. This should avoid keys being lost if a new backup is taken after replacing the db.

In a future version we can improve this in such a way that it combines existing and new backups into a single file. This is just a "quick fix" to increase the chances of keys being recoverable after accidentally bootstrapping without a backup.
2021-04-17 20:44:57 +01:00
CalDescent
2c0e099d1c Removed wildcard import that was automatically introduced by Intellij. 2021-04-17 14:36:24 +01:00
CalDescent
b1eb02eb1d Merge pull request #33 from QuickMythril/version-on-tooltip
add version on tooltip
2021-04-17 13:21:20 +01:00
CalDescent
c919797553 When syncing to a peer on a different fork, ensure that all blocks are obtained before applying them.
In version 1.4.6, we would still sync with a peer even if we only received a partial number of the requested blocks/summaries. This could create a new problem, because the BlockMinter would often try and make up the difference by minting a new fork of up to 5 blocks in quick succession. This could have added to network confusion.

Longer term we may want to adjust the BlockMinter code to prevent this from taking place altogether, but in the short term I will revert this change from 1.4.6 until we have a better way.
2021-04-17 13:09:52 +01:00
CalDescent
08dacab05c Make sure to give up if we are requesting block summaries when the core needs to shut down. 2021-04-17 12:57:28 +01:00
CalDescent
2efc9218df Improved the process of selecting the next peer to sync with
Added a new step, which attempts to filter out peers that are on inferior chains, by comparing them against each other and our chain. The basic logic is as follows:

1. Take the list of peers that we'd previously have chosen from randomly.
2. Figure out our common block with each of those peers (if its within 240 blocks), using cached data if possible.
3. Remove peers with no common block.
4. Find the earliest common block, and compare all peers with that common block against each other (and against our chain) using the chain weight method. This involves fetching (up to 200) summaries from each peer after the common block, and (up to 200) summaries from our own chain after the common block.
5. If our chain was superior, remove all peers with this common block, then move up to the next common block (in ascending order), and repeat from step 4.
6. If our chain was inferior, remove any peers with lower weights, then remove all peers with higher common blocks.
7. We end up with a reduced list of peers, that should in theory be on superior or equal chains to us. Pick one of those at random and sync to it.

This is a high risk feature - we don't yet know the impact on network load. Nor do we know whether it will cause issues due to prioritising longer chains, since the chain weight algorithm currently prefers them.
2021-04-17 12:52:19 +01:00
CalDescent
41505dae11 Treat two block summaries as equal if they have matching signatures 2021-04-16 09:40:22 +01:00
CalDescent
45efe7cd56 Slight reordering of vars. 2021-04-10 18:24:33 +01:00
CalDescent
78cac7f0e6 Updated usage info to reflect the fact that the "count" parameter is optional.
Usage:

block-timings.sh <startheight> [count] [target] [deviation] [power]
2021-04-10 18:12:09 +01:00
CalDescent
a1a1b8e94a Added tools/block-timings-sh which can be used to test out new block timings (specified in blockchain.json).
The script will fetch a set of blocks and then backtest the specified blockTimings settings (target, deviation, and power) against those real life blocks. This allows configurations to be fine tuned to tighten up block times, and to adjust the timestamp variance between levels.

Usage:
block-timings.sh <startheight> <count> [target] [deviation] [power]

startheight: a block height, preferably within the untrimmed range, to avoid data gaps
count: the number of blocks to request and analyse after the start height. Default: 100
target: the target block time in milliseconds. Originates from blockchain.json. Default: 60000
deviation: the allowed block time deviation in milliseconds. Originates from blockchain.json. Default: 30000
power: used when transforming key distance to a time offset. Originates from blockchain.json. Default: 0.2
2021-04-10 17:57:28 +01:00
CalDescent
641a658059 Added /blocks/byheight/{height}/mintinginfo API, which returns info on the minter level, key distance, and block timings. 2021-04-10 17:49:04 +01:00
CalDescent
44ec447014 Show an error in publish-auto-update.pl if both sha256sum and sha256 aren't found in PATH. 2021-04-01 08:27:56 +01:00
CalDescent
98308ecf98 Bump version to 1.4.6 2021-04-01 08:09:50 +01:00
CalDescent
8d613a6472 MAXIMUM_RETRIES reduced from 3 to 1 2021-03-30 13:07:34 +01:00
CalDescent
c3e5298ecd Added a few checks for Controller.isStopping() in synchronizer loops, to try and speed up the shutdown time. 2021-03-30 13:05:43 +01:00
CalDescent
e89d31eb5a Rewrite of Synchronizer.syncToPeerChain(), this time borrowing ideas from Synchronizer.applyNewBlocks().
Main differences / improvements:
- Only request a single batch of signatures upfront, instead of the entire peer's chain. There is no point in requesting them all, as the later ones may not be valid by the time we have finished requesting all the blocks before them.
- If we fail to fetch a block, clear any queued signatures that are in memory and re-fetch signatures after the last block received. This allows us to cope with peers that re-org whilst we are syncing with them.
- If we can't find any more block signatures, or the peer fails to respond to a block, apply our progress anyway. This should reduce wasted work and network congestion, and helps cope with larger peer re-orgs.
- The retry mechanism remains in place, but instead of fetching the same incorrect block over and over, it will attempt to locate a new block signature each time, as described above. To help reduce code complexity, block signature requests are no longer retried.
2021-03-30 12:29:27 +01:00
CalDescent
08f3d653cc Added new settings "maxBlocksPerRequest" and "maxBlocksPerResponse", to control the number of blocks requested and returned by nodes when using GetBlocksMessage and BlocksMessage. 2021-03-28 17:27:04 +01:00
CalDescent
f2bbafe6c2 Added missing break statement if a peer fails to respond with blocks when resolving a fork via fast sync. 2021-03-28 16:52:15 +01:00
CalDescent
cb80280eaf Bump Peer response timeout from 3s to 4s 2021-03-28 14:47:57 +01:00
CalDescent
f22f954ae3 Use MAXIMUM_BLOCKS_REQUEST_SIZE for GetBlocksMessage and BlockMessage, instead of MAXIMUM_REQUEST_SIZE.
Currently set to 1, as serialization of the BlocksMessage data on mainnet is too slow to use this for any significant number of blocks right now. Hopefully we can find a way to optimise this process, which will allow us to use this for multiple block syncing.

Until then, sticking with single blocks should still be enough to help solve the network congestion and re-orgs we are seeing, because it gives us the ability to request the next block based on the previous block's signature, which was unavailable using GET_BLOCK. This removes the requirement to fetch all block signatures upfront, and therefore it shouldn't matter if the peer does a partial re-org whilst a node is syncing to it.
2021-03-28 14:35:47 +01:00
CalDescent
2556855bd7 Added missing return statement if a peer fails to respond with blocks when fast syncing. 2021-03-28 14:19:53 +01:00
CalDescent
365662a2af MAXIMUM_RETRIES reduced from 3 to 1. It will now only retry once, which should save around 6 seconds of wasted synchronization time if a node is unable to respond with the requested block (due to a re-org, etc). 2021-03-27 19:25:27 +00:00
CalDescent
3e0ff7f43f Split Synchronizer.applyNewBlocks() into two methods.
If fast syncing is enabled in the settings (by default it's disabled) AND the peer is running at least v1.5.0, then it will route through to a new method which fetches multiple blocks at a time, in a very similar way to Synchronizer.syncToPeerChain().
If fast syncing is disabled in the settings, or we are communicating with a peer on an older version, it will continue to sync blocks individually.
2021-03-27 18:04:47 +00:00
CalDescent
8c3753326f Check "isFastSyncEnabledWhenResolvingFork" setting before requesting multiple blocks from a peer. This allows users to opt out of this functionality, by setting it to false in their settings. 2021-03-27 18:00:39 +00:00
CalDescent
dbcf6de2d5 Added new settings "fastSyncEnabled" (default: false) and "fastSyncEnabledWhenResolvingFork" (default: true). 2021-03-27 17:59:23 +00:00
CalDescent
a5308995b7 Bump version to 1.5.0, to allow nodes to start using the new syncing method. 2021-03-27 15:46:30 +00:00
CalDescent
270ac88b51 Added GetBlocksMessage and BlocksMessage, which allow multiple blocks to be transferred between peers in a single request.
When communicating with a peer that is running at least version 1.5.0, it will now sync multiple blocks at once in Synchronizer.syncToPeerChain(). This allows us to bypass the single block syncing (and retry mechanism), which has proven to be unviable when there are multiple active forks with several blocks in each chain.

For peers below v1.5.0, the logic should remain unaffected and it will continue to sync blocks individually.
2021-03-27 15:45:15 +00:00
CalDescent
30160e2843 Fixes to allow publish-auto-update.sh to work with sha256sum versions that add trailing characters. 2021-03-21 18:15:29 +00:00
catbref
503d22e4d0 Updated Qortal.aip for WindowsInstaller for v1.4.5 2021-03-21 18:05:38 +00:00
CalDescent
b9a0d489d7 Bump version to 1.4.5 2021-03-21 17:06:10 +00:00
catbref
d9d4c4c302 Bump Peer response timeout from 2s to 3s 2021-03-21 16:17:40 +00:00
catbref
81c6d75d62 Adjust Synchronizer.MAXIMUM_BLOCK_STEP to 128, which means final summaries request will have enough to cover MAXIMUM_COMMON_DELTA (8+16+32+64+128 = 248, which is >240) 2021-03-21 16:12:41 +00:00
catbref
d1419bdfbd Minor comments, adjust max step size when searching for common block 2021-03-21 15:57:00 +00:00
CalDescent
8566d9b7e5 Merge branch 'master' into synchronization-improvements 2021-03-21 15:04:43 +00:00
catbref
b319d6db6b Rework BlockMessage caching with new pseudo outgoing-only message that only caches raw bytes 2021-03-21 14:14:15 +00:00
CalDescent
35fd1d8455 Base58 encode signatures in recently added logs. 2021-03-21 14:12:04 +00:00
CalDescent
be21771e49 Use SYNC_BATCH_SIZE instead of MAXIMUM_BLOCK_SIGNATURES_PER_REQUEST. 2021-03-21 13:58:42 +00:00
catbref
745528a9b1 Peer.sendMessage() should return false when it can't send because it can't build the message 2021-03-21 13:19:59 +00:00
CalDescent
f1422af95b Added retry mechanisms in Synchronizer.syncToPeerChain()
Until now, we required a perfect success rate when syncing with a peer via Synchronizer.syncToPeerChain(). Blocks were requested individually, but the node would give up and lose all progress if a single request failed. In practice, this happened very regularly, and it was difficult to succeed when there were a large number of blocks (e.g. 20+) that needed to be requested.

This commit adds two retry mechanisms, causing each of the two request types (block sigs and blocks) to retry 3 times before giving up, potentially avoiding a lot of wasted work. The number of retries is configurable in the MAXIMUM_RETRIES constant, which we could move to settings at some point if this feature proves useful.

The original issue seemed to result in a few side effects:

1. Nodes would spend a large amount of time requesting blocks from peers, only to throw it all away afterwards. This potentially added to network congestion, as nodes were using unnecessary network time to unproductively serve peers.

2. A large number of sync attempts were failing, particularly when a fork emerged with a significant number of divergent blocks (20+). This issue reduced the ability for nodes to sync to the correct chain while they still had time to do so. With every block that passed, it became made it more and more difficult to switch to the correct chain. Eventually, the correct chain would become TOO_DIVERGENT at which point there is no way to automatically switch without manual intervention. I hope that this retry mechanism will increase the chances of nodes automatically moving onto the right chain quickly, avoiding the need for a user to intervene.

3. The POST /admin/forcesync API was unlikely to succeed when the peer's chain had started to diverge from the user's chain. This should increase the success rate.

Also included in this commit is a MAXIMUM_BLOCK_SIGNATURES_PER_REQUEST constant. This limits the number of block sigs requested in each batch (default 200). Without this, we are unable to increase MAXIMUM_COMMON_DELTA because it can try and request thousands of block sigs at once, which unsurprisingly doesn't succeed.
2021-03-21 09:41:36 +00:00
CalDescent
f92f4dc1e2 Fixed some log entries in Controller.syncToPeerChain() which were incorrectly reporting our height instead of the height of block(s) being requested from the peer. Now reporting the height of the block (or block sigs) being retrieved, which should make it easier to interpret the logs. 2021-03-20 16:18:25 +00:00
catbref
019cfdc1db Minor comment re-org 2021-03-20 11:45:11 +00:00
CalDescent
e694a51cdd Fix for "numberSignaturesRequired" calculation error in Synchronizer.syncToPeerChain()
This bug often prevented the correct amount of block signatures (and blocks) from being requested from a peer, when trying to sync to it.

It could result in quite serious consequences, as it would trigger orphaning back to the common block without first requesting all of the necessary blocks from the peer's chain. Rather than applying a complete copy of the peer's chain, it could orphan back to the common block and then only apply a few blocks beyond that, leaving the node in an unexpected state, potentially hundreds of blocks behind the peer's current height, which it then has to try and obtain from other peers.

When there are forks present, this could result in it hopping from chain to chain, each time being unable to fully synchronise with the peer. Given that we currently discard our chain if it is deemed that our latest block isn't "recent", it is very important that nodes are brought up to the latest block when synchronising with a peer, to avoid constantly triggering discards.

The severity of this bug increased when there was a large disparity between the peer's latest block and the common block height, and prevented us from being able to increase MAXIMUM_COMMON_DELTA.
2021-03-20 10:33:23 +00:00
CalDescent
16453ed602 Added unit tests for level 3+4, 5+6, 7+8, and 9+10 rewards.
These are simpler than the level 1+2 tests; they only test that the rewards are correct for each level post-shareBinFix. I don't think we need multiple instances of the pre-shareBinFix or block orphaning tests. There are a few subtle differences between each test, such as the online status of Bob, in order the make the tests slightly more comprehensive.
2021-03-17 08:50:53 +00:00
CalDescent
fde68dc598 Added unit test to test level 1 and 2 rewards.
1. Assign 3 minters (one founder, one level 1, one level 2)
2. Mint a block after the shareBinFix, ensuring that level 1 and 2 are being rewarded evenly from the same share bin.
3. Orphan the block and ensure the rewards are reversed.
4. Orphan two more blocks, each time checking that the balances are being reduced in accordance with the pre-shareBinFix mapping.
2021-03-16 09:11:49 +00:00
QuickMythril
22e3140ff0 add version on tooltip
add Version Number on Qortal Core tooltip.

https://i.imgur.com/eLnLnQ5.png
2021-03-16 03:00:55 -04:00
catbref
4824c4198b Bump version to 1.4.4 2021-03-15 11:00:20 +00:00
catbref
ec7d4f4498 Changed "too busy" logging from debug to trace 2021-03-13 18:30:43 +00:00
catbref
d635de44a8 Added TODO in HSQLDBRepository about deadlock log spam 2021-03-13 18:29:31 +00:00
catbref
bce66bf57f Move HSQLDBRepositoryFactory.POOL_SIZE into Settings as "repositoryConnectionPoolSize" 2021-03-13 18:14:11 +00:00
catbref
0fc5153f9b Merge 'trade-bot-timeout-fix' into master 2021-03-13 17:13:40 +00:00
catbref
0398c2fae1 Try to avoid clogging up network threads by discarding incoming TRANSACTION messages if we're too busy
As importing a transaction requires blockchain lock, all the network threads
can be used up blocking for that lock, especially if Synchronizer is active.

So we simply discard incoming TRANSACTION messages if we can't immediately
obtain the blockchain lock. Some other peer will probably attempt to
send the transaction soon again anyway.

Plus we swap transaction lists after connection handshake.
2021-03-13 17:03:38 +00:00
CalDescent
5fc495eb6a Fix for possible logic bug introduced in commit 33a8f31. 2021-03-12 22:05:38 +00:00
CalDescent
847e81e95c Fixed a mapping issue in Block->getShareBins(), to take effect at some future (undecided) height.
Post trigger, account levels will map correctly to share bins, subtracting 1 to account for the 0th element of the shareBinsByLevel array.
Pre-trigger, the legacy mapping will remain in effect.
2021-03-12 19:48:49 +00:00
CalDescent
7918622e2e Merge pull request #31 from sakumatto/master
Initial Italian translation by Pabs 2021
2021-03-11 11:06:03 +00:00
CalDescent
427fa1816d "blockCacheSize" can now be configured via settings.json. 2021-03-07 10:00:49 +00:00
catbref
0c7e388463 Bump to v1.4.3 2021-02-27 18:24:09 +00:00
catbref
be3af53011 Set new block sig go-live block height: block 320000 2021-02-27 18:23:49 +00:00
catbref
414399b2a0 Merge branch 'blocksig' into master 2021-02-27 18:20:13 +00:00
catbref
c592051a80 Speed up BlockMinter by filtering out 'unconfirmable' transaction types like CHAT & PRESENCE 2021-02-27 17:29:19 +00:00
catbref
33a8f311e5 Reduce logging noise from lost trade-bot ATs and self-clean if AT does not exist after 24 hours 2021-02-24 21:00:52 +00:00
catbref
018c3cdcd4 Allow users to delete trade-bot entries in any state if corresponding AT does not exist 2021-02-24 20:46:47 +00:00
sakumatto
384dffbf9a Initial Italian translation by Pabs 2021
UI localized to Italian by @Pabs
2021-02-22 20:03:11 +02:00
catbref
0306ecb03d AdvancedInstaller updates for v1.4.2 2021-02-21 17:26:32 +00:00
catbref
e5ce732557 More detail in AutoUpdates.md 2021-02-21 17:12:02 +00:00
catbref
f19e0498bf Bump to v1.4.2 2021-02-21 17:06:23 +00:00
CalDescent
e505067759 Added support for Native SegWit (Bech32) addresses, which have the prefixes "ltc1" and "bc1". Bitcoinj supports these automatically, as long as fromString() is used instead of fromBase58(), which is already the case. Tested on LTC mainnet, and BTC testnet. 2021-02-21 12:02:26 +00:00
CalDescent
32ec02225a Added optional "--testnet" or "-t" argument to stop.sh. When passing this argument, it will attempt to stop the core using the default testnet port (62391) rather than the default mainnet port (12391). 2021-02-21 11:50:02 +00:00
catbref
3920933fc7 Add block fetch TRACE logging to Synchronizer 2021-02-20 13:36:54 +00:00
catbref
1fdd7f156c Reduced logging noise from deleteExpiredTransactions but increased detection & logging on "serilization failures" from HSQLDB 2021-02-20 13:25:53 +00:00
catbref
91925cf931 Change block "minter" signature code, to take effect at some future (undecided) height.
Post trigger, this change will use all 128 bytes of previous block's signature when
calculating/validating next block's "minter" signature (itself the first 64 bytes of a block signature).

Prior to trigger, current behaviour is to only use first 64 bytes of previous block's
signature, which doesn't encompass transactions signature.

New block sig code should help reduce forking and help improve transactional
security.

Added "newBlockSigHeight" to blockchain.json but initially set to block 999999
pending decision on when to merge, auto-update, go-live, etc.
2021-02-20 12:08:51 +00:00
catbref
30e58f1c19 Merge branch 'hsqldb-checkpoint' into master 2021-02-20 10:56:22 +00:00
CalDescent
8d5c6db39f Exclude IntelliJ IDEA files from git. 2021-02-14 18:17:22 +00:00
catbref
3453f0efaf Rework HSQLDB CHECKPOINTing to defer until there are no ongoing SQL transactions, in order to prevent DB deadlocks.
Symptoms of a CHECKPOINT-related DB deadlock:

On Controller thread:
"Controller" #20 prio=5 os_prio=31 cpu=1577665.56ms elapsed=17666.97s allocated=475G defined_classes=412 tid=0x00007fe99f97b000 nid=0x1644b waiting on condition  [0x0000700009a21000]
   java.lang.Thread.State: WAITING (parking)
	at jdk.internal.misc.Unsafe.park(java.base@14.0.2/Native Method)
	- parking to wait for  <0x0000000602f2a6f8> (a org.hsqldb.lib.CountUpDownLatch$Sync)
[...some more lines...]
[this next line is the best indicator: ]
	at org.qortal.repository.hsqldb.HSQLDBRepository.checkpoint(HSQLDBRepository.java:385)
	at org.qortal.repository.RepositoryManager.checkpoint(RepositoryManager.java:51)
	at org.qortal.controller.Controller.run(Controller.java:544)

Other threads stuck at:
	- parking to wait for  <0x00000007ff09f0b0> (a org.hsqldb.lib.CountUpDownLatch$Sync)
	at java.util.concurrent.locks.LockSupport.park(java.base@14.0.2/LockSupport.java:211)
	at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(java.base@14.0.2/AbstractQueuedSynchronizer.java:714)
	at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(java.base@14.0.2/AbstractQueuedSynchronizer.java:1046)
	at org.hsqldb.lib.CountUpDownLatch.await(Unknown Source)
	at org.hsqldb.Session.executeCompiledStatement(Unknown Source)
2021-02-13 17:32:09 +00:00
catbref
eb23940996 Fix potential NPE when trying to obtain opportunistic database connection.
Could have affected:
Controller.deleteExpiredTransactions()
Network.getConnectablePeer()
Network.opportunisticMergePeers()
Network.prunePeers()

Symptoms:

2021-02-12 16:46:06 WARN  NetworkProcessor:152 - [1556] exception while trying to produce task
java.lang.NullPointerException: null
        at org.qortal.repository.hsqldb.HSQLDBRepository.<init>(HSQLDBRepository.java:92) ~[qortal.jar:1.4.1]
        at org.qortal.repository.hsqldb.HSQLDBRepositoryFactory.tryRepository(HSQLDBRepositoryFactory.java:97) ~[qortal.jar:1.4.1]
        at org.qortal.repository.RepositoryManager.tryRepository(RepositoryManager.java:33) ~[qortal.jar:1.4.1]
        at org.qortal.network.Network.getConnectablePeer(Network.java:525) ~[qortal.jar:1.4.1]
2021-02-13 11:24:26 +00:00
catbref
6cd86d86a6 Add "atFindNextTransactionFix" feature-trigger to all test blockchain configs 2021-02-13 11:24:26 +00:00
CalDescent
c3fa34f5b9 Fixed .gitignore to ensure that all .DS_Store files (autogenerated by macOS) are excluded. 2021-02-13 11:17:10 +00:00
CalDescent
0af0aaaa21 Merge pull request #29 from sakumatto/master
Initial translations of UI terms into Finnish
2021-02-13 11:12:24 +00:00
CalDescent
02100c502b Exit from stop.sh with an error if curl isn't installed. Based on code submitted by TRM13 in issue #28. 2021-02-12 16:13:01 +00:00
sakumatto
b55154cd3c Initial translations of UI terms into Finnish
Initial translations of UI terms into Finnish language fi_ by Saku Mättö
2021-02-12 17:02:31 +02:00
catbref
1e6e5e66da Fix trailing comma on blockchain.json! 2021-02-06 12:09:24 +00:00
catbref
9b0e88ca87 Only compare same number of blocks when comparing peer chains 2021-02-06 11:40:29 +00:00
catbref
3acc0babb7 More chain-weight tests 2021-02-06 11:19:39 +00:00
catbref
dc6eda1355 Added tool to help build release notes 2021-02-06 11:02:36 +00:00
catbref
6224bc3bca Updated AdvancedInstaller config file to latest Qortal v1.4.1 2021-02-06 11:00:32 +00:00
catbref
9ceac8c991 Documentation updates 2021-02-06 10:59:31 +00:00
catbref
834fcd80d7 Bump version to v1.4.1 2021-01-18 08:17:09 +00:00
catbref
20e4a79130 Reduce logging level for deleting older PRESENCE transactions 2021-01-17 16:49:47 +00:00
catbref
d336200d75 Fix for off-by-one bug when ATs look for next transaction. Currently configured to take effect block 275,000 2021-01-17 16:01:36 +00:00
catbref
e5bb3e2f0a Added defensive try-catch around network engine calls (actually ExecuteProduceConsume) 2021-01-17 15:34:50 +00:00
catbref
5b2b2bab46 Two-pronged fix for HSQLDB 'serialization failure' errors when receiving multiple PRESENCE transactions -- reported by marracc 2021-01-17 15:33:59 +00:00
catbref
c17eea3ed9 Added timeout to Peer sendMessage() - same timeout as for awaiting incoming responses 2021-01-17 15:32:39 +00:00
catbref
83f4e2f5bf Added API call to view single trade's detailed info 2021-01-17 15:31:44 +00:00
catbref
c8e7a00c08 Remove unused JDBC statement 2021-01-16 13:20:58 +00:00
catbref
190014cf96 Fix minor NPE during shutdown 2021-01-16 13:20:20 +00:00
catbref
385064e324 Return foreign-chain wallet transactions in newest-timestamp-first order 2021-01-08 17:37:12 +00:00
catbref
f3e1f088f8 Bump to v1.4.0 2021-01-07 07:49:13 +00:00
catbref
6eb9447bb9 Merge branch 'LTCv3-with-presence' into master 2021-01-07 07:45:47 +00:00
catbref
0ee8d7da0f Improve removal of expired PRESENCE txns in websocket cache 2020-12-31 11:26:13 +00:00
catbref
918a331609 Transaction.importAsUnconfirmed() now BLOCKS for blockchain lock, instead of non-blocking try()
This is a serious change as it affects many callers.

Controller.onNetworkTransactionMessager()
-- should be OK to block as it's only one EPC thread

TransactionsResource.processTransaction()
-- should be OK to block as it's only one Jetty thread
AND has its own 30s timeout wrapper anyway

Implementations of AcctTradeBot.progress()
e.g. BitcoinACCTv1TradeBot.progress() & LitecoinACCTv1TradeBot.progress()
TradeBot.updatePresence()
-- these are called via BlockMinter/Synchronizer
when blockchain lock is already held, so will are unaffected

AcctTradeBot.createTrade() and AcctTradeBot.startResponse()
-- these are called via API
and previously would only perform non-blocking blockchainLock.try()
but now perform blocking blockchainLock.lock()
thus preventing NO_BLOCKCHAIN_LOCK when creating/responding to trades
especially after a (wasted) MESSAGE PoW compute
but with potential downside that API response might be delayed
e.g. by a very slow sync round

Future work could look into removing the need for blockchain lock
when calling Transaction.importAsUnconfirmed().
2020-12-30 12:43:41 +00:00
catbref
78f62751e5 TESTNET ONLY: Correct "LitcoinACCTv1" TradeBotStates.acct_name values in DB 2020-12-29 14:16:47 +00:00
catbref
d59c30757c Fix conversion of double in ElectrumX JSON to long 2020-12-29 12:17:55 +00:00
catbref
30d2e4fdac Correct logging message to blockchain-agnostic text 2020-12-29 12:16:40 +00:00
catbref
d43a074cc1 Use *ACCT*.class.getSimpleName() for less error-prone ACCT.NAME 2020-12-29 11:00:43 +00:00
catbref
27783dc6de Add WARNING on start-up if repository is missing latest AT state data 2020-12-29 10:34:25 +00:00
catbref
e00579e1a2 Fix incorrect LatestATStates SQL table definition
This table was previously defined using the TEMPORARY keyword as the rows were used as a cached/index to speed up AT trimming SQL statements.

However, the table definition was lacking the "ON COMMIT PRESERVE ROWS" clause, and possibly the "GLOBAL" keyword, which caused the table contents to be emptied, immediately after being filled, when <tt>repository.saveChanges()</tt> was called (i.e. "COMMIT").

This caused latest AT states to be trimmed in error.

AtRepositoryTests.testGetLatestATStatePostTrimming also fixed so that it fails with the previous, broken code.
2020-12-29 09:11:11 +00:00
catbref
99f1a55de2 Improve logging for case where trade offer is locked to someone else 2020-12-29 08:56:26 +00:00
catbref
3ec307a2a1 Don't allow more than one (active) trade-bot entry per trade-offer 2020-12-28 15:55:14 +00:00
catbref
3fdef9ea6d Fix P2SH refund "non-final" error issue
According to Bitcoin source, CheckFinalTx() in validation.cpp ~line 223,
we need to make sure median blocktime has passed P2SH refund transaction's
nLockTime.

Previously we were erroneously checking that median blocktime was in the past.

This should fix issues where refunding P2SH results in a "non-final" error
from the ElectrumX server network.
2020-12-28 15:44:45 +00:00
catbref
332c917c94 Fix ALICE-based PRESENCE transactions.
PRESENCE transactions were previously validated using Bob's trade key (in address form).
But as PRESENCE transactions are already emitted by Alice, her trade key is also used
(if present in trade data by virtue of AT being locked to Alice).

Similarly, Alice's trade-bot won't even try to build PRESENCE transactions if her
trade key isn't publicly visible to other peers, i.e. after AT is locked to Alice.
2020-12-28 12:00:11 +00:00
catbref
35b0ac78b8 Bump ElectrumX transaction cache size from 100 to 200 2020-12-24 16:52:58 +00:00
catbref
047627a6e5 Force bitcoinj keychain lookaheadThreshold to zero so we always generate more keys 2020-12-24 16:48:47 +00:00
catbref
688f215dfd Allow PresenceType filtering on presence websocket via presenceType query param 2020-12-21 12:25:58 +00:00
catbref
7cbdbbcc8d Allow exception-free conversion from String to PresenceType 2020-12-21 12:25:11 +00:00
catbref
4e89b8fbac No need to create a map entry for null foreignBlockchain in TradeOffersWebSocket 2020-12-21 12:24:44 +00:00
catbref
70ec8cb11f Add public key in Qortal address form to tradeoffers and presence websockets
This aids matching PRESENCE to corresponding trade offers for use in UI.

Also tighten up visibility of some fields in ChainChainOfferSummary and
PresenceInfo to private.

PresenceInfo.address should map to CrossChainOfferSummary.qortalCreatorTradeAddress
which is "AT creator's ephemeral trading key-pair represented as Qortal address"
2020-12-18 11:42:53 +00:00
catbref
0f0266609f Add trading price estimate API call GET /crosschain/price/{blockchain} where blockchain is something like LITECOIN 2020-12-18 11:42:32 +00:00
catbref
1c6ea0a860 Improvements to ElectrumX, Bitcoin-y aspects, etc.
Add caching of transactions fetched via ElectrumX to reduce network load and speed up API response.

Fix handling ElectrumX servers that don't want to supply verbose transaction JSON.

Hide lots of data in BitcoinyTransaction that isn't needed by current API users.
2020-12-11 17:30:15 +00:00
catbref
3706cd5ff7 Return list, not set, of wallet transactions via API and provide better (usable) examples 2020-12-11 17:29:00 +00:00
catbref
934cd1d511 Add support for preferred blockchain to /websockets/crosschain/tradebot via foreignBlockchain query param 2020-12-10 17:08:32 +00:00
catbref
68e3d3b989 Add preferred-blockchain filtering to /websockets/crosschain/tradeoffers via foreignBlockchain query param 2020-12-10 16:00:01 +00:00
catbref
31fa916156 Add filtering by foreign blockchain to API crosschain calls 2020-12-10 14:52:06 +00:00
catbref
456bb3ca63 Include output addresses, if present, in BitcoinyTransaction 2020-12-10 14:01:40 +00:00
catbref
b07ad094c1 Fix DELETE /crosschain/tradebot by adding missing repository.saveChanges() 2020-12-10 13:11:48 +00:00
catbref
d766cfaa67 Fix API/websockets that were still BitcoinACCTv1-only 2020-12-10 12:48:46 +00:00
catbref
acc616c204 Add defensive code to EventBus to catch unexpected exceptions 2020-12-10 11:55:06 +00:00
catbref
8707f154ee Add support to ElectrumX for barring servers that don't give us the data we need 2020-12-10 11:32:09 +00:00
catbref
992427f0e0 Fix NPE due to unboxing null/no entry from PREVIOUS_STATES.get() in TradeBotWebSocket. Usually triggered when creating new trade-bot entry while having open websocket connection. 2020-12-10 11:31:37 +00:00
catbref
2c84add935 Bitcoiny improvements 2020-12-09 13:12:15 +00:00
catbref
e8fc91fd34 Minor work on ByteArray and associated tests 2020-12-08 15:19:12 +00:00
catbref
8c9cf4a02d Add API support for listing Bitcoin/Litecoin wallet transactions 2020-12-07 16:40:12 +00:00
catbref
23f0969b2d Requesting BTC/LTC wallet balance now accepts public key xpub/tpub too 2020-12-07 15:40:05 +00:00
catbref
cf82813280 Report foreignBlockchain and acctName in results for API call GET /crosschain/tradeoffers 2020-12-02 14:37:50 +00:00
catbref
753fa4dfa9 Remove extraneous boxing/unboxing from PresenceWebSocket 2020-12-02 13:20:37 +00:00
catbref
58ff338ab3 Now Transaction.importAsConfirmed() calls Controller.onNewTransaction(), removing call from API POST /transactions/process and Controller.onNetworkTransactionMessage() 2020-12-02 13:19:53 +00:00
catbref
064e12a57b Improve documentation regarding deadlock in EventBus 2020-12-02 13:16:18 +00:00
catbref
54bb8ed817 New PRESENCE-related API websocket at /websockets/presence 2020-12-01 17:24:02 +00:00
catbref
b651eae258 Notify EventBus with NewTransactionEvent in Controller.onNewTransaction 2020-12-01 17:23:10 +00:00
catbref
7562d9bbf8 Fix potential NPE when closing a websocket that failed to open properly 2020-12-01 17:22:40 +00:00
catbref
1b50dd5adf Modify API TransactionsResource to notify Controller after closing repository handle/session 2020-12-01 17:22:03 +00:00
catbref
c10a5db280 Fix PresenceTransaction to work with any ACCT, not only BitcoinACCTv1 2020-12-01 15:06:07 +00:00
catbref
500690be49 Add ATRepository.getAllATsByFunctionality() to fetch ATs matching selection of code hashes 2020-12-01 15:05:40 +00:00
catbref
778ac35ee6 Improve HSQLDBRepository.temporaryValuesTableSql to work with Collection, not just List 2020-12-01 15:04:50 +00:00
catbref
c16a664a78 Add toString() to ByteArray 2020-12-01 15:03:23 +00:00
catbref
75a265f89a Improve SupportedBlockchain enum 2020-12-01 15:02:58 +00:00
catbref
ddb55210b4 Merge branch 'presence-txn' into LTCv3-with-presence 2020-11-30 15:10:03 +00:00
catbref
e093520696 WIP: PRESENCE - TradeBot support, moved PresenceType enum, added Presence-only transaction deadline override 2020-11-27 18:06:24 +00:00
catbref
cfacddcb36 Change Controller.deleteExpiredTransactions to use Transaction-subclass .getDeadline() instance method call instead of static Transaction.getDeadline(transactionData) which allows Transaction subclasses to override with custom deadlines/expiry periods 2020-11-27 18:05:03 +00:00
catbref
d2dea3ff35 BlockMinter now notifies Controller of new block while still holding blockchain lock, bringing it in line with Synchronizer 2020-11-27 18:02:05 +00:00
catbref
865fcb95bf In pom.xml, have Maven surefire plugin skip tests by default 2020-11-27 14:26:01 +00:00
catbref
a52c089728 WIP: PRESENCE transactions - support only TRADE_BOT type and restrict to known trades 2020-11-26 16:33:21 +00:00
catbref
15d25649b2 WIP: PRESENCE transactions - repository support, removing older versions, tests 2020-11-26 16:11:11 +00:00
catbref
e0210635e3 Add repository support for fetching subset of unconfirmed transactions by type and/or creator 2020-11-26 16:10:13 +00:00
catbref
90b993e234 Refactor post-importAsUnconfirmed as method to be overridden by Transaction subclasses, e.g. CHAT 2020-11-26 16:08:56 +00:00
catbref
9b7c2c50fb Initial commit with PRESENCE transaction type -- untested 2020-11-24 17:02:05 +00:00
catbref
fccb3a3f0c Merge branch 'master' into LTCv3-merge-test 2020-11-13 10:46:45 +00:00
catbref
a9c7142d7b Speed up AT states reshape 2020-11-10 16:46:07 +00:00
catbref
7a40c3526f Bugfixes and tests for SLEEP_UNTIL_MESSAGE 2020-11-10 15:44:47 +00:00
catbref
3253d9d3fb WIP: initial implementation of AT sleep-until-message (untested) 2020-11-10 15:44:47 +00:00
catbref
7cd8ed6e23 HTLC secret and status caching to reduce repeated calls to ElectrumX network
Additional benefit is to speed up syncing if node has trade-bot entries,
as a batch of blocks processed within 30s will have the same HTLC responses
as neither Bitcoin nor Litecoin blocks are that fast. Once synced, the next
Qortal block (~60s) should pick up any new changes. Generally users will be
synced when using trade-bot anyway.

Also moved bitcoiny.getAddressTransactions(p2shAddress)
into BitcoinyHTLC.findHtlcSecret() so only called if secret,
or lack thereof, is not cached.

Added tests to cover caching.
2020-09-22 12:13:22 +01:00
catbref
4bc0edeeca Litecoin trade-bot added!
Several cross-chain API calls moved into separate classes,
although most of the URLs remain roughly the same to provide
backwards compatibility.

API /crosschain/at/build moved into /crosschain/BitcoinACCTv1

Converted DELETE /crosschain/tradeoffer to be ACCT-agnostic.
Changes to ACCT interface, etc. to support above.

Changes applied to other crosschain API calls to make them
independent of Bitcoin/Litecoin.

Corrections to fee calculations and usage in BitcoinACCTv1.

Added new LitecoinACCTv1 trade-bot, using LitecoinACCTv1.

Some minor typo corrections, rename of secretHash to hashOfSecret.

Some more Bitcoin-specific fields deprecated, but values duplicated
from newly-named fields for now.

Lower default fee (10sats/byte) for Litecoin spending transactions.
(Not P2SH fees which are 1000sats).

Changed ApiError INSUFFICIENT_BALANCE HTTP status from 422 to 402
as 422 isn't supported by Jetty?

CrossChainTradeSummary.btcAmount deprecated, use: foreignAmount

Modified pom.xml to generated package-info.java files for classes
inside org.qortal.api.model.** subdirectories.
2020-09-21 16:54:43 +01:00
catbref
7a06df6ccd Improve SQL prepared statement caching in HSQLDBATRepository, plus missing space in SQL in getMatchingFinalATStates 2020-09-21 16:42:43 +01:00
catbref
d9164a32e5 Improve SQL prepared statement caching in HSQLDBBlockRepository.getBlockInfos & test to cover 2020-09-21 16:38:45 +01:00
catbref
a8fbf32a88 Improve SQL prepared statement caching in HSQLDBAccountRepository.getEligibleLegacyQoraHolders() 2020-09-21 16:29:34 +01:00
catbref
514689d2f4 WIP: refactoring to support multiple foreign blockchains
API support for Litecoin wallet balance and sending LTC.

TradeBotCreateRequest rejigged to use blockchain-agnostic
field names, e.g. bitcoinAmount now foreignAmount,
and added foreignBlockchain field.

The massive API CrossChainResource class has been split into:

CrossChainAtResource: for building TRADE/REDEEM/CANCEL messages
(OFFER missing?)

CrossChainBitcoinResource: for Bitcoin wallet balance/spend
CrossChainLitecoinResource: ditto for Litecoin

CrossChainHtlcResource: for Bitcoiny-HTLC actions like:
deriving P2SH address
checking HTLC status
eventually: building refund/redeem transactions

CrossChainResource: for creating/cancelling/listing trade offers.

CrossChainTradeBotResource: for creating/cancelling trade-bot
entries, including responding to trade offers.

---

Other general trading changes:

TradeBot states are now specific to each individual trade-bot,
e.g. BitcoinACCTv1TradeBot or LitecoinACCTv1TradeBot, etc.

TradeBot states now a combination of int & String, instead of
enums due to above.

Extra columns added to DB TradeBotStates to store
blockchain, which ACCT in use, etc.

---

UNTESTED at this point!
2020-09-18 17:07:49 +01:00
catbref
76a15bb026 Initial implementation of Litecoin ACCT
Extracted AcctMode from BitcoinACCTv1.Mode as the values are
common to both Bitcoin/Litecoin ACCTs.

Added test apps for deploy, cancel, trade and redeem of LitecoinACCTv1.
2020-09-17 11:07:07 +01:00
catbref
b061f188f9 Update Bitcoinv1 DeployAT test app 2020-09-16 11:11:18 +01:00
catbref
af7d7d0966 Unified Bitcoin/Litecoin test apps 2020-09-16 10:57:45 +01:00
catbref
2ffd0770c6 Initial Litecoin support
Added altcoinj library as Maven dependency.

Added new Litecoin subclass of Bitcoiny,
with mainnet and testnet ElectrumX server lists.

Added litecoinNet settings variable and getter.

Added LitecoinTests.
Most tests work but testFindHtclSecret()
needs a redeemed HTLC on chain (not yet done).

Added litecoinNet to some test settings files
in resources.

Added Litecoin BuildHTLC, Refund test apps.

Added SendLTC app as Electrum-LTC seems a bit flaky?

So far managed to build HTLC P2SH, fund it and then
refund it!

---

As Bitcoin and Litecoin are both subclasses of Bitcoiny,
could unify some test apps with added Bitcoin/Litecoin
switch as first arg?
2020-09-15 17:53:54 +01:00
catbref
e3abeafc6b Refactoring to allow many foreign blockchains and multiple ACCTs
Bitcoin/Litecoin common aspects extracted in a "Bitcoiny" common class.
So:

Bitcoin (was BTC) extends Bitcoiny
Litecoin (future code) will also extend Bitcoiny

ElectrumX is now a BitcoinyBlockchainProvider
to allow easier future replacement and also tidier integration.

BTCP2SH is now BitcoinyHTLC as they are generic hash time-locked contracts,
probably Bitcoin/Litecoin agnostic.

BTCACCT is now BitcoinACCTv1, allowing for v2+ and also LitecoinACCTv1, etc.

BitcoinTransaction is now BitcoinyTransaction
as they are pretty much the same in Litecoin.

BitcoinException is now a more generic ForeignBlockchainException.

---

Bitcoiny subclasses instantiate a new BitcoinyBlockchainProvider
when creating their singleton instance. They pass relevant network
details to the BBP, like server lists, genesis block hash, etc.

Bitcoiny.WalletAwareUTXOProvider now only has the one key search mode
that is equivalent to the old REQUEST_MORE_IF_ANY_SPENT.

Tests tidied up.

---

Still to do:

Modifying TradeBot to handle multiple types of ACCTs,
like BitcoinACCTv2, LitecoinACCTv1...

Modifying API to support multiple types of ACCTs.

Actually add Litecoin support.

Build new ACCT without needing P2SH-B if possible.
2020-09-15 14:56:43 +01:00
1001 changed files with 163442 additions and 16585 deletions

37
.github/workflows/pr-testing.yml vendored Normal file
View File

@@ -0,0 +1,37 @@
name: PR testing
on:
push:
branches: [ master ]
jobs:
mavenTesting:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- name: Cache local Maven repository
uses: actions/cache@v3
with:
path: ~/.m2/repository
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
restore-keys: |
${{ runner.os }}-maven-
- name: Set up the Java JDK
uses: actions/setup-java@v3
with:
java-version: '11'
distribution: 'adopt'
- name: Load custom deps
run: |
mvn install -DskipTests=true --file pom.xml
- name: Run all tests
run: |
mvn -B clean test -DskipTests=false --file pom.xml
if [ -f "target/site/jacoco/index.html" ]; then echo "Total coverage: $(cat target/site/jacoco/index.html | grep -o 'Total[^%]*%' | grep -o '[0-9]*%')"; fi
- name: Log coverage percentage
run: |
if [ ! -f "target/site/jacoco/index.html" ]; then echo "No coverage information available"; fi
if [ -f "target/site/jacoco/index.html" ]; then echo "Total coverage: $(cat target/site/jacoco/index.html | grep -o 'Total[^%]*%' | grep -o '[0-9]*%')"; fi

21
.gitignore vendored
View File

@@ -1,6 +1,8 @@
/db*
/lists/
/bin/
/target/
/qortal-backup/
/log.txt.*
/arbitrary*
/Qortal-BTC*
@@ -12,7 +14,20 @@
/.mvn.classpath
/notes*
/settings.json
/testnet*
/settings*.json
/testchain.json
/run-testnet.sh
/testchain*.json
/run-testnet*.sh
/.idea
/qortal.iml
.DS_Store
/src/main/resources/resources
/*.jar
/run.pid
/run.log
/WindowsInstaller/Install Files/qortal.jar
/*.7z
/tmp
/wallets
/data*
/src/test/resources/arbitrary/*/.qortal/cache
apikey.txt

3
.vscode/settings.json vendored Normal file
View File

@@ -0,0 +1,3 @@
{
"java.compile.nullAnalysis.mode": "automatic"
}

View File

@@ -1,5 +1,20 @@
# Auto Updates
## TL;DR: how-to
* Prepare new release version (see way below for details)
* Assuming you are in git 'master' branch, at HEAD
* Shutdown local node if running
* Build auto-update download: `tools/build-auto-update.sh` - uploads auto-update file into new git branch
* Restart local node
* Publish auto-update transaction using *private key* for **non-admin** member of "dev" group:
`tools/publish-auto-update.pl non-admin-dev-member-private-key-in-base58`
* Wait for auto-update `ARBITRARY` transaction to be confirmed into a block
* Have "dev" group admins 'approve' auto-update using `tools/approve-auto-update.sh`
This tool will prompt for *private key* of **admin** of "dev" group
* A minimum number of admins are required for approval, and a minimum number of blocks must pass also.
* Nodes will start to download, and apply, the update over the next 20 minutes or so (see CHECK_INTERVAL in AutoUpdate.java)
## Theory
* Using a specific git commit (e.g. abcdef123) we produce a determinstic JAR with consistent hash.
* To avoid issues with over-eager anti-virus / firewalls we obfuscate JAR using very simplistic XOR-based method.
@@ -25,8 +40,8 @@ The same method is used to obfuscate and de-obfuscate:
## Typical download locations
The git SHA1 commit hash is used to replace `%s` in various download locations, e.g.:
* https://github.com/QORT/qortal/raw/%s/qortal.update
* https://raw.githubusercontent.com@151.101.16.133/QORT/qortal/%s/qortal.update
* https://github.com/Qortal/qortal/raw/%s/qortal.update
* https://raw.githubusercontent.com@151.101.16.133/Qortal/qortal/%s/qortal.update
These locations are part of the org.qortal.settings.Settings class and can be overriden in settings.json like:
```
@@ -45,4 +60,12 @@ $ java -cp qortal.jar org.qortal.XorUpdate
usage: XorUpdate <input-file> <output-file>
$ java -cp qortal.jar org.qortal.XorUpdate qortal.jar qortal.update
$
```
```
## Preparing new release version
* Shutdown local node
* Modify `pom.xml` and increase version inside `<version>` tag
* Commit new `pom.xml` and push to github, e.g. `git commit -m 'Bumped to v1.4.2' -- pom.xml; git push`
* Tag this new commit with same version: `git tag v1.4.2`
* Push tag up to github: `git push origin v1.4.2`

View File

@@ -4,10 +4,10 @@ You can examine your node's database using [HSQLDB's "sqltool"](http://www.hsqld
It's a good idea to install "rlwrap" (ReadLine wrapper) too as sqltool doesn't support command history/editing.
Typical command line for sqltool would be:
`rlwrap java -cp ${HSQLDB_JAR}:${SQLTOOL_JAR} org.hsqldb.cmdline.SqlTool --rcFile=${SQLTOOL_RC} qora`
`rlwrap java -cp ${HSQLDB_JAR}:${SQLTOOL_JAR} org.hsqldb.cmdline.SqlTool --rcFile=${SQLTOOL_RC} qortal`
`${HSQLDB_JAR}` should be set with pathname where Maven downloaded hsqldb,
typically `${HOME}/.m2/repository/org/hsqldb/hsqldb/2.5.0/hsqldb-2.5.0.jar`
typically `${HOME}/.m2/repository/org/hsqldb/hsqldb/2.5.1/hsqldb-2.5.1.jar`
`${SQLTOOL_JAR}` should be set with pathname where Maven downloaded sqltool,
typically `${HOME}/.m2/repository/org/hsqldb/sqltool/2.5.0/sqltool-2.5.0.jar`
@@ -25,10 +25,16 @@ Above `url` component `file:db/blockchain` assumes you will call `sqltool` from
Another idea is to assign a shell alias in your `.bashrc` like:
```
export HSQLDB_JAR=${HOME}/.m2/repository/org/hsqldb/hsqldb/2.5.0/hsqldb-2.5.0.jar
export HSQLDB_JAR=${HOME}/.m2/repository/org/hsqldb/hsqldb/2.5.1/hsqldb-2.5.1.jar
export SQLTOOL_JAR=${HOME}/.m2/repository/org/hsqldb/sqltool/2.5.0/sqltool-2.5.0.jar
alias sqltool='rlwrap java -cp ${HSQLDB_JAR}:${SQLTOOL_JAR} org.hsqldb.cmdline.SqlTool --rcFile=${SQLTOOL_RC}'
```
So you can simply type: `sqltool qortal`
Don't forget to use `SHUTDOWN;` before exiting sqltool so that database files are closed cleanly.
## Quick and dirty version
With `sqltool-2.5.0.jar` and `qortal.jar` in current directory, and database in `db/`
`java -cp qortal.jar:sqltool-2.5.0.jar org.hsqldb.cmdline.SqlTool --inlineRc=url=jdbc:hsqldb:file:db/blockchain`

26
Dockerfile Normal file
View File

@@ -0,0 +1,26 @@
FROM maven:3-openjdk-11 as builder
WORKDIR /work
COPY ./ /work/
RUN mvn clean package
###
FROM openjdk:11
RUN useradd -r -u 1000 -g users qortal && \
mkdir /usr/local/qortal /qortal && \
chown 1000:100 /qortal
COPY --from=builder /work/log4j2.properties /usr/local/qortal/
COPY --from=builder /work/target/qortal*.jar /usr/local/qortal/qortal.jar
USER 1000:100
EXPOSE 12391 12392
HEALTHCHECK --start-period=5m CMD curl -sf http://127.0.0.1:12391/admin/info || exit 1
WORKDIR /qortal
VOLUME /qortal
ENTRYPOINT ["java"]
CMD ["-Djava.net.preferIPv4Stack=false", "-jar", "/usr/local/qortal/qortal.jar"]

919
Q-Apps.md Normal file
View File

@@ -0,0 +1,919 @@
# Qortal Project - Q-Apps Documentation
## Introduction
Q-Apps are static web apps written in javascript, HTML, CSS, and other static assets. The key difference between a Q-App and a fully static site is its ability to interact with both the logged-in user and on-chain data. This is achieved using the API described in this document.
# Section 0: Basic QDN concepts
## Introduction to QDN resources
Each published item on QDN (Qortal Data Network) is referred to as a "resource". A resource could contain anything from a few characters of text, to a multi-layered directory structure containing thousands of files.
Resources are stored on-chain, however the data payload is generally stored off-chain, and verified using an on-chain SHA-256 hash.
To publish a resource, a user must first register a name, and then include that name when publishing the data. Accounts without a registered name are unable to publish to QDN from a Q-App at this time.
Owning the name grants update privileges to the data. If that name is later sold or transferred, the permission to update that resource is moved to the new owner.
## Name, service & identifier
Each QDN resource has 3 important fields:
- `name` - the registered name of the account that is publishing the data (which will hold update/edit privileges going forwards).<br /><br />
- `service` - the type of content (e.g. IMAGE or JSON). Different services have different validation rules. See [list of available services](#services).<br /><br />
- `identifier` - an optional string to allow more than one resource to exist for a given name/service combination. For example, the name `QortalDemo` may wish to publish multiple images. This can be achieved by using a different identifier string for each. The identifier is only unique to the name in question, and so it doesn't matter if another name is using the same service and identifier string.
## Shared identifiers
Since an identifier can be used by multiple names, this can be used to the advantage of Q-App developers as it allows for data to be stored in a deterministic location.
An example of this is the user's avatar. This will always be published with service `THUMBNAIL` and identifier `qortal_avatar`, along with the user's name. So, an app can display the avatar of a user just by specifying their name when requesting the data. The same applies when publishing data.
## "Default" resources
A "default" resource refers to one without an identifier. For example, when a website is published via the UI, it will use the user's name and the service `WEBSITE`. These do not have an identifier, and are therefore the "default" website for this name. When requesting or publishing data without an identifier, apps can either omit the `identifier` key entirely, or include `"identifier": "default"` to indicate that the resource(s) being queried or published do not have an identifier.
<a name="services"></a>
## Available service types
Here is a list of currently available services that can be used in Q-Apps:
### Public services ###
The services below are intended to be used for publicly accessible data.
IMAGE,
THUMBNAIL,
VIDEO,
AUDIO,
PODCAST,
VOICE,
ARBITRARY_DATA,
JSON,
DOCUMENT,
LIST,
PLAYLIST,
METADATA,
BLOG,
BLOG_POST,
BLOG_COMMENT,
GIF_REPOSITORY,
ATTACHMENT,
FILE,
FILES,
CHAIN_DATA,
STORE,
PRODUCT,
OFFER,
COUPON,
CODE,
PLUGIN,
EXTENSION,
GAME,
ITEM,
NFT,
DATABASE,
SNAPSHOT,
COMMENT,
CHAIN_COMMENT,
WEBSITE,
APP,
QCHAT_ATTACHMENT,
QCHAT_IMAGE,
QCHAT_AUDIO,
QCHAT_VOICE
### Private services ###
For the services below, data is encrypted for a single recipient, and can only be decrypted using the private key of the recipient's wallet.
QCHAT_ATTACHMENT_PRIVATE,
ATTACHMENT_PRIVATE,
FILE_PRIVATE,
IMAGE_PRIVATE,
VIDEO_PRIVATE,
AUDIO_PRIVATE,
VOICE_PRIVATE,
DOCUMENT_PRIVATE,
MAIL_PRIVATE,
MESSAGE_PRIVATE
## Single vs multi-file resources
Some resources, such as those published with the `IMAGE` or `JSON` service, consist of a single file or piece of data (the image or the JSON string). This is the most common type of QDN resource, especially in the context of Q-Apps. These can be published by supplying a base64-encoded string containing the data.
Other resources, such as those published with the `WEBSITE`, `APP`, or `GIF_REPOSITORY` service, can contain multiple files and directories. Publishing these kinds of files is not yet available for Q-Apps, however it is possible to retrieve multi-file resources that are already published. When retrieving this data (via FETCH_QDN_RESOURCE), a `filepath` must be included to indicate the file that you would like to retrieve. There is no need to specify a filepath for single file resources, as these will automatically return the contents of the single file.
## App-specific data
Some apps may want to make all QDN data for a particular service available. However, others may prefer to only deal with data that has been published by their app (if a specific format/schema is being used for instance).
Identifiers can be used to allow app developers to locate data that has been published by their app. The recommended approach for this is to use the app name as a prefix on all identifiers when publishing data.
For instance, an app called `MyApp` could allow users to publish JSON data. The app could choose to prefix all identifiers with the string `myapp_`, and then use a random string for each published resource (resulting in identifiers such as `myapp_qR5ndZ8v`). Then, to locate data that has potentially been published by users of MyApp, it can later search the QDN database for items with `"service": "JSON"` and `"identifier": "myapp_"`. The SEARCH_QDN_RESOURCES action has a `prefix` option in order to match identifiers beginning with the supplied string.
Note that QDN is a permissionless system, and therefore it's not possible to verify that a resource was actually published by the app. It is recommended that apps validate the contents of the resource to ensure it is formatted correctly, instead of making assumptions.
## Updating a resource
To update a resource, it can be overwritten by publishing with the same `name`, `service`, and `identifier` combination. Note that the authenticated account must currently own the name in order to publish an update.
## Routing
If a non-existent `filepath` is accessed, the default behaviour of QDN is to return a `404: File not found` error. This includes anything published using the `WEBSITE` service.
However, routing is handled differently for anything published using the `APP` service.
For apps, QDN automatically sends all unhandled requests to the index file (generally index.html). This allows the app to use custom routing, as it is able to listen on any path. If a file exists at a path, the file itself will be served, and so the request won't be sent to the index file.
It's recommended that all apps return a 404 page if a request isn't able to be routed.
# Section 1: Simple links and image loading via HTML
## Section 1a: Linking to other QDN websites / resources
The `qortal://` protocol can be used to access QDN data from within Qortal websites and apps. The basic format is as follows:
```
<a href="qortal://{service}/{name}/{identifier}/{path}">link text</a>
```
However, the system will support the omission of the `identifier` and/or `path` components to allow for simpler URL formats.
A simple link to another website can be achieved with this HTML code:
```
<a href="qortal://WEBSITE/QortalDemo">link text</a>
```
To link to a specific page of another website:
```
<a href="qortal://WEBSITE/QortalDemo/minting-leveling/index.html">link text</a>
```
To link to a standalone resource, such as an avatar
```
<a href="qortal://THUMBNAIL/QortalDemo/qortal_avatar">avatar</a>
```
For cases where you would prefer to explicitly include an identifier (to remove ambiguity) you can use the keyword `default` to access a resource that doesn't have an identifier. For instance:
```
<a href="qortal://WEBSITE/QortalDemo/default">link to root of website</a>
<a href="qortal://WEBSITE/QortalDemo/default/minting-leveling/index.html">link to subpage of website</a>
```
## Section 1b: Linking to other QDN images
The same applies for images, such as displaying an avatar:
```
<img src="qortal://THUMBNAIL/QortalDemo/qortal_avatar" />
```
...or even an image from an entirely different website:
```
<img src="qortal://WEBSITE/AlphaX/assets/img/logo.png" />
```
# Section 2: Integrating a Javascript app
Javascript apps allow for much more complex integrations with Qortal's blockchain data.
## Section 2a: Direct API calls
The standard [Qortal Core API](http://localhost:12391/api-documentation) is available to websites and apps, and can be called directly using a standard AJAX request, such as:
```
async function getNameInfo(name) {
const response = await fetch("/names/" + name);
const nameData = await response.json();
console.log("nameData: " + JSON.stringify(nameData));
}
getNameInfo("QortalDemo");
```
However, this only works for read-only data, such as looking up transactions, names, balances, etc. Also, since the address of the logged in account can't be retrieved from the core, apps can't show personalized data with this approach.
## Section 2b: User interaction via qortalRequest()
To take things a step further, the qortalRequest() function can be used to interact with the user, in order to:
- Request address and public key of the logged in account
- Publish data to QDN
- Send chat messages
- Join groups
- Deploy ATs (smart contracts)
- Send QORT or any supported foreign coin
- Add/remove items from lists
In addition to the above, qortalRequest() also supports many read-only functions that are also available via direct core API calls. Using qortalRequest() helps with futureproofing, as the core APIs can be modified without breaking functionality of existing Q-Apps.
### Making a request
Qortal core will automatically inject the `qortalRequest()` javascript function to all websites/apps, which returns a Promise. This can be used to fetch data or publish data to the Qortal blockchain. This functionality supports async/await, as well as try/catch error handling.
```
async function myfunction() {
try {
let res = await qortalRequest({
action: "GET_ACCOUNT_DATA",
address: "QZLJV7wbaFyxaoZQsjm6rb9MWMiDzWsqM2"
});
console.log(JSON.stringify(res)); // Log the response to the console
} catch(e) {
console.log("Error: " + JSON.stringify(e));
}
}
myfunction();
```
### Timeouts
Request timeouts are handled automatically when using qortalRequest(). The timeout value will differ based on the action being used - see `getDefaultTimeout()` in [q-apps.js](src/main/resources/q-apps/q-apps.js) for the current values.
If a request times out it will throw an error - `The request timed out` - which can be handled by the Q-App.
It is also possible to specify a custom timeout using `qortalRequestWithTimeout(request, timeout)`, however this is discouraged. It's more reliable and futureproof to let the core handle the timeout values.
# Section 3: qortalRequest Documentation
## Supported actions
Here is a list of currently supported actions:
- GET_USER_ACCOUNT
- GET_ACCOUNT_DATA
- GET_ACCOUNT_NAMES
- SEARCH_NAMES
- GET_NAME_DATA
- LIST_QDN_RESOURCES
- SEARCH_QDN_RESOURCES
- GET_QDN_RESOURCE_STATUS
- GET_QDN_RESOURCE_PROPERTIES
- GET_QDN_RESOURCE_METADATA
- GET_QDN_RESOURCE_URL
- LINK_TO_QDN_RESOURCE
- FETCH_QDN_RESOURCE
- PUBLISH_QDN_RESOURCE
- PUBLISH_MULTIPLE_QDN_RESOURCES
- DECRYPT_DATA
- SAVE_FILE
- GET_WALLET_BALANCE
- GET_BALANCE
- SEND_COIN
- SEARCH_CHAT_MESSAGES
- SEND_CHAT_MESSAGE
- LIST_GROUPS
- JOIN_GROUP
- DEPLOY_AT
- GET_AT
- GET_AT_DATA
- LIST_ATS
- FETCH_BLOCK
- FETCH_BLOCK_RANGE
- SEARCH_TRANSACTIONS
- GET_PRICE
- GET_LIST_ITEMS
- ADD_LIST_ITEMS
- DELETE_LIST_ITEM
More functionality will be added in the future.
## Example Requests
Here are some example requests for each of the above:
### Get address of logged in account
_Will likely require user approval_
```
let account = await qortalRequest({
action: "GET_USER_ACCOUNT"
});
let address = account.address;
```
### Get public key of logged in account
_Will likely require user approval_
```
let pubkey = await qortalRequest({
action: "GET_USER_ACCOUNT"
});
let publicKey = account.publicKey;
```
### Get account data
```
let res = await qortalRequest({
action: "GET_ACCOUNT_DATA",
address: "QZLJV7wbaFyxaoZQsjm6rb9MWMiDzWsqM2"
});
```
### Get names owned by account
```
let res = await qortalRequest({
action: "GET_ACCOUNT_NAMES",
address: "QZLJV7wbaFyxaoZQsjm6rb9MWMiDzWsqM2"
});
```
### Search names
```
let res = await qortalRequest({
action: "SEARCH_NAMES",
query: "search query goes here",
prefix: false, // Optional - if true, only the beginning of the name is matched
limit: 100,
offset: 0,
reverse: false
});
```
### Get name data
```
let res = await qortalRequest({
action: "GET_NAME_DATA",
name: "QortalDemo"
});
```
### List QDN resources
```
let res = await qortalRequest({
action: "LIST_QDN_RESOURCES",
service: "THUMBNAIL",
name: "QortalDemo", // Optional (exact match)
identifier: "qortal_avatar", // Optional (exact match)
default: true, // Optional
includeStatus: false, // Optional - will take time to respond, so only request if necessary
includeMetadata: false, // Optional - will take time to respond, so only request if necessary
followedOnly: false, // Optional - include followed names only
excludeBlocked: false, // Optional - exclude blocked content
limit: 100,
offset: 0,
reverse: true
});
```
### Search QDN resources
```
let res = await qortalRequest({
action: "SEARCH_QDN_RESOURCES",
service: "THUMBNAIL",
query: "search query goes here", // Optional - searches both "identifier" and "name" fields
identifier: "search query goes here", // Optional - searches only the "identifier" field
name: "search query goes here", // Optional - searches only the "name" field
prefix: false, // Optional - if true, only the beginning of fields are matched in all of the above filters
exactMatchNames: true, // Optional - if true, partial name matches are excluded
default: false, // Optional - if true, only resources without identifiers are returned
mode: "LATEST", // Optional - whether to return all resources or just the latest for a name/service combination. Possible values: ALL,LATEST. Default: LATEST
minLevel: 1, // Optional - whether to filter results by minimum account level
includeStatus: false, // Optional - will take time to respond, so only request if necessary
includeMetadata: false, // Optional - will take time to respond, so only request if necessary
nameListFilter: "QApp1234Subscriptions", // Optional - will only return results if they are from a name included in supplied list
followedOnly: false, // Optional - include followed names only
excludeBlocked: false, // Optional - exclude blocked content
// before: 1683546000000, // Optional - limit to resources created before timestamp
// after: 1683546000000, // Optional - limit to resources created after timestamp
limit: 100,
offset: 0,
reverse: true
});
```
### Search QDN resources (multiple names)
```
let res = await qortalRequest({
action: "SEARCH_QDN_RESOURCES",
service: "THUMBNAIL",
query: "search query goes here", // Optional - searches both "identifier" and "name" fields
identifier: "search query goes here", // Optional - searches only the "identifier" field
names: ["QortalDemo", "crowetic", "AlphaX"], // Optional - searches only the "name" field for any of the supplied names
prefix: false, // Optional - if true, only the beginning of fields are matched in all of the above filters
exactMatchNames: true, // Optional - if true, partial name matches are excluded
default: false, // Optional - if true, only resources without identifiers are returned
mode: "LATEST", // Optional - whether to return all resources or just the latest for a name/service combination. Possible values: ALL,LATEST. Default: LATEST
includeStatus: false, // Optional - will take time to respond, so only request if necessary
includeMetadata: false, // Optional - will take time to respond, so only request if necessary
nameListFilter: "QApp1234Subscriptions", // Optional - will only return results if they are from a name included in supplied list
followedOnly: false, // Optional - include followed names only
excludeBlocked: false, // Optional - exclude blocked content
// before: 1683546000000, // Optional - limit to resources created before timestamp
// after: 1683546000000, // Optional - limit to resources created after timestamp
limit: 100,
offset: 0,
reverse: true
});
```
### Fetch QDN single file resource
```
let res = await qortalRequest({
action: "FETCH_QDN_RESOURCE",
name: "QortalDemo",
service: "THUMBNAIL",
identifier: "qortal_avatar", // Optional. If omitted, the default resource is returned, or you can alternatively use the keyword "default"
encoding: "base64", // Optional. If omitted, data is returned in raw form
rebuild: false
});
```
### Fetch file from multi file QDN resource
Data is returned in the base64 format
```
let res = await qortalRequest({
action: "FETCH_QDN_RESOURCE",
name: "QortalDemo",
service: "WEBSITE",
identifier: "default", // Optional. If omitted, the default resource is returned, or you can alternatively request that using the keyword "default", as shown here
filepath: "index.html", // Required only for resources containing more than one file
rebuild: false
});
```
### Get QDN resource status
```
let res = await qortalRequest({
action: "GET_QDN_RESOURCE_STATUS",
name: "QortalDemo",
service: "THUMBNAIL",
identifier: "qortal_avatar", // Optional
build: true // Optional - request that the resource is fetched & built in the background
});
```
### Get QDN resource properties
```
let res = await qortalRequest({
action: "GET_QDN_RESOURCE_PROPERTIES",
name: "QortalDemo",
service: "THUMBNAIL",
identifier: "qortal_avatar" // Optional
});
// Returns: filename, size, mimeType (where available)
```
### Get QDN resource metadata
```
let res = await qortalRequest({
action: "GET_QDN_RESOURCE_METADATA",
name: "QortalDemo",
service: "THUMBNAIL",
identifier: "qortal_avatar" // Optional
});
```
### Publish a single file to QDN
_Requires user approval_.<br />
Note: this publishes a single, base64-encoded file. Multi-file resource publishing (such as a WEBSITE or GIF_REPOSITORY) is not yet supported via a Q-App. It will be added in a future update.
```
let res = await qortalRequest({
action: "PUBLISH_QDN_RESOURCE",
name: "Demo", // Publisher must own the registered name - use GET_ACCOUNT_NAMES for a list
service: "IMAGE",
identifier: "myapp-image1234" // Optional
data64: "base64_encoded_data",
// filename: "image.jpg", // Optional - to help apps determine the file's type
// title: "Title", // Optional
// description: "Description", // Optional
// category: "TECHNOLOGY", // Optional
// tag1: "any", // Optional
// tag2: "strings", // Optional
// tag3: "can", // Optional
// tag4: "go", // Optional
// tag5: "here", // Optional
// encrypt: true, // Optional - to be used with a private service
// recipientPublicKey: "publickeygoeshere" // Only required if `encrypt` is set to true
});
```
### Publish multiple resources at once to QDN
_Requires user approval_.<br />
Note: each resource being published consists of a single, base64-encoded file, each in its own transaction. Useful for publishing two or more related things, such as a video and a video thumbnail.
```
let res = await qortalRequest({
action: "PUBLISH_MULTIPLE_QDN_RESOURCES",
resources: [
name: "Demo", // Publisher must own the registered name - use GET_ACCOUNT_NAMES for a list
service: "IMAGE",
identifier: "myapp-image1234" // Optional
data64: "base64_encoded_data",
// filename: "image.jpg", // Optional - to help apps determine the file's type
// title: "Title", // Optional
// description: "Description", // Optional
// category: "TECHNOLOGY", // Optional
// tag1: "any", // Optional
// tag2: "strings", // Optional
// tag3: "can", // Optional
// tag4: "go", // Optional
// tag5: "here", // Optional
// encrypt: true, // Optional - to be used with a private service
// recipientPublicKey: "publickeygoeshere" // Only required if `encrypt` is set to true
],
[
... more resources here if needed ...
]
});
```
### Decrypt encrypted/private data
```
let res = await qortalRequest({
action: "DECRYPT_DATA",
encryptedData: 'qortalEncryptedDatabMx4fELNTV+ifJxmv4+GcuOIJOTo+3qAvbWKNY2L1r',
publicKey: 'publickeygoeshere'
});
// Returns base64 encoded string of plaintext data
```
### Prompt user to save a file to disk
Note: mimeType not required but recommended. If not specified, saving will fail if the mimeType is unable to be derived from the Blob.
```
let res = await qortalRequest({
action: "SAVE_FILE",
blob: dataBlob,
filename: "myfile.pdf",
mimeType: "application/pdf" // Optional but recommended
});
```
### Get wallet balance (QORT)
_Requires user approval_
```
let res = await qortalRequest({
action: "GET_WALLET_BALANCE",
coin: "QORT"
});
```
### Get address or asset balance
```
let res = await qortalRequest({
action: "GET_BALANCE",
address: "QZLJV7wbaFyxaoZQsjm6rb9MWMiDzWsqM2"
});
```
```
let res = await qortalRequest({
action: "GET_BALANCE",
assetId: 1,
address: "QZLJV7wbaFyxaoZQsjm6rb9MWMiDzWsqM2"
});
```
### Send QORT to address
_Requires user approval_
```
let res = await qortalRequest({
action: "SEND_COIN",
coin: "QORT",
destinationAddress: "QZLJV7wbaFyxaoZQsjm6rb9MWMiDzWsqM2",
amount: 1.00000000 // 1 QORT
});
```
### Send foreign coin to address
_Requires user approval_<br />
Note: default fees can be found [here](https://github.com/Qortal/qortal-ui/blob/master/plugins/plugins/core/qdn/browser/browser.src.js#L205-L209).
```
let res = await qortalRequest({
action: "SEND_COIN",
coin: "LTC",
destinationAddress: "LSdTvMHRm8sScqwCi6x9wzYQae8JeZhx6y",
amount: 1.00000000, // 1 LTC
fee: 0.00000020 // Optional fee per byte (default fee used if omitted, recommended) - not used for QORT or ARRR
});
```
### Search or list chat messages
```
let res = await qortalRequest({
action: "SEARCH_CHAT_MESSAGES",
before: 999999999999999,
after: 0,
txGroupId: 0, // Optional (must specify either txGroupId or two involving addresses)
// involving: ["QZLJV7wbaFyxaoZQsjm6rb9MWMiDzWsqM2", "QSefrppsDCsZebcwrqiM1gNbWq7YMDXtG2"], // Optional (must specify either txGroupId or two involving addresses)
// reference: "reference", // Optional
// chatReference: "chatreference", // Optional
// hasChatReference: true, // Optional
encoding: "BASE64", // Optional (defaults to BASE58 if omitted)
limit: 100,
offset: 0,
reverse: true
});
```
### Send a group chat message
_Requires user approval_
```
let res = await qortalRequest({
action: "SEND_CHAT_MESSAGE",
groupId: 0,
message: "Test"
});
```
### Send a private chat message
_Requires user approval_
```
let res = await qortalRequest({
action: "SEND_CHAT_MESSAGE",
destinationAddress: "QZLJV7wbaFyxaoZQsjm6rb9MWMiDzWsqM2",
message: "Test"
});
```
### List groups
```
let res = await qortalRequest({
action: "LIST_GROUPS",
limit: 100,
offset: 0,
reverse: true
});
```
### Join a group
_Requires user approval_
```
let res = await qortalRequest({
action: "JOIN_GROUP",
groupId: 100
});
```
### Deploy an AT
_Requires user approval_
```
let res = await qortalRequest({
action: "DEPLOY_AT",
creationBytes: "12345", // Must be Base58 encoded
name: "test name",
description: "test description",
type: "test type",
tags: "test tags",
amount: 1.00000000, // 1 QORT
assetId: 0,
// fee: 0.002 // optional - will use default fee if excluded
});
```
### Get AT info
```
let res = await qortalRequest({
action: "GET_AT",
atAddress: "ASRUsCjk6fa5bujv3oWYmWaVqNtvxydpPH"
});
```
### Get AT data bytes (base58 encoded)
```
let res = await qortalRequest({
action: "GET_AT_DATA",
atAddress: "ASRUsCjk6fa5bujv3oWYmWaVqNtvxydpPH"
});
```
### List ATs by functionality
```
let res = await qortalRequest({
action: "LIST_ATS",
codeHash58: "4KdJETRAdymE7dodDmJbf5d9L1bp4g5Nxky8m47TBkvA",
isExecutable: true,
limit: 100,
offset: 0,
reverse: true
});
```
### Fetch block by signature
```
let res = await qortalRequest({
action: "FETCH_BLOCK",
signature: "875yGFUy1zHV2hmxNWzrhtn9S1zkeD7SQppwdXFysvTXrankCHCz4iyAUgCBM3GjvibbnyRQpriuy1cyu953U1u5uQdzuH3QjQivi9UVwz86z1Akn17MGd5Z5STjpDT7248K6vzMamuqDei57Znonr8GGgn8yyyABn35CbZUCeAuXju"
});
```
### Fetch block by height
```
let res = await qortalRequest({
action: "FETCH_BLOCK",
height: "1139850"
});
```
### Fetch a range of blocks
```
let res = await qortalRequest({
action: "FETCH_BLOCK_RANGE",
height: "1139800",
count: 20,
reverse: false
});
```
### Search transactions
```
let res = await qortalRequest({
action: "SEARCH_TRANSACTIONS",
// startBlock: 1139000,
// blockLimit: 1000,
txGroupId: 0,
txType: [
"PAYMENT",
"REWARD_SHARE"
],
confirmationStatus: "CONFIRMED",
limit: 10,
offset: 0,
reverse: false
});
```
### Get an estimate of the QORT price
```
let res = await qortalRequest({
action: "GET_PRICE",
blockchain: "LITECOIN",
// maxtrades: 10,
inverse: true
});
```
### Get URL to load a QDN resource
Note: this returns a "Resource does not exist" error if a non-existent resource is requested.
```
let url = await qortalRequest({
action: "GET_QDN_RESOURCE_URL",
service: "THUMBNAIL",
name: "QortalDemo",
identifier: "qortal_avatar"
// path: "filename.jpg" // optional - not needed if resource contains only one file
});
```
### Get URL to load a QDN website
Note: this returns a "Resource does not exist" error if a non-existent resource is requested.
```
let url = await qortalRequest({
action: "GET_QDN_RESOURCE_URL",
service: "WEBSITE",
name: "QortalDemo",
});
```
### Get URL to load a specific file from a QDN website
Note: this returns a "Resource does not exist" error if a non-existent resource is requested.
```
let url = await qortalRequest({
action: "GET_QDN_RESOURCE_URL",
service: "WEBSITE",
name: "AlphaX",
path: "/assets/img/logo.png"
});
```
### Link/redirect to another QDN website
Note: an alternate method is to include `<a href="qortal://WEBSITE/QortalDemo">link text</a>` within your HTML code.
```
let res = await qortalRequest({
action: "LINK_TO_QDN_RESOURCE",
service: "WEBSITE",
name: "QortalDemo",
});
```
### Link/redirect to a specific path of another QDN website
Note: an alternate method is to include `<a href="qortal://WEBSITE/QortalDemo/minting-leveling/index.html">link text</a>` within your HTML code.
```
let res = await qortalRequest({
action: "LINK_TO_QDN_RESOURCE",
service: "WEBSITE",
name: "QortalDemo",
path: "/minting-leveling/index.html"
});
```
### Get the contents of a list
_Requires user approval_
```
let res = await qortalRequest({
action: "GET_LIST_ITEMS",
list_name: "followedNames"
});
```
### Add one or more items to a list
_Requires user approval_
```
let res = await qortalRequest({
action: "ADD_LIST_ITEMS",
list_name: "blockedNames",
items: ["QortalDemo"]
});
```
### Delete a single item from a list
_Requires user approval_.
Items must be deleted one at a time.
```
let res = await qortalRequest({
action: "DELETE_LIST_ITEM",
list_name: "blockedNames",
item: "QortalDemo"
});
```
# Section 4: Examples
Some example projects can be found [here](https://github.com/Qortal/Q-Apps). These can be cloned and modified, or used as a reference when creating a new app.
## Sample App
Here is a sample application to display the logged-in user's avatar:
```
<html>
<head>
<script>
async function showAvatar() {
try {
// Get QORT address of logged in account
let account = await qortalRequest({
action: "GET_USER_ACCOUNT"
});
let address = account.address;
console.log("address: " + address);
// Get names owned by this account
let names = await qortalRequest({
action: "GET_ACCOUNT_NAMES",
address: address
});
console.log("names: " + JSON.stringify(names));
if (names.length == 0) {
console.log("User has no registered names");
return;
}
// Download base64-encoded avatar of the first registered name
let avatar = await qortalRequest({
action: "FETCH_QDN_RESOURCE",
name: names[0].name,
service: "THUMBNAIL",
identifier: "qortal_avatar",
encoding: "base64"
});
console.log("Avatar size: " + avatar.length + " bytes");
// Display the avatar image on the screen
document.getElementById("avatar").src = "data:image/png;base64," + avatar;
} catch(e) {
console.log("Error: " + JSON.stringify(e));
}
}
</script>
</head>
<body onload="showAvatar()">
<img width="500" id="avatar" />
</body>
</html>
```
# Section 5: Testing and Development
Publishing an in-development app to mainnet isn't recommended. There are several options for developing and testing a Q-app before publishing to mainnet:
### Preview mode
Select "Preview" in the UI after choosing the zip. This allows for full Q-App testing without the need to publish any data.
### Testnets
For an end-to-end test of Q-App publishing, you can use the official testnet, or set up a single node testnet of your own (often referred to as devnet) on your local machine. See [Single Node Testnet Quick Start Guide](testnet/README.md#quick-start).
### Debugging
It is recommended that you develop and test in a web browser, to allow access to the javascript console. To do this:
1. Open the UI app, then minimise it.
2. In a Chromium-based web browser, visit: http://localhost:12388/
3. Log in to your account and then preview your app/website.
4. Go to `View > Developer > JavaScript Console`. Here you can monitor console logs, errors, and network requests from your app, in the same way as any other web-app.

View File

@@ -1,12 +1,56 @@
# Qortal Project - Official Repo
# Qortal Project - Qortal Core - Primary Repository
The Qortal Core is the blockchain and node component of the overall project. It contains the primary API, and ability to make calls to create transactions, and interact with the Qortal Blockchain Network.
## Build / run
In order to run the Qortal Core, a machine with java 11+ installed is required. Minimum RAM specs will vary depending on settings, but as low as 4GB of RAM should be acceptable in most scenarios.
Qortal is a complete infrastructure platform with a blockchain backend, it is capable of indefinite web and application hosting with no continual fees, replacement of DNS and centralized name systems and communications systems, and is the foundation of the next generation digital infrastructure of the world. Qortal is unique in nearly every way, and was written from scratch to address as many concerns from both the existing 'blockchain space' and the 'typical internet' as possible, while maintaining a system that is easy to use and able to run on 'any' computer.
Qortal contains extensive functionality geared toward complete decentralization of the digital world. Removal of 'middlemen' of any kind from all transactions, and ability to publish websites and applications that require no continual fees, on a name that is truly owned by the account that registered it, or purchased it from another. A single name on Qortal is capable of being both a namespace and a 'username'. That single name can have an application, website, public and private data, communications, authentication, the namespace itself and more, which can subsequently be sold to anyone else without the need to change any type of 'hosting' or DNS entries that do not exist, email that doesn't exist, etc. Maintaining the same functionality as those replaced features of web 2.0.
Over time Qortal has progressed into a fully featured environment catering to any and all types of people and organizations, and will continue to advance as time goes on. Brining more features, capability, device support, and availale replacements for web2.0. Ultimately building a new, completely user-controlled digital world without limits.
Qortal has no owner, no company on top of it, and is completely community built, run, and funded. A community-established and run group of developers known as the 'dev-group' or Qortal Development Group, make group_approval based decisions for the project's future. If you are a developer interested in assisting with the project, you meay reach out to the Qortal Development Group in any of the available Qortal community locations. Either on the Qortal network itself, or on one of the temporary centralized social media locations.
Building the future one block at a time. Welcome to Qortal.
# Building the Qortal Core from Source
## Build / Run
- Requires Java 11. OpenJDK 11 recommended over Java SE.
- Install Maven
- Use Maven to fetch dependencies and build: `mvn clean package`
- Update Maven dependencies: `mvn install`
- Built JAR should be something like `target/qortal-1.0.jar`
- Create basic *settings.json* file: `echo '{}' > settings.json`
- Run JAR in same working directory as *settings.json*: `java -jar target/qortal-1.0.jar`
- Wrap in shell script, add JVM flags, redirection, backgrounding, etc. as necessary.
- Or use supplied example shell script: *run.sh*
- Or use supplied example shell script: *start.sh*
## IntelliJ IDEA Configuration
- Run -> Edit Configurations
- Add New Application
- Name: qortal
- SDK: java 11
- Main Class: org.qortal.controller.Controller
- Program arguments: settings.json -Dlog4j.configurationFile=log4j2.properties -ea
- Environment variables: Djava.net.preferIPv4Stack=false
# Using a pre-built Qortal 'jar' binary
If you prefer to utilize a released version of Qortal, you may do so by downloading one of the available releases from the releases page, that are also linked on https://qortal.org and https://qortal.dev.
# Learning Q-App Development
https://qortal.dev contains dev documentation for building JS/React (and other client-side languages) applications or 'Q-Apps' on Qortal. Q-Apps are published on Registered Qortal Names, and aside from a single Name Registration fee, and a fraction of QORT for a publish transaction, require zero continual costs. These applications get more redundant with each new access from a new Qortal Node, making your application faster for the next user to download, and stronger as time goes on. Q-Apps live indefinitely in the history of the blockchain-secured Qortal Data Network (QDN).
# How to learn more
If the project interests you, you may learn more from the various web2 and QDN based websites focused on introductory information.
https://qortal.org - primary internet presence
https://qortal.dev - secondary and development focused website with links to many new developments and documentation
https://wiki.qortal.org - community built and managed wiki with detailed information regarding the project
links to telegram and discord communities are at the top of https://qortal.org as well.

View File

@@ -1,3 +1,4 @@
{
"apiDocumentationEnabled": true
"apiDocumentationEnabled": true,
"apiWhitelistEnabled": false
}

View File

@@ -1,7 +1,7 @@
rootLogger.level = info
# On Windows, uncomment next line to set dirname:
# property.dirname = ${sys:user.home}\\AppData\\Local\\qortal\\
property.filename = ${sys:log4j2.filenameTemplate:-log.txt}
# property.filename = ${sys:log4j2.filenameTemplate:-log.txt}
rootLogger.appenderRef.console.ref = stdout
rootLogger.appenderRef.rolling.ref = FILE
@@ -59,11 +59,14 @@ appender.console.filter.threshold.level = error
appender.rolling.type = RollingFile
appender.rolling.name = FILE
appender.rolling.fileName = qortal.log
appender.rolling.filePattern = qortal.%d{dd-MMM}.log.gz
appender.rolling.layout.type = PatternLayout
appender.rolling.layout.pattern = %d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
appender.rolling.filePattern = ${dirname:-}${filename}.%i
appender.rolling.policy.type = SizeBasedTriggeringPolicy
appender.rolling.policy.size = 4MB
appender.rolling.policy.size = 10MB
appender.rolling.strategy.type = DefaultRolloverStrategy
appender.rolling.strategy.max = 7
# Set the immediate flush to true (default)
# appender.rolling.immediateFlush = true
# Set the append to true (default), should not overwrite

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.7 MiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 160 KiB

File diff suppressed because it is too large Load Diff

View File

@@ -2,23 +2,29 @@
## Prerequisites
* AdvancedInstaller v16 or better, and enterprise licence if translations are required
* Installed AdoptOpenJDK v11 64bit, full JDK *not* JRE
* AdvancedInstaller v19.4 or better, and enterprise licence.
* Qortal has an open source license, however it currently (as of December 2024) only supports up to version 19. (We may need to reach out to Advanced Installer again to obtain a new license at some point, if needed.
* Reach out to @crowetic for links to the installer install files, and license.
* Installed AdoptOpenJDK v17 64bit, full JDK *not* JRE
## General build instructions
If this is your first time opening the `qortal.aip` file then you might need to adjust
configured paths, or create a dummy `D:` drive with the expected layout.
Opening the aip file from within a clone of the qortal repo also works, if you have a separate windows machine setup to do the build.
You May need to change the location of the 'jre64' files inside Advanced Installer, if it is set to a path that your build machine doesn't have.
The Java Memory Arguments can be set manually, but as of December 2024 they have been reset back to system defaults. This should include G1GC Garbage Collector.
Typical build procedure:
* Overwrite the `qortal.jar` file in `Install-Files\`
* Place the `qortal.jar` file in `Install-Files\`
* Open AdvancedInstaller with qortal.aip file
* If releasing a new version, change version number in:
+ "Product Information" side menu
+ "Product Details" side menu entry
+ "Product Details" tab in "Product Details" pane
+ "Product Version" entry box
* Click away to a different side menu entry, e.g. "Resources" -> "Files and Folders"
* You should be prompted whether to generate a new product key, click "Generate New"
* Click "Build" button

BIN
WindowsInstaller/qortal.ico Executable file → Normal file

Binary file not shown.

Before

Width:  |  Height:  |  Size: 250 KiB

After

Width:  |  Height:  |  Size: 42 KiB

Binary file not shown.

View File

@@ -1,9 +1,9 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd" xmlns="http://maven.apache.org/POM/4.0.0"
<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd" xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
<modelVersion>4.0.0</modelVersion>
<groupId>org.ciyam</groupId>
<artifactId>AT</artifactId>
<version>1.3.8</version>
<groupId>com.dosse</groupId>
<artifactId>WaifUPnP</artifactId>
<version>1.1</version>
<description>POM was created from install:install-file</description>
</project>

Binary file not shown.

View File

@@ -1,9 +1,9 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd" xmlns="http://maven.apache.org/POM/4.0.0"
<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd" xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
<modelVersion>4.0.0</modelVersion>
<groupId>org.ciyam</groupId>
<artifactId>AT</artifactId>
<version>1.3.7</version>
<groupId>com.dosse</groupId>
<artifactId>WaifUPnP</artifactId>
<version>1.2</version>
<description>POM was created from install:install-file</description>
</project>

View File

@@ -0,0 +1,13 @@
<?xml version="1.0" encoding="UTF-8"?>
<metadata>
<groupId>com.dosse</groupId>
<artifactId>WaifUPnP</artifactId>
<versioning>
<release>1.2</release>
<versions>
<version>1.1</version>
<version>1.2</version>
</versions>
<lastUpdated>20231026200127</lastUpdated>
</versioning>
</metadata>

Binary file not shown.

Binary file not shown.

View File

@@ -1,16 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<metadata>
<groupId>org.ciyam</groupId>
<artifactId>AT</artifactId>
<versioning>
<release>1.3.8</release>
<versions>
<version>1.3.4</version>
<version>1.3.5</version>
<version>1.3.6</version>
<version>1.3.7</version>
<version>1.3.8</version>
</versions>
<lastUpdated>20200925114415</lastUpdated>
</versioning>
</metadata>

View File

@@ -0,0 +1,5 @@
This is the production hsqldb-2.7.4 with the manifest file updated
Sealed: false
Allows the addition of the custom Qortal HSQLDBPool and Monitoring Classes

Binary file not shown.

View File

@@ -1,7 +1,7 @@
rootLogger.level = info
# On Windows, uncomment next line to set dirname:
# property.dirname = ${sys:user.home}\\AppData\\Local\\qortal\\
property.filename = ${sys:log4j2.filenameTemplate:-log.txt}
# property.filename = ${sys:log4j2.filenameTemplate:-log.txt}
rootLogger.appenderRef.console.ref = stdout
rootLogger.appenderRef.rolling.ref = FILE
@@ -59,11 +59,14 @@ appender.console.filter.threshold.level = error
appender.rolling.type = RollingFile
appender.rolling.name = FILE
appender.rolling.fileName = qortal.log
appender.rolling.filePattern = qortal.%d{dd-MMM}.log.gz
appender.rolling.layout.type = PatternLayout
appender.rolling.layout.pattern = %d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
appender.rolling.filePattern = ${dirname:-}${filename}.%i
appender.rolling.policy.type = SizeBasedTriggeringPolicy
appender.rolling.policy.size = 4MB
appender.rolling.policy.size = 10MB
appender.rolling.strategy.type = DefaultRolloverStrategy
appender.rolling.strategy.max = 7
# Set the immediate flush to true (default)
# appender.rolling.immediateFlush = true
# Set the append to true (default), should not overwrite

366
pom.xml
View File

@@ -3,27 +3,69 @@
<modelVersion>4.0.0</modelVersion>
<groupId>org.qortal</groupId>
<artifactId>qortal</artifactId>
<version>1.3.9</version>
<version>5.1.0</version> <!-- Version must be <X.Y.Z> -->
<packaging>jar</packaging>
<properties>
<skipTests>true</skipTests>
<bitcoinj.version>0.15.5</bitcoinj.version>
<bouncycastle.version>1.64</bouncycastle.version>
<build.timestamp>${maven.build.timestamp}</build.timestamp>
<ciyam-at.version>1.3.8</ciyam-at.version>
<commons-net.version>3.6</commons-net.version>
<commons-text.version>1.8</commons-text.version>
<dagger.version>1.2.2</dagger.version>
<guava.version>28.1-jre</guava.version>
<hsqldb.version>2.5.1</hsqldb.version>
<jersey.version>2.29.1</jersey.version>
<jetty.version>9.4.29.v20200521</jetty.version>
<log4j.version>2.12.1</log4j.version>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<slf4j.version>1.7.12</slf4j.version>
<swagger-api.version>2.0.9</swagger-api.version>
<swagger-ui.version>3.23.8</swagger-ui.version>
<package-info-maven-plugin.version>1.1.0</package-info-maven-plugin.version>
<skipJUnitTests>true</skipJUnitTests>
<altcoinj.version>d7cf6ac</altcoinj.version> <!-- BC v16 / Updated Abstract Classes / alertSigningKey -->
<bitcoinj.version>0.16.3</bitcoinj.version>
<bouncycastle.version>1.73</bouncycastle.version>
<build.timestamp>${maven.build.timestamp}</build.timestamp>
<ciyam-at.version>1b731d1</ciyam-at.version> <!-- This is the hash for v1.4.3 -->
<commons-net.version>3.8.0</commons-net.version>
<!-- <commons-net.version>3.9.0</commons-net.version> v5.2.0 coming soon -->
<commons-text.version>1.12.0</commons-text.version>
<commons-io.version>2.18.0</commons-io.version>
<commons-compress.version>1.27.1</commons-compress.version>
<commons-lang3.version>3.17.0</commons-lang3.version>
<dagger.version>1.2.2</dagger.version>
<extendedset.version>0.12.3</extendedset.version>
<git-commit-id-plugin.version>4.9.10</git-commit-id-plugin.version>
<grpc.version>1.68.1</grpc.version>
<!-- <grpc.version>1.68.3</grpc.version> v5.2.0 coming soon -->
<guava.version>33.3.1-jre</guava.version>
<hamcrest-library.version>2.2</hamcrest-library.version>
<homoglyph.version>1.2.1</homoglyph.version>
<hsqldb.version>2.7.4</hsqldb.version>
<icu4j.version>76.1</icu4j.version>
<java-diff-utils.version>4.15</java-diff-utils.version>
<javax.servlet-api.version>4.0.1</javax.servlet-api.version>
<jaxb-runtime.version>2.3.9</jaxb-runtime.version>
<jersey.version>2.42</jersey.version>
<jetty.version>9.4.56.v20240826</jetty.version>
<!-- <jetty.version>9.4.57.v20241219</jetty.version> v5.2.0 Coming Soon -->
<json-simple.version>1.1.1</json-simple.version>
<json.version>20240303</json.version>
<jsoup.version>1.18.1</jsoup.version>
<junit-jupiter-engine.version>5.11.0-M2</junit-jupiter-engine.version>
<lifecycle-mapping.version>1.0.0</lifecycle-mapping.version>
<log4j.version>2.23.1</log4j.version>
<mail.version>1.5.0-b01</mail.version>
<maven-build-helper-plugin.version>3.6.0</maven-build-helper-plugin.version>
<maven-compiler-plugin.version>3.13.0</maven-compiler-plugin.version>
<maven-dependency-plugin.version>3.6.1</maven-dependency-plugin.version>
<maven-jar-plugin.version>3.4.2</maven-jar-plugin.version>
<maven-package-info-plugin.version>1.1.0</maven-package-info-plugin.version>
<maven-plugin.version>2.18.0</maven-plugin.version>
<maven-reproducible-build-plugin.version>0.17</maven-reproducible-build-plugin.version>
<maven-resources-plugin.version>3.3.1</maven-resources-plugin.version>
<maven-shade-plugin.version>3.6.0</maven-shade-plugin.version>
<maven-install-plugin.version>3.1.3</maven-install-plugin.version>
<maven-surefire-plugin.version>3.5.2</maven-surefire-plugin.version>
<!-- <maven-surefire-plugin.version>3.5.3</maven-surefire-plugin.version> v5.2.0 Coming Soon -->
<protobuf.version>3.25.3</protobuf.version>
<!-- <protobuf.version>3.25.7</protobuf.version> v 5.1 -->
<replacer.version>1.5.3</replacer.version>
<simplemagic.version>1.17</simplemagic.version>
<slf4j.version>1.7.36</slf4j.version>
<!-- <swagger-api.version>2.2.30</swagger-api.version> need code upgrade Future Release -->
<!-- <swagger-api.version>2.1.13</swagger-api.version> need code upgrade Future Release -->
<swagger-api.version>2.0.10</swagger-api.version>
<swagger-ui.version>5.18.2</swagger-ui.version>
<upnp.version>1.2</upnp.version>
<xz.version>1.10</xz.version>
</properties>
<build>
<sourceDirectory>src/main/java</sourceDirectory>
@@ -38,14 +80,14 @@
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>versions-maven-plugin</artifactId>
<version>2.5</version>
<version>${maven-plugin.version}</version>
<configuration>
<generateBackupPoms>false</generateBackupPoms>
</configuration>
</plugin>
<plugin>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.8.0</version>
<version>${maven-compiler-plugin.version}</version>
<configuration>
<release>11</release>
</configuration>
@@ -76,7 +118,7 @@
<plugin>
<groupId>pl.project13.maven</groupId>
<artifactId>git-commit-id-plugin</artifactId>
<version>4.0.0</version>
<version>${git-commit-id-plugin.version}</version>
<executions>
<execution>
<id>get-the-git-infos</id>
@@ -108,7 +150,7 @@
<plugin>
<groupId>com.google.code.maven-replacer-plugin</groupId>
<artifactId>replacer</artifactId>
<version>1.5.3</version>
<version>${replacer.version}</version>
<executions>
<execution>
<id>replace-swagger-ui</id>
@@ -119,22 +161,35 @@
<inherited>false</inherited>
<configuration>
<file>${project.build.directory}/swagger-ui.unpacked/META-INF/resources/webjars/swagger-ui/${swagger-ui.version}/index.html</file>
<replacements>
<replacement>
<token>Swagger UI</token>
<value>Qortal API Documentation</value>
</replacement>
</replacements>
</configuration>
</execution>
<execution>
<id>replace-swagger-ui-json</id>
<phase>generate-resources</phase>
<goals>
<goal>replace</goal>
</goals>
<inherited>false</inherited>
<configuration>
<file>${project.build.directory}/swagger-ui.unpacked/META-INF/resources/webjars/swagger-ui/${swagger-ui.version}/swagger-initializer.js</file>
<replacements>
<replacement>
<token>https://petstore.swagger.io/v2/swagger.json</token>
<value>/openapi.json</value>
</replacement>
<replacement>
<token>Swagger UI</token>
<value>API Documentation</value>
</replacement>
<replacement>
<token>deepLinking: true,</token>
<value>
deepLinking: true,
tagsSorter: "alpha",
operationsSorter:
"alpha",
operationsSorter: "alpha",
validatorUrl: false,
</value>
</replacement>
</replacements>
@@ -150,11 +205,13 @@
<configuration>
<file>${project.build.outputDirectory}/git.properties</file>
<regex>true</regex>
<regexFlags><regexFlag>MULTILINE</regexFlag></regexFlags>
<regexFlags>
<regexFlag>MULTILINE</regexFlag>
</regexFlags>
<replacements>
<replacement>
<token>^(#.*$[\n\r]*)</token>
<value></value>
<value/>
</replacement>
</replacements>
</configuration>
@@ -164,7 +221,10 @@
<!-- add swagger-ui as resource to output package -->
<plugin>
<artifactId>maven-resources-plugin</artifactId>
<version>3.1.0</version>
<version>${maven-resources-plugin.version}</version>
<configuration>
<propertiesEncoding>ISO-8859-1</propertiesEncoding>
</configuration>
<executions>
<execution>
<id>copy-resources</id>
@@ -188,7 +248,7 @@
<plugin>
<groupId>com.github.bohnman</groupId>
<artifactId>package-info-maven-plugin</artifactId>
<version>${package-info-maven-plugin.version}</version>
<version>${maven-package-info-plugin.version}</version>
<configuration>
<packages>
<package>
@@ -199,6 +259,10 @@
<pattern>org.qortal.api.model**</pattern>
<template>${project.build.sourceDirectory}/org/qortal/data/package-info.java</template>
</package>
<package>
<pattern>org.qortal.api.model.**</pattern>
<template>${project.build.sourceDirectory}/org/qortal/data/package-info.java</template>
</package>
</packages>
<outputDirectory>${project.build.directory}/generated-sources/package-info</outputDirectory>
</configuration>
@@ -214,7 +278,7 @@
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>build-helper-maven-plugin</artifactId>
<version>3.0.0</version>
<version>${maven-build-helper-plugin.version}</version>
<executions>
<execution>
<phase>generate-sources</phase>
@@ -232,31 +296,58 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<version>3.2.0</version>
<configuration>
<archive>
<manifest>
<addDefaultEntries>false</addDefaultEntries>
<addDefaultImplementationEntries>true</addDefaultImplementationEntries>
</manifest>
<manifestEntries>
<Last-Commit-Id>${git.commit.id.full}</Last-Commit-Id>
<Last-Commit-Time>${git.commit.time}</Last-Commit-Time>
<Reproducible-Build>true</Reproducible-Build>
</manifestEntries>
</archive>
</configuration>
<version>${maven-jar-plugin.version}</version>
<executions>
<execution>
<configuration>
<archive>
<manifest>
<addDefaultEntries>false</addDefaultEntries>
<addDefaultImplementationEntries>true</addDefaultImplementationEntries>
</manifest>
<manifestEntries>
<Last-Commit-Id>${git.commit.id.full}</Last-Commit-Id>
<Last-Commit-Time>${git.commit.time}</Last-Commit-Time>
<Reproducible-Build>true</Reproducible-Build>
</manifestEntries>
</archive>
</configuration>
</execution>
</executions>
</plugin>
<!-- Copy modified hsqldb.jar to install / modified MANIFEST.MF-->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-install-plugin</artifactId>
<version>${maven-install-plugin.version}</version>
<configuration>
<groupId>org.hsqldb</groupId>
<artifactId>hsqldb</artifactId>
<version>${hsqldb.version}</version>
<packaging>jar</packaging>
</configuration>
<executions>
<execution>
<phase>install</phase>
<goals>
<goal>install-file</goal>
</goals>
<configuration>
<file>${project.basedir}/lib/org/hsqldb/hsqldb/${hsqldb.version}/hsqldb.jar</file>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>2.4.3</version>
<version>${maven-shade-plugin.version}</version>
<configuration>
<createDependencyReducedPom>false</createDependencyReducedPom>
<artifactSet>
<excludes>
<!-- Don't include original swagger-UI as we're including our own
modified version -->
<!-- Don't include original swagger-UI as we're including our own modified version -->
<exclude>org.webjars:swagger-ui</exclude>
<!-- Don't include JUnit as it's for testing only! -->
<exclude>junit:junit</exclude>
@@ -288,6 +379,7 @@
implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
<mainClass>org.qortal.controller.Controller</mainClass>
<manifestEntries>
<Multi-Release>true</Multi-Release>
<Class-Path>. ..</Class-Path>
</manifestEntries>
</transformer>
@@ -296,10 +388,11 @@
</execution>
</executions>
</plugin>
<!-- Removed, now use Maven reproducible by default v4.0, IntelliJ v2025.1 and later -->
<plugin>
<groupId>io.github.zlika</groupId>
<artifactId>reproducible-build-maven-plugin</artifactId>
<version>0.11</version>
<version>${maven-reproducible-build-plugin.version}</version>
<executions>
<execution>
<phase>package</phase>
@@ -316,9 +409,9 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>2.22.2</version>
<version>${maven-surefire-plugin.version}</version>
<configuration>
<skipTests>${skipTests}</skipTests>
<skipTests>${skipJUnitTests}</skipTests>
</configuration>
</plugin>
</plugins>
@@ -328,46 +421,34 @@
<plugin>
<groupId>org.eclipse.m2e</groupId>
<artifactId>lifecycle-mapping</artifactId>
<version>1.0.0</version>
<version>${lifecycle-mapping.version}</version>
<configuration>
<lifecycleMappingMetadata>
<pluginExecutions>
<pluginExecution>
<pluginExecutionFilter>
<groupId>
org.apache.maven.plugins
</groupId>
<artifactId>
maven-dependency-plugin
</artifactId>
<versionRange>
[2.8,)
</versionRange>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<version>${maven-dependency-plugin.version}</version>
<goals>
<goal>unpack</goal>
</goals>
</pluginExecutionFilter>
<action>
<execute></execute>
<execute/>
</action>
</pluginExecution>
<pluginExecution>
<pluginExecutionFilter>
<groupId>
com.google.code.maven-replacer-plugin
</groupId>
<artifactId>
replacer
</artifactId>
<versionRange>
[1.5.3,)
</versionRange>
<groupId>com.google.code.maven-replacer-plugin</groupId>
<artifactId>replacer</artifactId>
<version>${replacer.version}</version>
<goals>
<goal>replace</goal>
</goals>
</pluginExecutionFilter>
<action>
<execute></execute>
<execute/>
</action>
</pluginExecution>
</pluginExecutions>
@@ -383,23 +464,30 @@
<name>project</name>
<url>file:${project.basedir}/lib</url>
</repository>
<!-- jitpack for build-on-demand of altcoinj -->
<repository>
<id>jitpack.io</id>
<url>https://jitpack.io</url>
</repository>
</repositories>
<dependencies>
<!-- https://mvnrepository.com/artifact/org.codehaus.mojo/build-helper-maven-plugin -->
<dependency>
<groupId>org.codehaus.mojo</groupId>
<artifactId>build-helper-maven-plugin</artifactId>
<version>3.0.0</version>
<scope>provided</scope><!-- needed for build, not for runtime -->
<version>${maven-build-helper-plugin.version}</version>
<scope>provided</scope>
<!-- needed for build, not for runtime -->
</dependency>
<!-- https://mvnrepository.com/artifact/com.github.bohnman/package-info-maven-plugin -->
<dependency>
<groupId>com.github.bohnman</groupId>
<artifactId>package-info-maven-plugin</artifactId>
<version>${package-info-maven-plugin.version}</version>
<scope>provided</scope><!-- needed for build, not for runtime -->
<version>${maven-package-info-plugin.version}</version>
<scope>provided</scope>
<!-- needed for build, not for runtime -->
</dependency>
<!-- HSQLDB for repository -->
<!-- HSQLDB for repository should use local version with Sealed: false -->
<dependency>
<groupId>org.hsqldb</groupId>
<artifactId>hsqldb</artifactId>
@@ -407,32 +495,69 @@
</dependency>
<!-- CIYAM AT (automated transactions) -->
<dependency>
<groupId>org.ciyam</groupId>
<groupId>com.github.iceburst</groupId>
<artifactId>AT</artifactId>
<version>${ciyam-at.version}</version>
</dependency>
<!-- UPnP support -->
<dependency>
<groupId>com.dosse</groupId>
<artifactId>WaifUPnP</artifactId>
<version>${upnp.version}</version>
</dependency>
<!-- Bitcoin support -->
<dependency>
<groupId>org.bitcoinj</groupId>
<artifactId>bitcoinj-core</artifactId>
<version>${bitcoinj.version}</version>
</dependency>
<!-- For Litecoin, etc. support, requires bitcoinj -->
<dependency>
<groupId>com.github.iceburst</groupId>
<artifactId>altcoinj</artifactId>
<version>${altcoinj.version}</version>
</dependency>
<!-- Utilities -->
<dependency>
<groupId>com.googlecode.json-simple</groupId>
<artifactId>json-simple</artifactId>
<version>1.1.1</version>
<version>${json-simple.version}</version>
</dependency>
<dependency>
<groupId>org.json</groupId>
<artifactId>json</artifactId>
<version>${json.version}</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-text</artifactId>
<version>${commons-text.version}</version>
</dependency>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<version>${commons-io.version}</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-compress</artifactId>
<version>${commons-compress.version}</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<version>${commons-lang3.version}</version>
</dependency>
<dependency>
<groupId>org.tukaani</groupId>
<artifactId>xz</artifactId>
<version>${xz.version}</version>
</dependency>
<!-- For bitset/bitmap compression -->
<dependency>
<groupId>io.druid</groupId>
<artifactId>extendedset</artifactId>
<version>0.12.3</version>
<version>${extendedset.version}</version>
<exclusions>
<!-- exclude old versions of jackson-annotations / jackson-core -->
<exclusion>
@@ -503,18 +628,29 @@
<dependency>
<groupId>javax.servlet</groupId>
<artifactId>javax.servlet-api</artifactId>
<version>4.0.1</version>
<version>${javax.servlet-api.version}</version>
</dependency>
<dependency>
<groupId>javax.mail</groupId>
<artifactId>mail</artifactId>
<version>1.5.0-b01</version>
<version>${mail.version}</version>
</dependency>
<!-- Unicode homoglyph utilities -->
<dependency>
<groupId>net.codebox</groupId>
<artifactId>homoglyph</artifactId>
<version>1.2.0</version>
<version>${homoglyph.version}</version>
</dependency>
<!-- Unicode support -->
<dependency>
<groupId>com.ibm.icu</groupId>
<artifactId>icu4j</artifactId>
<version>${icu4j.version}</version>
</dependency>
<dependency>
<groupId>com.ibm.icu</groupId>
<artifactId>icu4j-charset</artifactId>
<version>${icu4j.version}</version>
</dependency>
<!-- Jetty -->
<dependency>
@@ -566,7 +702,8 @@
<artifactId>jersey-hk2</artifactId>
<version>${jersey.version}</version>
<exclusions>
<exclusion><!-- exclude javax.inject-1.jar because other jersey modules include javax.inject v2+ -->
<exclusion>
<!-- exclude javax.inject-1.jar because other jersey modules include javax.inject v2+ -->
<groupId>javax.inject</groupId>
<artifactId>javax.inject</artifactId>
</exclusion>
@@ -593,7 +730,8 @@
<artifactId>swagger-jaxrs2-servlet-initializer</artifactId>
<version>${swagger-api.version}</version>
<exclusions>
<exclusion><!-- excluded because included in swagger-jaxrs2-servlet-initializer -->
<exclusion>
<!-- excluded because included in swagger-jaxrs2-servlet-initializer -->
<groupId>io.swagger.core.v3</groupId>
<artifactId>swagger-integration</artifactId>
</exclusion>
@@ -609,24 +747,70 @@
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-engine</artifactId>
<version>5.3.1</version>
<version>${junit-jupiter-engine.version}</version>
</dependency>
<dependency>
<groupId>org.hamcrest</groupId>
<artifactId>hamcrest-library</artifactId>
<version>1.3</version>
<version>${hamcrest-library.version}</version>
</dependency>
-->
<!-- BouncyCastle for crypto, including TLS secure networking -->
<dependency>
<groupId>org.bouncycastle</groupId>
<artifactId>bcprov-jdk15on</artifactId>
<artifactId>bcprov-jdk15to18</artifactId>
<version>${bouncycastle.version}</version>
</dependency>
<dependency>
<groupId>org.bouncycastle</groupId>
<artifactId>bctls-jdk15on</artifactId>
<artifactId>bctls-jdk15to18</artifactId>
<version>${bouncycastle.version}</version>
</dependency>
<dependency>
<groupId>org.jsoup</groupId>
<artifactId>jsoup</artifactId>
<version>${jsoup.version}</version>
</dependency>
<dependency>
<groupId>io.github.java-diff-utils</groupId>
<artifactId>java-diff-utils</artifactId>
<version>${java-diff-utils.version}</version>
</dependency>
<dependency>
<groupId>io.grpc</groupId>
<artifactId>grpc-netty</artifactId>
<version>${grpc.version}</version>
</dependency>
<dependency>
<groupId>io.grpc</groupId>
<artifactId>grpc-protobuf</artifactId>
<version>${grpc.version}</version>
</dependency>
<dependency>
<groupId>io.grpc</groupId>
<artifactId>grpc-stub</artifactId>
<version>${grpc.version}</version>
</dependency>
<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
<version>${protobuf.version}</version>
</dependency>
<dependency>
<groupId>com.j256.simplemagic</groupId>
<artifactId>simplemagic</artifactId>
<version>${simplemagic.version}</version>
</dependency>
<!-- JAXB runtime for WADL support -->
<dependency>
<groupId>org.glassfish.jaxb</groupId>
<artifactId>jaxb-runtime</artifactId>
<version>${jaxb-runtime.version}</version>
</dependency>
<dependency>
<groupId>org.apache.tika</groupId>
<artifactId>tika-core</artifactId>
<version>3.1.0</version>
</dependency>
</dependencies>
</project>

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,100 @@
/*
* Copyright (C) 2009 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/*
* LiteWalletJni code based on https://github.com/PirateNetwork/cordova-plugin-litewallet
*
* MIT License
*
* Copyright (c) 2020 Zero Currency Coin
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package com.rust.litewalletjni;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.qortal.controller.PirateChainWalletController;
import java.nio.file.Path;
import java.nio.file.Paths;
public class LiteWalletJni {
protected static final Logger LOGGER = LogManager.getLogger(LiteWalletJni.class);
public static native String initlogging();
public static native String initnew(final String serveruri, final String params, final String saplingOutputb64, final String saplingSpendb64);
public static native String initfromseed(final String serveruri, final String params, final String seed, final String birthday, final String saplingOutputb64, final String saplingSpendb64);
public static native String initfromb64(final String serveruri, final String params, final String datab64, final String saplingOutputb64, final String saplingSpendb64);
public static native String save();
public static native String execute(final String cmd, final String args);
public static native String getseedphrase();
public static native String getseedphrasefromentropyb64(final String entropy64);
public static native String checkseedphrase(final String input);
private static boolean loaded = false;
public static void loadLibrary() {
if (loaded) {
return;
}
String osName = System.getProperty("os.name");
String osArchitecture = System.getProperty("os.arch");
LOGGER.info("OS Name: {}", osName);
LOGGER.info("OS Architecture: {}", osArchitecture);
try {
String libFileName = PirateChainWalletController.getRustLibFilename();
if (libFileName == null) {
LOGGER.info("Library not found for OS: {}, arch: {}", osName, osArchitecture);
return;
}
Path libPath = Paths.get(PirateChainWalletController.getRustLibOuterDirectory().toString(), libFileName);
System.load(libPath.toAbsolutePath().toString());
loaded = true;
}
catch (UnsatisfiedLinkError e) {
LOGGER.info("Unable to load library");
}
}
public static boolean isLoaded() {
return loaded;
}
}

View File

@@ -1,10 +1,10 @@
package org.hsqldb.jdbc;
import org.hsqldb.jdbc.pool.JDBCPooledConnection;
import java.sql.Connection;
import java.sql.SQLException;
import org.hsqldb.jdbc.pool.JDBCPooledConnection;
public class HSQLDBPool extends JDBCPool {
public HSQLDBPool(int poolSize) {
@@ -21,18 +21,28 @@ public class HSQLDBPool extends JDBCPool {
public Connection tryConnection() throws SQLException {
for (int i = 0; i < states.length(); i++) {
if (states.compareAndSet(i, RefState.available, RefState.allocated)) {
return connections[i].getConnection();
JDBCPooledConnection pooledConnection = connections[i];
if (pooledConnection == null)
// Probably shutdown situation
return null;
return pooledConnection.getConnection();
}
if (states.compareAndSet(i, RefState.empty, RefState.allocated)) {
try {
JDBCPooledConnection connection = (JDBCPooledConnection) source.getPooledConnection();
JDBCPooledConnection pooledConnection = (JDBCPooledConnection) source.getPooledConnection();
connection.addConnectionEventListener(this);
connection.addStatementEventListener(this);
connections[i] = connection;
if (pooledConnection == null)
// Probably shutdown situation
return null;
return connections[i].getConnection();
pooledConnection.addConnectionEventListener(this);
pooledConnection.addStatementEventListener(this);
connections[i] = pooledConnection;
return pooledConnection.getConnection();
} catch (SQLException e) {
states.set(i, RefState.empty);
}

View File

@@ -0,0 +1,173 @@
package org.hsqldb.jdbc;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.hsqldb.jdbc.pool.JDBCPooledConnection;
import org.qortal.data.system.DbConnectionInfo;
import org.qortal.repository.hsqldb.HSQLDBRepositoryFactory;
import javax.sql.ConnectionEvent;
import javax.sql.PooledConnection;
import java.sql.Connection;
import java.sql.SQLException;
import java.util.Comparator;
import java.util.List;
import java.util.concurrent.ConcurrentHashMap;
import java.util.stream.Collectors;
/**
* Class HSQLDBPoolMonitored
*
* This class uses the same logic as HSQLDBPool. The only difference is it monitors the state of every connection
* to the database. This is used for debugging purposes only.
*/
public class HSQLDBPoolMonitored extends HSQLDBPool {
private static final Logger LOGGER = LogManager.getLogger(HSQLDBRepositoryFactory.class);
private static final String EMPTY = "Empty";
private static final String AVAILABLE = "Available";
private static final String ALLOCATED = "Allocated";
private ConcurrentHashMap<Integer, DbConnectionInfo> infoByIndex;
public HSQLDBPoolMonitored(int poolSize) {
super(poolSize);
this.infoByIndex = new ConcurrentHashMap<>(poolSize);
}
/**
* Tries to retrieve a new connection using the properties that have already been
* set.
*
* @return a connection to the data source, or null if no spare connections in pool
* @exception SQLException if a database access error occurs
*/
public Connection tryConnection() throws SQLException {
for (int i = 0; i < states.length(); i++) {
if (states.compareAndSet(i, RefState.available, RefState.allocated)) {
JDBCPooledConnection pooledConnection = connections[i];
if (pooledConnection == null)
// Probably shutdown situation
return null;
infoByIndex.put(i, new DbConnectionInfo(System.currentTimeMillis(), Thread.currentThread().getName(), ALLOCATED));
return pooledConnection.getConnection();
}
if (states.compareAndSet(i, RefState.empty, RefState.allocated)) {
try {
JDBCPooledConnection pooledConnection = (JDBCPooledConnection) source.getPooledConnection();
if (pooledConnection == null)
// Probably shutdown situation
return null;
pooledConnection.addConnectionEventListener(this);
pooledConnection.addStatementEventListener(this);
connections[i] = pooledConnection;
infoByIndex.put(i, new DbConnectionInfo(System.currentTimeMillis(), Thread.currentThread().getName(), ALLOCATED));
return pooledConnection.getConnection();
} catch (SQLException e) {
states.set(i, RefState.empty);
infoByIndex.put(i, new DbConnectionInfo(System.currentTimeMillis(), Thread.currentThread().getName(), EMPTY));
}
}
}
return null;
}
public Connection getConnection() throws SQLException {
int var1 = 300;
if (this.source.loginTimeout != 0) {
var1 = this.source.loginTimeout * 10;
}
if (this.closed) {
throw new SQLException("connection pool is closed");
} else {
for(int var2 = 0; var2 < var1; ++var2) {
for(int var3 = 0; var3 < this.states.length(); ++var3) {
if (this.states.compareAndSet(var3, 1, 2)) {
infoByIndex.put(var3, new DbConnectionInfo(System.currentTimeMillis(), Thread.currentThread().getName(), ALLOCATED));
return this.connections[var3].getConnection();
}
if (this.states.compareAndSet(var3, 0, 2)) {
try {
JDBCPooledConnection var4 = (JDBCPooledConnection)this.source.getPooledConnection();
var4.addConnectionEventListener(this);
var4.addStatementEventListener(this);
this.connections[var3] = var4;
infoByIndex.put(var3, new DbConnectionInfo(System.currentTimeMillis(), Thread.currentThread().getName(), ALLOCATED));
return this.connections[var3].getConnection();
} catch (SQLException var6) {
this.states.set(var3, 0);
infoByIndex.put(var3, new DbConnectionInfo(System.currentTimeMillis(), Thread.currentThread().getName(), EMPTY));
}
}
}
try {
Thread.sleep(100L);
} catch (InterruptedException var5) {
}
}
throw JDBCUtil.invalidArgument();
}
}
public void connectionClosed(ConnectionEvent event) {
PooledConnection connection = (PooledConnection) event.getSource();
for (int i = 0; i < connections.length; i++) {
if (connections[i] == connection) {
states.set(i, RefState.available);
infoByIndex.put(i, new DbConnectionInfo(System.currentTimeMillis(), Thread.currentThread().getName(), AVAILABLE));
break;
}
}
}
public void connectionErrorOccurred(ConnectionEvent event) {
PooledConnection connection = (PooledConnection) event.getSource();
for (int i = 0; i < connections.length; i++) {
if (connections[i] == connection) {
states.set(i, RefState.allocated);
connections[i] = null;
states.set(i, RefState.empty);
infoByIndex.put(i, new DbConnectionInfo(System.currentTimeMillis(), Thread.currentThread().getName(), EMPTY));
break;
}
}
}
public List<DbConnectionInfo> getDbConnectionsStates() {
return infoByIndex.values().stream()
.sorted(Comparator.comparingLong(DbConnectionInfo::getUpdated))
.collect(Collectors.toList());
}
private int findConnectionIndex(ConnectionEvent connectionEvent) {
PooledConnection pooledConnection = (PooledConnection) connectionEvent.getSource();
for(int i = 0; i < this.connections.length; ++i) {
if (this.connections[i] == pooledConnection) {
return i;
}
}
return -1;
}
}

View File

@@ -0,0 +1,227 @@
package org.qortal;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.bouncycastle.jce.provider.BouncyCastleProvider;
import org.bouncycastle.jsse.provider.BouncyCastleJsseProvider;
import org.qortal.api.ApiKey;
import org.qortal.api.ApiRequest;
import org.qortal.controller.BootstrapNode;
import org.qortal.settings.Settings;
import java.io.IOException;
import java.lang.management.ManagementFactory;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.attribute.BasicFileAttributes;
import java.security.Security;
import java.util.*;
import java.util.stream.Collectors;
import static org.qortal.controller.BootstrapNode.AGENTLIB_JVM_HOLDER_ARG;
public class ApplyBootstrap {
static {
// This static block will be called before others if using ApplyBootstrap.main()
// Log into different files for bootstrap - this has to be before LogManger.getLogger() calls
System.setProperty("log4j2.filenameTemplate", "log-apply-bootstrap.txt");
// This must go before any calls to LogManager/Logger
System.setProperty("java.util.logging.manager", "org.apache.logging.log4j.jul.LogManager");
}
private static final Logger LOGGER = LogManager.getLogger(ApplyBootstrap.class);
private static final String JAR_FILENAME = BootstrapNode.JAR_FILENAME;
private static final String WINDOWS_EXE_LAUNCHER = "qortal.exe";
private static final String JAVA_TOOL_OPTIONS_NAME = "JAVA_TOOL_OPTIONS";
private static final String JAVA_TOOL_OPTIONS_VALUE = "";
private static final long CHECK_INTERVAL = 15 * 1000L; // ms
private static final int MAX_ATTEMPTS = 20;
public static void main(String[] args) {
Security.insertProviderAt(new BouncyCastleProvider(), 0);
Security.insertProviderAt(new BouncyCastleJsseProvider(), 1);
// Load/check settings, which potentially sets up blockchain config, etc.
if (args.length > 0)
Settings.fileInstance(args[0]);
else
Settings.getInstance();
LOGGER.info("Applying bootstrap...");
// Shutdown node using API
if (!shutdownNode())
return;
// Delete db
deleteDB();
// Restart node
restartNode(args);
LOGGER.info("Bootstrapping...");
}
private static boolean shutdownNode() {
String baseUri = "http://localhost:" + Settings.getInstance().getApiPort() + "/";
LOGGER.info(() -> String.format("Shutting down node using API via %s", baseUri));
// The /admin/stop endpoint requires an API key, which may or may not be already generated
boolean apiKeyNewlyGenerated = false;
ApiKey apiKey = null;
try {
apiKey = new ApiKey();
if (!apiKey.generated()) {
apiKey.generate();
apiKeyNewlyGenerated = true;
LOGGER.info("Generated API key");
}
} catch (IOException e) {
LOGGER.info("Error loading API key: {}", e.getMessage());
}
// Create GET params
Map<String, String> params = new HashMap<>();
if (apiKey != null) {
params.put("apiKey", apiKey.toString());
}
// Attempt to stop the node
int attempt;
for (attempt = 0; attempt < MAX_ATTEMPTS; ++attempt) {
final int attemptForLogging = attempt;
LOGGER.info(() -> String.format("Attempt #%d out of %d to shutdown node", attemptForLogging + 1, MAX_ATTEMPTS));
String response = ApiRequest.perform(baseUri + "admin/stop", params);
if (response == null) {
// No response - consider node shut down
if (apiKeyNewlyGenerated) {
// API key was newly generated for bootstrapping node, so we need to remove it
ApplyBootstrap.removeGeneratedApiKey();
}
return true;
}
LOGGER.info(() -> String.format("Response from API: %s", response));
try {
Thread.sleep(CHECK_INTERVAL);
} catch (InterruptedException e) {
// We still need to check...
break;
}
}
if (apiKeyNewlyGenerated) {
// API key was newly generated for bootstrapping node, so we need to remove it
ApplyBootstrap.removeGeneratedApiKey();
}
if (attempt == MAX_ATTEMPTS) {
LOGGER.error("Failed to shutdown node - giving up");
return false;
}
return true;
}
private static void removeGeneratedApiKey() {
try {
LOGGER.info("Removing newly generated API key...");
// Delete the API key since it was only generated for bootstrapping node
ApiKey apiKey = new ApiKey();
apiKey.delete();
} catch (IOException e) {
LOGGER.info("Error loading or deleting API key: {}", e.getMessage());
}
}
private static void deleteDB() {
// Get the repository path from settings
String repositoryPath = Settings.getInstance().getRepositoryPath();
LOGGER.debug(String.format("Repository path: %s", repositoryPath));
try {
Path directory = Paths.get(repositoryPath);
Files.walkFileTree(directory, new SimpleFileVisitor<Path>() {
@Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
Files.delete(file);
return FileVisitResult.CONTINUE;
}
@Override
public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException {
Files.delete(dir);
return FileVisitResult.CONTINUE;
}
});
} catch (IOException e) {
LOGGER.error("Error deleting DB: {}", e.getMessage());
}
}
private static void restartNode(String[] args) {
String javaHome = System.getProperty("java.home");
LOGGER.debug(() -> String.format("Java home: %s", javaHome));
Path javaBinary = Paths.get(javaHome, "bin", "java");
LOGGER.debug(() -> String.format("Java binary: %s", javaBinary));
Path exeLauncher = Paths.get(WINDOWS_EXE_LAUNCHER);
LOGGER.debug(() -> String.format("Windows EXE launcher: %s", exeLauncher));
List<String> javaCmd;
if (Files.exists(exeLauncher)) {
javaCmd = Arrays.asList(exeLauncher.toString());
} else {
javaCmd = new ArrayList<>();
// Java runtime binary itself
javaCmd.add(javaBinary.toString());
// JVM arguments
javaCmd.addAll(ManagementFactory.getRuntimeMXBean().getInputArguments());
// Reapply any retained, but disabled, -agentlib JVM arg
javaCmd = javaCmd.stream()
.map(arg -> arg.replace(AGENTLIB_JVM_HOLDER_ARG, "-agentlib"))
.collect(Collectors.toList());
// Call mainClass in JAR
javaCmd.addAll(Arrays.asList("-jar", JAR_FILENAME));
// Add saved command-line args
javaCmd.addAll(Arrays.asList(args));
}
try {
LOGGER.info(String.format("Restarting node with: %s", String.join(" ", javaCmd)));
ProcessBuilder processBuilder = new ProcessBuilder(javaCmd);
if (Files.exists(exeLauncher)) {
LOGGER.debug(() -> String.format("Setting env %s to %s", JAVA_TOOL_OPTIONS_NAME, JAVA_TOOL_OPTIONS_VALUE));
processBuilder.environment().put(JAVA_TOOL_OPTIONS_NAME, JAVA_TOOL_OPTIONS_VALUE);
}
// New process will inherit our stdout and stderr
processBuilder.redirectOutput(ProcessBuilder.Redirect.INHERIT);
processBuilder.redirectError(ProcessBuilder.Redirect.INHERIT);
Process process = processBuilder.start();
// Nothing to pipe to new process, so close output stream (process's stdin)
process.getOutputStream().close();
} catch (Exception e) {
LOGGER.error(String.format("Failed to restart node (BAD): %s", e.getMessage()));
}
}
}

View File

@@ -0,0 +1,241 @@
package org.qortal;
import org.apache.commons.io.FileUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.bouncycastle.jce.provider.BouncyCastleProvider;
import org.bouncycastle.jsse.provider.BouncyCastleJsseProvider;
import org.qortal.api.ApiKey;
import org.qortal.api.ApiRequest;
import org.qortal.controller.Controller;
import org.qortal.controller.RestartNode;
import org.qortal.settings.Settings;
import java.io.File;
import java.io.IOException;
import java.lang.management.ManagementFactory;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.security.Security;
import java.util.*;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.ReentrantLock;
import java.util.stream.Collectors;
import static org.qortal.controller.RestartNode.AGENTLIB_JVM_HOLDER_ARG;
public class ApplyRestart {
static {
// This static block will be called before others if using ApplyRestart.main()
// Log into different files for restart node - this has to be before LogManger.getLogger() calls
System.setProperty("log4j2.filenameTemplate", "log-apply-restart.txt");
// This must go before any calls to LogManager/Logger
System.setProperty("java.util.logging.manager", "org.apache.logging.log4j.jul.LogManager");
}
private static final Logger LOGGER = LogManager.getLogger(ApplyRestart.class);
private static final String JAR_FILENAME = RestartNode.JAR_FILENAME;
private static final String WINDOWS_EXE_LAUNCHER = "qortal.exe";
private static final String JAVA_TOOL_OPTIONS_NAME = "JAVA_TOOL_OPTIONS";
private static final String JAVA_TOOL_OPTIONS_VALUE = "";
private static final long CHECK_INTERVAL = 30 * 1000L; // ms
private static final int MAX_ATTEMPTS = 12;
public static void main(String[] args) {
Security.insertProviderAt(new BouncyCastleProvider(), 0);
Security.insertProviderAt(new BouncyCastleJsseProvider(), 1);
// Load/check settings, which potentially sets up blockchain config, etc.
if (args.length > 0)
Settings.fileInstance(args[0]);
else
Settings.getInstance();
LOGGER.info("Applying restart this can take up to 5 minutes...");
// Shutdown node using API
if (!shutdownNode())
return;
try {
// Give some time for shutdown
TimeUnit.SECONDS.sleep(60);
// Remove blockchain lock if exist
ReentrantLock blockchainLock = Controller.getInstance().getBlockchainLock();
if (blockchainLock.isLocked())
blockchainLock.unlock();
// Remove blockchain lock file if still exist
TimeUnit.SECONDS.sleep(60);
deleteLock();
// Restart node
TimeUnit.SECONDS.sleep(15);
restartNode(args);
LOGGER.info("Restarting...");
} catch (InterruptedException e) {
LOGGER.error("Unable to restart", e);
}
}
private static boolean shutdownNode() {
String baseUri = "http://localhost:" + Settings.getInstance().getApiPort() + "/";
LOGGER.debug(() -> String.format("Shutting down node using API via %s", baseUri));
// The /admin/stop endpoint requires an API key, which may or may not be already generated
boolean apiKeyNewlyGenerated = false;
ApiKey apiKey = null;
try {
apiKey = new ApiKey();
if (!apiKey.generated()) {
apiKey.generate();
apiKeyNewlyGenerated = true;
LOGGER.info("Generated API key");
}
} catch (IOException e) {
LOGGER.info("Error loading API key: {}", e.getMessage());
}
// Create GET params
Map<String, String> params = new HashMap<>();
if (apiKey != null) {
params.put("apiKey", apiKey.toString());
}
// Attempt to stop the node
int attempt;
for (attempt = 0; attempt < MAX_ATTEMPTS; ++attempt) {
final int attemptForLogging = attempt;
LOGGER.info(() -> String.format("Attempt #%d out of %d to shutdown node", attemptForLogging + 1, MAX_ATTEMPTS));
String response = ApiRequest.perform(baseUri + "admin/stop", params);
if (response == null) {
// No response - consider node shut down
try {
TimeUnit.SECONDS.sleep(30);
} catch (InterruptedException e) {
throw new RuntimeException(e);
}
if (apiKeyNewlyGenerated) {
// API key was newly generated for restarting node, so we need to remove it
ApplyRestart.removeGeneratedApiKey();
}
return true;
}
LOGGER.info(() -> String.format("Response from API: %s", response));
try {
Thread.sleep(CHECK_INTERVAL);
} catch (InterruptedException e) {
// We still need to check...
break;
}
}
if (apiKeyNewlyGenerated) {
// API key was newly generated for restarting node, so we need to remove it
ApplyRestart.removeGeneratedApiKey();
}
if (attempt == MAX_ATTEMPTS) {
LOGGER.error("Failed to shutdown node - giving up");
return false;
}
return true;
}
private static void removeGeneratedApiKey() {
try {
LOGGER.info("Removing newly generated API key...");
// Delete the API key since it was only generated for restarting node
ApiKey apiKey = new ApiKey();
apiKey.delete();
} catch (IOException e) {
LOGGER.error("Error loading or deleting API key: {}", e.getMessage());
}
}
private static void deleteLock() {
// Get the repository path from settings
String repositoryPath = Settings.getInstance().getRepositoryPath();
LOGGER.debug(String.format("Repository path is: %s", repositoryPath));
try {
Path root = Paths.get(repositoryPath);
File lockFile = new File(root.resolve("blockchain.lck").toUri());
LOGGER.debug("Lockfile is: {}", lockFile);
FileUtils.forceDelete(FileUtils.getFile(lockFile));
} catch (IOException e) {
LOGGER.debug("Error deleting blockchain lock file: {}", e.getMessage());
}
}
private static void restartNode(String[] args) {
String javaHome = System.getProperty("java.home");
LOGGER.debug(() -> String.format("Java home: %s", javaHome));
Path javaBinary = Paths.get(javaHome, "bin", "java");
LOGGER.debug(() -> String.format("Java binary: %s", javaBinary));
Path exeLauncher = Paths.get(WINDOWS_EXE_LAUNCHER);
LOGGER.debug(() -> String.format("Windows EXE launcher: %s", exeLauncher));
List<String> javaCmd;
if (Files.exists(exeLauncher)) {
javaCmd = List.of(exeLauncher.toString());
} else {
javaCmd = new ArrayList<>();
// Java runtime binary itself
javaCmd.add(javaBinary.toString());
// JVM arguments
javaCmd.addAll(ManagementFactory.getRuntimeMXBean().getInputArguments());
// Reapply any retained, but disabled, -agentlib JVM arg
javaCmd = javaCmd.stream()
.map(arg -> arg.replace(AGENTLIB_JVM_HOLDER_ARG, "-agentlib"))
.collect(Collectors.toList());
// Call mainClass in JAR
javaCmd.addAll(Arrays.asList("-jar", JAR_FILENAME));
// Add saved command-line args
javaCmd.addAll(Arrays.asList(args));
}
try {
LOGGER.debug(String.format("Restarting node with: %s", String.join(" ", javaCmd)));
ProcessBuilder processBuilder = new ProcessBuilder(javaCmd);
if (Files.exists(exeLauncher)) {
LOGGER.debug(() -> String.format("Setting env %s to %s", JAVA_TOOL_OPTIONS_NAME, JAVA_TOOL_OPTIONS_VALUE));
processBuilder.environment().put(JAVA_TOOL_OPTIONS_NAME, JAVA_TOOL_OPTIONS_VALUE);
}
// New process will inherit our stdout and stderr
processBuilder.redirectOutput(ProcessBuilder.Redirect.INHERIT);
processBuilder.redirectError(ProcessBuilder.Redirect.INHERIT);
Process process = processBuilder.start();
// Nothing to pipe to new process, so close output stream (process's stdin)
process.getOutputStream().close();
} catch (Exception e) {
LOGGER.error(String.format("Failed to restart node (BAD): %s", e.getMessage()));
}
}
}

View File

@@ -1,5 +1,14 @@
package org.qortal;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.bouncycastle.jce.provider.BouncyCastleProvider;
import org.bouncycastle.jsse.provider.BouncyCastleJsseProvider;
import org.qortal.api.ApiKey;
import org.qortal.api.ApiRequest;
import org.qortal.controller.AutoUpdate;
import org.qortal.settings.Settings;
import java.io.IOException;
import java.lang.management.ManagementFactory;
import java.nio.file.Files;
@@ -7,17 +16,10 @@ import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import java.security.Security;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.*;
import java.util.stream.Collectors;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.bouncycastle.jce.provider.BouncyCastleProvider;
import org.bouncycastle.jsse.provider.BouncyCastleJsseProvider;
import org.qortal.api.ApiRequest;
import org.qortal.controller.AutoUpdate;
import org.qortal.settings.Settings;
import static org.qortal.controller.AutoUpdate.AGENTLIB_JVM_HOLDER_ARG;
public class ApplyUpdate {
@@ -36,9 +38,9 @@ public class ApplyUpdate {
private static final String NEW_JAR_FILENAME = AutoUpdate.NEW_JAR_FILENAME;
private static final String WINDOWS_EXE_LAUNCHER = "qortal.exe";
private static final String JAVA_TOOL_OPTIONS_NAME = "JAVA_TOOL_OPTIONS";
private static final String JAVA_TOOL_OPTIONS_VALUE = "-XX:MaxRAMFraction=4";
private static final String JAVA_TOOL_OPTIONS_VALUE = "";
private static final long CHECK_INTERVAL = 10 * 1000L; // ms
private static final long CHECK_INTERVAL = 30 * 1000L; // ms
private static final int MAX_ATTEMPTS = 12;
public static void main(String[] args) {
@@ -70,14 +72,40 @@ public class ApplyUpdate {
String baseUri = "http://localhost:" + Settings.getInstance().getApiPort() + "/";
LOGGER.info(() -> String.format("Shutting down node using API via %s", baseUri));
// The /admin/stop endpoint requires an API key, which may or may not be already generated
boolean apiKeyNewlyGenerated = false;
ApiKey apiKey = null;
try {
apiKey = new ApiKey();
if (!apiKey.generated()) {
apiKey.generate();
apiKeyNewlyGenerated = true;
LOGGER.info("Generated API key");
}
} catch (IOException e) {
LOGGER.info("Error loading API key: {}", e.getMessage());
}
// Create GET params
Map<String, String> params = new HashMap<>();
if (apiKey != null) {
params.put("apiKey", apiKey.toString());
}
// Attempt to stop the node
int attempt;
for (attempt = 0; attempt < MAX_ATTEMPTS; ++attempt) {
final int attemptForLogging = attempt;
LOGGER.info(() -> String.format("Attempt #%d out of %d to shutdown node", attemptForLogging + 1, MAX_ATTEMPTS));
String response = ApiRequest.perform(baseUri + "admin/stop", null);
if (response == null)
String response = ApiRequest.perform(baseUri + "admin/stop", params);
if (response == null) {
// No response - consider node shut down
if (apiKeyNewlyGenerated) {
// API key was newly generated for this auto update, so we need to remove it
ApplyUpdate.removeGeneratedApiKey();
}
return true;
}
LOGGER.info(() -> String.format("Response from API: %s", response));
@@ -89,6 +117,11 @@ public class ApplyUpdate {
}
}
if (apiKeyNewlyGenerated) {
// API key was newly generated for this auto update, so we need to remove it
ApplyUpdate.removeGeneratedApiKey();
}
if (attempt == MAX_ATTEMPTS) {
LOGGER.error("Failed to shutdown node - giving up");
return false;
@@ -97,6 +130,19 @@ public class ApplyUpdate {
return true;
}
private static void removeGeneratedApiKey() {
try {
LOGGER.info("Removing newly generated API key...");
// Delete the API key since it was only generated for this auto update
ApiKey apiKey = new ApiKey();
apiKey.delete();
} catch (IOException e) {
LOGGER.error("Error loading or deleting API key: {}", e.getMessage());
}
}
private static void replaceJar() {
// Assuming current working directory contains the JAR files
Path realJar = Paths.get(JAR_FILENAME);
@@ -135,13 +181,13 @@ public class ApplyUpdate {
private static void restartNode(String[] args) {
String javaHome = System.getProperty("java.home");
LOGGER.info(() -> String.format("Java home: %s", javaHome));
LOGGER.debug(() -> String.format("Java home: %s", javaHome));
Path javaBinary = Paths.get(javaHome, "bin", "java");
LOGGER.info(() -> String.format("Java binary: %s", javaBinary));
LOGGER.debug(() -> String.format("Java binary: %s", javaBinary));
Path exeLauncher = Paths.get(WINDOWS_EXE_LAUNCHER);
LOGGER.info(() -> String.format("Windows EXE launcher: %s", exeLauncher));
LOGGER.debug(() -> String.format("Windows EXE launcher: %s", exeLauncher));
List<String> javaCmd;
if (Files.exists(exeLauncher)) {
@@ -154,6 +200,11 @@ public class ApplyUpdate {
// JVM arguments
javaCmd.addAll(ManagementFactory.getRuntimeMXBean().getInputArguments());
// Reapply any retained, but disabled, -agentlib JVM arg
javaCmd = javaCmd.stream()
.map(arg -> arg.replace(AGENTLIB_JVM_HOLDER_ARG, "-agentlib"))
.collect(Collectors.toList());
// Call mainClass in JAR
javaCmd.addAll(Arrays.asList("-jar", JAR_FILENAME));
@@ -162,17 +213,24 @@ public class ApplyUpdate {
}
try {
LOGGER.info(() -> String.format("Restarting node with: %s", String.join(" ", javaCmd)));
LOGGER.debug(String.format("Restarting node with: %s", String.join(" ", javaCmd)));
ProcessBuilder processBuilder = new ProcessBuilder(javaCmd);
if (Files.exists(exeLauncher)) {
LOGGER.info(() -> String.format("Setting env %s to %s", JAVA_TOOL_OPTIONS_NAME, JAVA_TOOL_OPTIONS_VALUE));
LOGGER.debug(() -> String.format("Setting env %s to %s", JAVA_TOOL_OPTIONS_NAME, JAVA_TOOL_OPTIONS_VALUE));
processBuilder.environment().put(JAVA_TOOL_OPTIONS_NAME, JAVA_TOOL_OPTIONS_VALUE);
}
processBuilder.start();
} catch (IOException e) {
// New process will inherit our stdout and stderr
processBuilder.redirectOutput(ProcessBuilder.Redirect.INHERIT);
processBuilder.redirectError(ProcessBuilder.Redirect.INHERIT);
Process process = processBuilder.start();
// Nothing to pipe to new process, so close output stream (process's stdin)
process.getOutputStream().close();
} catch (Exception e) {
LOGGER.error(String.format("Failed to restart node (BAD): %s", e.getMessage()));
}
}

View File

@@ -1,7 +1,5 @@
package org.qortal;
import java.security.Security;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.bouncycastle.jce.provider.BouncyCastleProvider;
@@ -14,6 +12,9 @@ import org.qortal.repository.RepositoryManager;
import org.qortal.repository.hsqldb.HSQLDBRepositoryFactory;
import org.qortal.settings.Settings;
import java.security.Security;
import java.util.concurrent.TimeoutException;
public class RepositoryMaintenance {
static {
@@ -57,10 +58,10 @@ public class RepositoryMaintenance {
LOGGER.info("Starting repository periodic maintenance. This can take a while...");
try (final Repository repository = RepositoryManager.getRepository()) {
repository.performPeriodicMaintenance();
repository.performPeriodicMaintenance(null);
LOGGER.info("Repository periodic maintenance completed");
} catch (DataException e) {
} catch (DataException | TimeoutException e) {
LOGGER.error("Repository periodic maintenance failed", e);
}

View File

@@ -1,5 +1,7 @@
package org.qortal;
import org.qortal.controller.AutoUpdate;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
@@ -7,8 +9,6 @@ import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import org.qortal.controller.AutoUpdate;
public class XorUpdate {
private static final byte XOR_VALUE = AutoUpdate.XOR_VALUE;

View File

@@ -1,19 +1,33 @@
package org.qortal.account;
import static org.qortal.utils.Amounts.prettyAmount;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.qortal.api.resource.TransactionsResource;
import org.qortal.block.BlockChain;
import org.qortal.controller.LiteNode;
import org.qortal.data.account.AccountBalanceData;
import org.qortal.data.account.AccountData;
import org.qortal.data.account.RewardShareData;
import org.qortal.data.naming.NameData;
import org.qortal.data.transaction.TransactionData;
import org.qortal.repository.DataException;
import org.qortal.repository.GroupRepository;
import org.qortal.repository.NameRepository;
import org.qortal.repository.Repository;
import org.qortal.settings.Settings;
import org.qortal.utils.Base58;
import org.qortal.utils.Groups;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.qortal.block.BlockChain;
import org.qortal.data.account.AccountBalanceData;
import org.qortal.data.account.AccountData;
import org.qortal.data.account.RewardShareData;
import org.qortal.repository.DataException;
import org.qortal.repository.Repository;
import org.qortal.utils.Base58;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import static org.qortal.utils.Amounts.prettyAmount;
@XmlAccessorType(XmlAccessType.NONE) // Stops JAX-RS errors when unmarshalling blockchain config
public class Account {
@@ -59,7 +73,17 @@ public class Account {
// Balance manipulations - assetId is 0 for QORT
public long getConfirmedBalance(long assetId) throws DataException {
AccountBalanceData accountBalanceData = this.repository.getAccountRepository().getBalance(this.address, assetId);
AccountBalanceData accountBalanceData;
if (Settings.getInstance().isLite()) {
// Lite nodes request data from peers instead of the local db
accountBalanceData = LiteNode.getInstance().fetchAccountBalance(this.address, assetId);
}
else {
// All other node types fetch from the local db
accountBalanceData = this.repository.getAccountRepository().getBalance(this.address, assetId);
}
if (accountBalanceData == null)
return 0;
@@ -181,30 +205,99 @@ public class Account {
/** Returns whether account can be considered a "minting account".
* <p>
* To be considered a "minting account", the account needs to pass at least one of these tests:<br>
* To be considered a "minting account", the account needs to pass some of these tests:<br>
* <ul>
* <li>account's level is at least <tt>minAccountLevelToMint</tt> from blockchain config</li>
* <li>account has 'founder' flag set</li>
* <li>account's address has registered a name</li>
* <li>account's address is a member of the minter group</li>
* </ul>
*
*
* @param isGroupValidated true if this account has already been validated for MINTER Group membership
* @return true if account can be considered "minting account"
* @throws DataException
*/
public boolean canMint() throws DataException {
public boolean canMint(boolean isGroupValidated) throws DataException {
AccountData accountData = this.repository.getAccountRepository().getAccount(this.address);
if (accountData == null)
return false;
NameRepository nameRepository = this.repository.getNameRepository();
GroupRepository groupRepository = this.repository.getGroupRepository();
String myAddress = accountData.getAddress();
Integer level = accountData.getLevel();
if (level != null && level >= BlockChain.getInstance().getMinAccountLevelToMint())
return true;
int blockchainHeight = this.repository.getBlockRepository().getBlockchainHeight();
if (Account.isFounder(accountData.getFlags()))
return true;
int levelToMint;
if( blockchainHeight >= BlockChain.getInstance().getIgnoreLevelForRewardShareHeight() ) {
levelToMint = 0;
}
else {
levelToMint = BlockChain.getInstance().getMinAccountLevelToMint();
}
int level = accountData.getLevel();
List<Integer> groupIdsToMint = Groups.getGroupIdsToMint( BlockChain.getInstance(), blockchainHeight );
int nameCheckHeight = BlockChain.getInstance().getOnlyMintWithNameHeight();
int groupCheckHeight = BlockChain.getInstance().getGroupMemberCheckHeight();
int removeNameCheckHeight = BlockChain.getInstance().getRemoveOnlyMintWithNameHeight();
// Can only mint if:
// Account's level is at least minAccountLevelToMint from blockchain config
if (blockchainHeight < nameCheckHeight) {
if (Account.isFounder(accountData.getFlags())) {
return accountData.getBlocksMintedPenalty() == 0;
} else {
return level >= levelToMint;
}
}
// Can only mint on onlyMintWithNameHeight from blockchain config if:
// Account's level is at least minAccountLevelToMint from blockchain config
// Account's address has registered a name
if (blockchainHeight >= nameCheckHeight && blockchainHeight < groupCheckHeight) {
List<NameData> myName = nameRepository.getNamesByOwner(myAddress);
if (Account.isFounder(accountData.getFlags())) {
return accountData.getBlocksMintedPenalty() == 0 && !myName.isEmpty();
} else {
return level >= levelToMint && !myName.isEmpty();
}
}
// Can only mint on groupMemberCheckHeight from blockchain config if:
// Account's level is at least minAccountLevelToMint from blockchain config
// Account's address has registered a name
// Account's address is a member of the minter group
if (blockchainHeight >= groupCheckHeight && blockchainHeight < removeNameCheckHeight) {
List<NameData> myName = nameRepository.getNamesByOwner(myAddress);
if (Account.isFounder(accountData.getFlags())) {
return accountData.getBlocksMintedPenalty() == 0 && !myName.isEmpty() && (isGroupValidated || Groups.memberExistsInAnyGroup(groupRepository, groupIdsToMint, myAddress));
} else {
return level >= levelToMint && !myName.isEmpty() && (isGroupValidated || Groups.memberExistsInAnyGroup(groupRepository, groupIdsToMint, myAddress));
}
}
// Can only mint on removeOnlyMintWithNameHeight from blockchain config if:
// Account's level is at least minAccountLevelToMint from blockchain config
// Account's address is a member of the minter group
if (blockchainHeight >= removeNameCheckHeight) {
if (Account.isFounder(accountData.getFlags())) {
return accountData.getBlocksMintedPenalty() == 0 && (isGroupValidated || Groups.memberExistsInAnyGroup(groupRepository, groupIdsToMint, myAddress));
} else {
return level >= levelToMint && (isGroupValidated || Groups.memberExistsInAnyGroup(groupRepository, groupIdsToMint, myAddress));
}
}
return false;
}
/** Returns account's blockMinted (0+) or null if account not found in repository. */
public Integer getBlocksMinted() throws DataException {
return this.repository.getAccountRepository().getMintedBlockCount(this.address);
}
/** Returns account's blockMintedPenalty or null if account not found in repository. */
public Integer getBlocksMintedPenalty() throws DataException {
return this.repository.getAccountRepository().getBlocksMintedPenaltyCount(this.address);
}
/** Returns whether account can build reward-shares.
* <p>
* To be able to create reward-shares, the account needs to pass at least one of these tests:<br>
@@ -218,6 +311,7 @@ public class Account {
*/
public boolean canRewardShare() throws DataException {
AccountData accountData = this.repository.getAccountRepository().getAccount(this.address);
if (accountData == null)
return false;
@@ -225,7 +319,10 @@ public class Account {
if (level != null && level >= BlockChain.getInstance().getMinAccountLevelToRewardShare())
return true;
if (Account.isFounder(accountData.getFlags()))
if (Account.isFounder(accountData.getFlags()) && accountData.getBlocksMintedPenalty() == 0)
return true;
if( this.repository.getBlockRepository().getBlockchainHeight() >= BlockChain.getInstance().getIgnoreLevelForRewardShareHeight() )
return true;
return false;
@@ -253,7 +350,7 @@ public class Account {
/**
* Returns 'effective' minting level, or zero if account does not exist/cannot mint.
* <p>
* For founder accounts, this returns "founderEffectiveMintingLevel" from blockchain config.
* For founder accounts with no penalty, this returns "founderEffectiveMintingLevel" from blockchain config.
*
* @return 0+
* @throws DataException
@@ -263,19 +360,172 @@ public class Account {
if (accountData == null)
return 0;
if (Account.isFounder(accountData.getFlags()))
// Founders are assigned a different effective minting level, as long as they have no penalty
if (Account.isFounder(accountData.getFlags()) && accountData.getBlocksMintedPenalty() == 0)
return BlockChain.getInstance().getFounderEffectiveMintingLevel();
return accountData.getLevel();
}
/**
* Returns 'effective' minting level, or zero if reward-share does not exist.
* <p>
* For founder accounts, this returns "founderEffectiveMintingLevel" from blockchain config.
* Get Primary Name
*
* @return the primary name for this address if present, otherwise empty
*
* @throws DataException
*/
public Optional<String> getPrimaryName() throws DataException {
return this.repository.getNameRepository().getPrimaryName(this.address);
}
/**
* Remove Primary Name
*
* @throws DataException
*/
public void removePrimaryName() throws DataException {
this.repository.getNameRepository().removePrimaryName(this.address);
}
/**
* Reset Primary Name
*
* Set primary name based on the names (and their history) this account owns.
*
* @param confirmationStatus the status of the transactions for the determining the primary name
*
* @return the primary name, empty if their isn't one
*
* @throws DataException
*/
public Optional<String> resetPrimaryName(TransactionsResource.ConfirmationStatus confirmationStatus) throws DataException {
Optional<String> primaryName = determinePrimaryName(confirmationStatus);
if(primaryName.isPresent()) {
return setPrimaryName(primaryName.get());
}
else {
return primaryName;
}
}
/**
* Determine Primary Name
*
* Determine primary name based on a list of registered names.
*
* @param confirmationStatus the status of the transactions for this determination
*
* @return the primary name, empty if there is no primary name
*
* @throws DataException
*/
public Optional<String> determinePrimaryName(TransactionsResource.ConfirmationStatus confirmationStatus) throws DataException {
// all registered names for the owner
List<NameData> names = this.repository.getNameRepository().getNamesByOwner(this.address);
Optional<String> primaryName;
// if no registered names, the no primary name possible
if (names.isEmpty()) {
primaryName = Optional.empty();
}
// if names
else {
// if one name, then that is the primary name
if (names.size() == 1) {
primaryName = Optional.of( names.get(0).getName() );
}
// if more than one name, then seek the earliest name acquisition that was never released
else {
Map<String, TransactionData> txByName = new HashMap<>(names.size());
// for each name, get the latest transaction
for (NameData nameData : names) {
// since the name is currently registered to the owner,
// we assume the latest transaction involving this name was the transaction that the acquired
// name through registration, purchase or update
Optional<TransactionData> latestTransaction
= this.repository
.getTransactionRepository()
.getTransactionsInvolvingName(
nameData.getName(),
confirmationStatus
)
.stream()
.sorted(Comparator.comparing(
TransactionData::getTimestamp).reversed()
)
.findFirst(); // first is the last, since it was reversed
// if there is a latest transaction, expected for all registered names
if (latestTransaction.isPresent()) {
txByName.put(nameData.getName(), latestTransaction.get());
}
// if there is no latest transaction, then
else {
LOGGER.warn("No matching transaction for name: " + nameData.getName());
}
}
// get the first name aqcuistion for this address
Optional<Map.Entry<String, TransactionData>> firstNameEntry
= txByName.entrySet().stream().sorted(Comparator.comparing(entry -> entry.getValue().getTimestamp())).findFirst();
// if their is a name acquisition, then the first one is the primary name
if (firstNameEntry.isPresent()) {
primaryName = Optional.of( firstNameEntry.get().getKey() );
}
// if there is no nameacquistion, then there is no primary name
else {
primaryName = Optional.empty();
}
}
}
return primaryName;
}
/**
* Set Primary Name
*
* @param primaryName the primary to set to this address
*
* @return the primary name if successful, empty if unsuccessful
*
* @throws DataException
*/
public Optional<String> setPrimaryName( String primaryName ) throws DataException {
int changed = this.repository.getNameRepository().setPrimaryName(this.address, primaryName);
return changed > 0 ? Optional.of(primaryName) : Optional.empty();
}
/**
* Returns reward-share minting address, or unknown if reward-share does not exist.
*
* @param repository
* @param rewardSharePublicKey
* @return address or unknown
* @throws DataException
*/
public static String getRewardShareMintingAddress(Repository repository, byte[] rewardSharePublicKey) throws DataException {
// Find actual minter address
RewardShareData rewardShareData = repository.getAccountRepository().getRewardShare(rewardSharePublicKey);
if (rewardShareData == null)
return "Unknown";
return rewardShareData.getMinter();
}
/**
* Returns 'effective' minting level, or zero if reward-share does not exist.
*
* @param repository
* @param rewardSharePublicKey
* @return 0+
* @throws DataException
*/
@@ -289,4 +539,26 @@ public class Account {
return rewardShareMinter.getEffectiveMintingLevel();
}
/**
* Returns 'effective' minting level, with a fix for the zero level.
* <p>
* For founder accounts with no penalty, this returns "founderEffectiveMintingLevel" from blockchain config.
*
* @param repository
* @param rewardSharePublicKey
* @return 0+
* @throws DataException
*/
public static int getRewardShareEffectiveMintingLevelIncludingLevelZero(Repository repository, byte[] rewardSharePublicKey) throws DataException {
// Find actual minter and get their effective minting level
RewardShareData rewardShareData = repository.getAccountRepository().getRewardShare(rewardSharePublicKey);
if (rewardShareData == null)
return 0;
else if (!rewardShareData.getMinter().equals(rewardShareData.getRecipient())) // Sponsorship reward share
return 0;
Account rewardShareMinter = new Account(repository, rewardShareData.getMinter());
return rewardShareMinter.getEffectiveMintingLevel();
}
}

View File

@@ -1,15 +1,15 @@
package org.qortal.account;
import java.util.HashMap;
import java.util.Map;
import java.util.Map.Entry;
import java.util.function.BinaryOperator;
import org.qortal.data.account.AccountData;
import org.qortal.repository.DataException;
import org.qortal.repository.Repository;
import org.qortal.utils.Pair;
import java.util.HashMap;
import java.util.Map;
import java.util.Map.Entry;
import java.util.function.BinaryOperator;
/**
* Account lastReference caching
* <p>

View File

@@ -11,15 +11,15 @@ public class PrivateKeyAccount extends PublicKeyAccount {
private final Ed25519PrivateKeyParameters edPrivateKeyParams;
/**
* Create PrivateKeyAccount using byte[32] seed.
* Create PrivateKeyAccount using byte[32] private key.
*
* @param seed
* @param privateKey
* byte[32] used to create private/public key pair
* @throws IllegalArgumentException
* if passed invalid seed
* if passed invalid privateKey
*/
public PrivateKeyAccount(Repository repository, byte[] seed) {
this(repository, new Ed25519PrivateKeyParameters(seed, 0));
public PrivateKeyAccount(Repository repository, byte[] privateKey) {
this(repository, new Ed25519PrivateKeyParameters(privateKey, 0));
}
private PrivateKeyAccount(Repository repository, Ed25519PrivateKeyParameters edPrivateKeyParams) {
@@ -37,10 +37,6 @@ public class PrivateKeyAccount extends PublicKeyAccount {
return this.privateKey;
}
public static byte[] toPublicKey(byte[] seed) {
return new Ed25519PrivateKeyParameters(seed, 0).generatePublicKey().getEncoded();
}
public byte[] sign(byte[] message) {
return Crypto.sign(this.edPrivateKeyParams, message);
}

View File

@@ -1,17 +1,41 @@
package org.qortal.account;
import org.bouncycastle.crypto.generators.Ed25519KeyPairGenerator;
import org.bouncycastle.crypto.params.Ed25519KeyGenerationParameters;
import org.bouncycastle.crypto.params.Ed25519PublicKeyParameters;
import org.qortal.crypto.Crypto;
import org.qortal.data.account.AccountData;
import org.qortal.repository.Repository;
import java.security.SecureRandom;
public class PublicKeyAccount extends Account {
protected final byte[] publicKey;
protected final Ed25519PublicKeyParameters edPublicKeyParams;
/** <p>Constructor for generating a PublicKeyAccount</p>
*
* @param repository Block Chain
* @param publicKey 32 byte Public Key
* @since v4.7.3
*/
public PublicKeyAccount(Repository repository, byte[] publicKey) {
this(repository, new Ed25519PublicKeyParameters(publicKey, 0));
super(repository, Crypto.toAddress(publicKey));
Ed25519PublicKeyParameters t = null;
try {
t = new Ed25519PublicKeyParameters(publicKey, 0);
} catch (Exception e) {
var gen = new Ed25519KeyPairGenerator();
gen.init(new Ed25519KeyGenerationParameters(new SecureRandom()));
var keyPair = gen.generateKeyPair();
t = (Ed25519PublicKeyParameters) keyPair.getPublic();
} finally {
this.edPublicKeyParams = t;
}
this.publicKey = publicKey;
}
protected PublicKeyAccount(Repository repository, Ed25519PublicKeyParameters edPublicKeyParams) {

View File

@@ -0,0 +1,366 @@
package org.qortal.account;
import org.qortal.api.resource.TransactionsResource;
import org.qortal.asset.Asset;
import org.qortal.data.account.AccountData;
import org.qortal.data.naming.NameData;
import org.qortal.data.transaction.*;
import org.qortal.repository.DataException;
import org.qortal.repository.Repository;
import org.qortal.transaction.Transaction.TransactionType;
import java.util.*;
import java.util.stream.Collectors;
public class SelfSponsorshipAlgoV1 {
private final Repository repository;
private final String address;
private final AccountData accountData;
private final long snapshotTimestamp;
private final boolean override;
private int registeredNameCount = 0;
private int suspiciousCount = 0;
private int suspiciousPercent = 0;
private int consolidationCount = 0;
private int bulkIssuanceCount = 0;
private int recentSponsorshipCount = 0;
private List<RewardShareTransactionData> sponsorshipRewardShares = new ArrayList<>();
private final Map<String, List<TransactionData>> paymentsByAddress = new HashMap<>();
private final Set<String> sponsees = new LinkedHashSet<>();
private Set<String> consolidatedAddresses = new LinkedHashSet<>();
private final Set<String> zeroTransactionAddreses = new LinkedHashSet<>();
private final Set<String> penaltyAddresses = new LinkedHashSet<>();
public SelfSponsorshipAlgoV1(Repository repository, String address, long snapshotTimestamp, boolean override) throws DataException {
this.repository = repository;
this.address = address;
this.accountData = this.repository.getAccountRepository().getAccount(this.address);
this.snapshotTimestamp = snapshotTimestamp;
this.override = override;
}
public String getAddress() {
return this.address;
}
public Set<String> getPenaltyAddresses() {
return this.penaltyAddresses;
}
public void run() throws DataException {
if (this.accountData == null) {
// Nothing to do
return;
}
this.fetchSponsorshipRewardShares();
if (this.sponsorshipRewardShares.isEmpty()) {
// Nothing to do
return;
}
this.findConsolidatedRewards();
this.findBulkIssuance();
this.findRegisteredNameCount();
this.findRecentSponsorshipCount();
int score = this.calculateScore();
if (score <= 0 && !override) {
return;
}
String newAddress = this.getDestinationAccount(this.address);
while (newAddress != null) {
// Found destination account
this.penaltyAddresses.add(newAddress);
// Run algo for this address, but in "override" mode because it has already been flagged
SelfSponsorshipAlgoV1 algoV1 = new SelfSponsorshipAlgoV1(this.repository, newAddress, this.snapshotTimestamp, true);
algoV1.run();
this.penaltyAddresses.addAll(algoV1.getPenaltyAddresses());
newAddress = this.getDestinationAccount(newAddress);
}
this.penaltyAddresses.add(this.address);
if (this.override || this.recentSponsorshipCount < 20) {
this.penaltyAddresses.addAll(this.consolidatedAddresses);
this.penaltyAddresses.addAll(this.zeroTransactionAddreses);
}
else {
this.penaltyAddresses.addAll(this.sponsees);
}
}
private String getDestinationAccount(String address) throws DataException {
List<TransactionData> transferPrivsTransactions = fetchTransferPrivsForAddress(address);
if (transferPrivsTransactions.isEmpty()) {
// No TRANSFER_PRIVS transactions for this address
return null;
}
AccountData accountData = this.repository.getAccountRepository().getAccount(address);
if (accountData == null) {
return null;
}
for (TransactionData transactionData : transferPrivsTransactions) {
TransferPrivsTransactionData transferPrivsTransactionData = (TransferPrivsTransactionData) transactionData;
if (Arrays.equals(transferPrivsTransactionData.getSenderPublicKey(), accountData.getPublicKey())) {
return transferPrivsTransactionData.getRecipient();
}
}
return null;
}
private void findConsolidatedRewards() throws DataException {
List<String> sponseesThatSentRewards = new ArrayList<>();
Map<String, Integer> paymentRecipients = new HashMap<>();
// Collect outgoing payments of each sponsee
for (String sponseeAddress : this.sponsees) {
// Firstly fetch all payments for address, since the functions below depend on this data
this.fetchPaymentsForAddress(sponseeAddress);
// Check if the address has zero relevant transactions
if (this.hasZeroTransactions(sponseeAddress)) {
this.zeroTransactionAddreses.add(sponseeAddress);
}
// Get payment recipients
List<String> allPaymentRecipients = this.fetchOutgoingPaymentRecipientsForAddress(sponseeAddress);
if (allPaymentRecipients.isEmpty()) {
continue;
}
sponseesThatSentRewards.add(sponseeAddress);
List<String> addressesPaidByThisSponsee = new ArrayList<>();
for (String paymentRecipient : allPaymentRecipients) {
if (addressesPaidByThisSponsee.contains(paymentRecipient)) {
// We already tracked this association - don't allow multiple to stack up
continue;
}
addressesPaidByThisSponsee.add(paymentRecipient);
// Increment count for this recipient, or initialize to 1 if not present
if (paymentRecipients.computeIfPresent(paymentRecipient, (k, v) -> v + 1) == null) {
paymentRecipients.put(paymentRecipient, 1);
}
}
}
// Exclude addresses with a low number of payments
Map<String, Integer> filteredPaymentRecipients = paymentRecipients.entrySet().stream()
.filter(p -> p.getValue() != null && p.getValue() >= 10)
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
// Now check how many sponsees have sent to this subset of addresses
Map<String, Integer> sponseesThatConsolidatedRewards = new HashMap<>();
for (String sponseeAddress : sponseesThatSentRewards) {
List<String> allPaymentRecipients = this.fetchOutgoingPaymentRecipientsForAddress(sponseeAddress);
// Remove any that aren't to one of the flagged recipients (i.e. consolidation)
allPaymentRecipients.removeIf(r -> !filteredPaymentRecipients.containsKey(r));
int count = allPaymentRecipients.size();
if (count == 0) {
continue;
}
if (sponseesThatConsolidatedRewards.computeIfPresent(sponseeAddress, (k, v) -> v + count) == null) {
sponseesThatConsolidatedRewards.put(sponseeAddress, count);
}
}
// Remove sponsees that have only sent a low number of payments to the filtered addresses
Map<String, Integer> filteredSponseesThatConsolidatedRewards = sponseesThatConsolidatedRewards.entrySet().stream()
.filter(p -> p.getValue() != null && p.getValue() >= 2)
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
this.consolidationCount = sponseesThatConsolidatedRewards.size();
this.consolidatedAddresses = new LinkedHashSet<>(filteredSponseesThatConsolidatedRewards.keySet());
this.suspiciousCount = this.consolidationCount + this.zeroTransactionAddreses.size();
this.suspiciousPercent = (int)(this.suspiciousCount / (float) this.sponsees.size() * 100);
}
private void findBulkIssuance() {
Long lastTimestamp = null;
for (RewardShareTransactionData rewardShareTransactionData : sponsorshipRewardShares) {
long timestamp = rewardShareTransactionData.getTimestamp();
if (timestamp >= this.snapshotTimestamp) {
continue;
}
if (lastTimestamp != null) {
if (timestamp - lastTimestamp < 3*60*1000L) {
this.bulkIssuanceCount++;
}
}
lastTimestamp = timestamp;
}
}
private void findRegisteredNameCount() throws DataException {
int registeredNameCount = 0;
for (String sponseeAddress : sponsees) {
List<NameData> names = repository.getNameRepository().getNamesByOwner(sponseeAddress);
for (NameData name : names) {
if (name.getRegistered() < this.snapshotTimestamp) {
registeredNameCount++;
break;
}
}
}
this.registeredNameCount = registeredNameCount;
}
private void findRecentSponsorshipCount() {
final long referenceTimestamp = this.snapshotTimestamp - (365 * 24 * 60 * 60 * 1000L);
int recentSponsorshipCount = 0;
for (RewardShareTransactionData rewardShare : sponsorshipRewardShares) {
if (rewardShare.getTimestamp() >= referenceTimestamp) {
recentSponsorshipCount++;
}
}
this.recentSponsorshipCount = recentSponsorshipCount;
}
private int calculateScore() {
final int suspiciousMultiplier = (this.suspiciousCount >= 100) ? this.suspiciousPercent : 1;
final int nameMultiplier = (this.sponsees.size() >= 50 && this.registeredNameCount == 0) ? 2 : 1;
final int consolidationMultiplier = Math.max(this.consolidationCount, 1);
final int bulkIssuanceMultiplier = Math.max(this.bulkIssuanceCount / 2, 1);
final int offset = 9;
return suspiciousMultiplier * nameMultiplier * consolidationMultiplier * bulkIssuanceMultiplier - offset;
}
private void fetchSponsorshipRewardShares() throws DataException {
List<RewardShareTransactionData> sponsorshipRewardShares = new ArrayList<>();
// Define relevant transactions
List<TransactionType> txTypes = List.of(TransactionType.REWARD_SHARE);
List<TransactionData> transactionDataList = fetchTransactions(repository, txTypes, this.address, false);
for (TransactionData transactionData : transactionDataList) {
if (transactionData.getType() != TransactionType.REWARD_SHARE) {
continue;
}
RewardShareTransactionData rewardShareTransactionData = (RewardShareTransactionData) transactionData;
// Skip removals
if (rewardShareTransactionData.getSharePercent() < 0) {
continue;
}
// Skip if not sponsored by this account
if (!Arrays.equals(rewardShareTransactionData.getCreatorPublicKey(), accountData.getPublicKey())) {
continue;
}
// Skip self shares
if (Objects.equals(rewardShareTransactionData.getRecipient(), this.address)) {
continue;
}
boolean duplicateFound = false;
for (RewardShareTransactionData existingRewardShare : sponsorshipRewardShares) {
if (Objects.equals(existingRewardShare.getRecipient(), rewardShareTransactionData.getRecipient())) {
// Duplicate
duplicateFound = true;
break;
}
}
if (!duplicateFound) {
sponsorshipRewardShares.add(rewardShareTransactionData);
this.sponsees.add(rewardShareTransactionData.getRecipient());
}
}
this.sponsorshipRewardShares = sponsorshipRewardShares;
}
private List<TransactionData> fetchTransferPrivsForAddress(String address) throws DataException {
return fetchTransactions(repository,
List.of(TransactionType.TRANSFER_PRIVS),
address, true);
}
private void fetchPaymentsForAddress(String address) throws DataException {
List<TransactionData> payments = fetchTransactions(repository,
Arrays.asList(TransactionType.PAYMENT, TransactionType.TRANSFER_ASSET),
address, false);
this.paymentsByAddress.put(address, payments);
}
private List<String> fetchOutgoingPaymentRecipientsForAddress(String address) {
List<String> outgoingPaymentRecipients = new ArrayList<>();
List<TransactionData> transactionDataList = this.paymentsByAddress.get(address);
if (transactionDataList == null) transactionDataList = new ArrayList<>();
transactionDataList.removeIf(t -> t.getTimestamp() >= this.snapshotTimestamp);
for (TransactionData transactionData : transactionDataList) {
switch (transactionData.getType()) {
case PAYMENT:
PaymentTransactionData paymentTransactionData = (PaymentTransactionData) transactionData;
if (!Objects.equals(paymentTransactionData.getRecipient(), address)) {
// Outgoing payment from this account
outgoingPaymentRecipients.add(paymentTransactionData.getRecipient());
}
break;
case TRANSFER_ASSET:
TransferAssetTransactionData transferAssetTransactionData = (TransferAssetTransactionData) transactionData;
if (transferAssetTransactionData.getAssetId() == Asset.QORT) {
if (!Objects.equals(transferAssetTransactionData.getRecipient(), address)) {
// Outgoing payment from this account
outgoingPaymentRecipients.add(transferAssetTransactionData.getRecipient());
}
}
break;
default:
break;
}
}
return outgoingPaymentRecipients;
}
private boolean hasZeroTransactions(String address) {
List<TransactionData> transactionDataList = this.paymentsByAddress.get(address);
if (transactionDataList == null) {
return true;
}
transactionDataList.removeIf(t -> t.getTimestamp() >= this.snapshotTimestamp);
return transactionDataList.isEmpty();
}
private static List<TransactionData> fetchTransactions(Repository repository, List<TransactionType> txTypes, String address, boolean reverse) throws DataException {
// Fetch all relevant transactions for this account
List<byte[]> signatures = repository.getTransactionRepository()
.getSignaturesMatchingCriteria(null, null, null, txTypes,
null, null, address, TransactionsResource.ConfirmationStatus.CONFIRMED,
null, null, reverse);
List<TransactionData> transactionDataList = new ArrayList<>();
for (byte[] signature : signatures) {
// Fetch transaction data
TransactionData transactionData = repository.getTransactionRepository().fromSignature(signature);
if (transactionData == null) {
continue;
}
transactionDataList.add(transactionData);
}
return transactionDataList;
}
}

View File

@@ -0,0 +1,249 @@
package org.qortal.account;
import org.qortal.api.resource.TransactionsResource;
import org.qortal.asset.Asset;
import org.qortal.block.BlockChain;
import org.qortal.data.account.AccountData;
import org.qortal.data.transaction.*;
import org.qortal.repository.DataException;
import org.qortal.repository.Repository;
import org.qortal.transaction.Transaction.TransactionType;
import java.util.*;
public class SelfSponsorshipAlgoV2 {
private final long snapshotTimestampV1 = BlockChain.getInstance().getSelfSponsorshipAlgoV1SnapshotTimestamp();
private final long snapshotTimestampV2 = BlockChain.getInstance().getSelfSponsorshipAlgoV2SnapshotTimestamp();
private final long referenceTimestamp = BlockChain.getInstance().getReferenceTimestampBlock();
private final boolean override;
private final Repository repository;
private final String address;
private int recentAssetSendCount = 0;
private int recentSponsorshipCount = 0;
private final Set<String> assetAddresses = new LinkedHashSet<>();
private final Set<String> penaltyAddresses = new LinkedHashSet<>();
private final Set<String> sponsorAddresses = new LinkedHashSet<>();
private List<RewardShareTransactionData> sponsorshipRewardShares = new ArrayList<>();
private List<TransferAssetTransactionData> transferAssetForAddress = new ArrayList<>();
public SelfSponsorshipAlgoV2(Repository repository, String address, boolean override) {
this.repository = repository;
this.address = address;
this.override = override;
}
public String getAddress() {
return this.address;
}
public Set<String> getPenaltyAddresses() {
return this.penaltyAddresses;
}
public void run() throws DataException {
if (!override) {
this.getAccountPrivs(this.address);
}
if (override) {
this.fetchTransferAssetForAddress(this.address);
this.findRecentAssetSendCount();
if (this.recentAssetSendCount >= 6) {
this.penaltyAddresses.add(this.address);
this.penaltyAddresses.addAll(this.assetAddresses);
}
}
}
private void getAccountPrivs(String address) throws DataException {
AccountData accountData = this.repository.getAccountRepository().getAccount(address);
List<TransactionData> transferPrivsTransactions = fetchTransferPrivsForAddress(address);
transferPrivsTransactions.removeIf(t -> t.getTimestamp() > this.referenceTimestamp || accountData.getAddress().equals(t.getRecipient()));
if (transferPrivsTransactions.isEmpty()) {
// Nothing to do
return;
}
for (TransactionData transactionData : transferPrivsTransactions) {
TransferPrivsTransactionData transferPrivsTransactionData = (TransferPrivsTransactionData) transactionData;
this.penaltyAddresses.add(transferPrivsTransactionData.getRecipient());
this.fetchSponsorshipRewardShares(transferPrivsTransactionData.getRecipient());
this.findRecentSponsorshipCount();
if (this.recentSponsorshipCount >= 1) {
this.penaltyAddresses.addAll(this.sponsorAddresses);
}
String newAddress = this.getDestinationAccount(transferPrivsTransactionData.getRecipient());
while (newAddress != null) {
// Found destination account
this.penaltyAddresses.add(newAddress);
this.fetchSponsorshipRewardShares(newAddress);
this.findRecentSponsorshipCount();
if (this.recentSponsorshipCount >= 1) {
this.penaltyAddresses.addAll(this.sponsorAddresses);
}
newAddress = this.getDestinationAccount(newAddress);
}
}
}
private String getDestinationAccount(String address) throws DataException {
AccountData accountData = this.repository.getAccountRepository().getAccount(address);
List<TransactionData> transferPrivsTransactions = fetchTransferPrivsForAddress(address);
transferPrivsTransactions.removeIf(t -> t.getTimestamp() > this.referenceTimestamp || accountData.getAddress().equals(t.getRecipient()));
if (transferPrivsTransactions.isEmpty()) {
return null;
}
if (accountData == null) {
return null;
}
for (TransactionData transactionData : transferPrivsTransactions) {
TransferPrivsTransactionData transferPrivsTransactionData = (TransferPrivsTransactionData) transactionData;
if (Arrays.equals(transferPrivsTransactionData.getSenderPublicKey(), accountData.getPublicKey())) {
return transferPrivsTransactionData.getRecipient();
}
}
return null;
}
private void fetchSponsorshipRewardShares(String address) throws DataException {
AccountData accountDataRs = this.repository.getAccountRepository().getAccount(address);
List<RewardShareTransactionData> sponsorshipRewardShares = new ArrayList<>();
// Define relevant transactions
List<TransactionType> txTypes = List.of(TransactionType.REWARD_SHARE);
List<TransactionData> transactionDataList = fetchTransactions(repository, txTypes, address, false);
for (TransactionData transactionData : transactionDataList) {
if (transactionData.getType() != TransactionType.REWARD_SHARE) {
continue;
}
RewardShareTransactionData rewardShareTransactionData = (RewardShareTransactionData) transactionData;
// Skip removals
if (rewardShareTransactionData.getSharePercent() < 0) {
continue;
}
// Skip if not sponsored by this account
if (!Arrays.equals(rewardShareTransactionData.getCreatorPublicKey(), accountDataRs.getPublicKey())) {
continue;
}
// Skip self shares
if (Objects.equals(rewardShareTransactionData.getRecipient(), address)) {
continue;
}
boolean duplicateFound = false;
for (RewardShareTransactionData existingRewardShare : sponsorshipRewardShares) {
if (Objects.equals(existingRewardShare.getRecipient(), rewardShareTransactionData.getRecipient())) {
// Duplicate
duplicateFound = true;
break;
}
}
if (!duplicateFound) {
sponsorshipRewardShares.add(rewardShareTransactionData);
this.sponsorAddresses.add(rewardShareTransactionData.getRecipient());
}
}
this.sponsorshipRewardShares = sponsorshipRewardShares;
}
private void fetchTransferAssetForAddress(String address) throws DataException {
List<TransferAssetTransactionData> transferAssetForAddress = new ArrayList<>();
// Define relevant transactions
List<TransactionType> txTypes = List.of(TransactionType.TRANSFER_ASSET);
List<TransactionData> transactionDataList = fetchTransactions(repository, txTypes, address, false);
transactionDataList.removeIf(t -> t.getTimestamp() <= this.snapshotTimestampV1 || t.getTimestamp() >= this.snapshotTimestampV2);
for (TransactionData transactionData : transactionDataList) {
if (transactionData.getType() != TransactionType.TRANSFER_ASSET) {
continue;
}
TransferAssetTransactionData transferAssetTransactionData = (TransferAssetTransactionData) transactionData;
if (transferAssetTransactionData.getAssetId() == Asset.QORT) {
if (!Objects.equals(transferAssetTransactionData.getRecipient(), address)) {
// Outgoing transfer asset for this account
transferAssetForAddress.add(transferAssetTransactionData);
this.assetAddresses.add(transferAssetTransactionData.getRecipient());
}
}
}
this.transferAssetForAddress = transferAssetForAddress;
}
private void findRecentSponsorshipCount() {
int recentSponsorshipCount = 0;
for (RewardShareTransactionData rewardShare : sponsorshipRewardShares) {
if (rewardShare.getTimestamp() >= this.snapshotTimestampV1) {
recentSponsorshipCount++;
}
}
this.recentSponsorshipCount = recentSponsorshipCount;
}
private void findRecentAssetSendCount() {
int recentAssetSendCount = 0;
for (TransferAssetTransactionData assetSend : transferAssetForAddress) {
if (assetSend.getTimestamp() >= this.snapshotTimestampV1) {
recentAssetSendCount++;
}
}
this.recentAssetSendCount = recentAssetSendCount;
}
private List<TransactionData> fetchTransferPrivsForAddress(String address) throws DataException {
return fetchTransactions(repository,
List.of(TransactionType.TRANSFER_PRIVS),
address, true);
}
private static List<TransactionData> fetchTransactions(Repository repository, List<TransactionType> txTypes, String address, boolean reverse) throws DataException {
// Fetch all relevant transactions for this account
List<byte[]> signatures = repository.getTransactionRepository()
.getSignaturesMatchingCriteria(null, null, null, txTypes,
null, null, address, TransactionsResource.ConfirmationStatus.CONFIRMED,
null, null, reverse);
List<TransactionData> transactionDataList = new ArrayList<>();
for (byte[] signature : signatures) {
// Fetch transaction data
TransactionData transactionData = repository.getTransactionRepository().fromSignature(signature);
if (transactionData == null) {
continue;
}
transactionDataList.add(transactionData);
}
return transactionDataList;
}
}

View File

@@ -0,0 +1,370 @@
package org.qortal.account;
import org.qortal.api.resource.TransactionsResource;
import org.qortal.asset.Asset;
import org.qortal.data.account.AccountData;
import org.qortal.data.naming.NameData;
import org.qortal.data.transaction.*;
import org.qortal.repository.DataException;
import org.qortal.repository.Repository;
import org.qortal.transaction.Transaction.TransactionType;
import java.util.*;
import java.util.stream.Collectors;
public class SelfSponsorshipAlgoV3 {
private final Repository repository;
private final String address;
private final AccountData accountData;
private final long snapshotTimestampV1;
private final long snapshotTimestampV3;
private final boolean override;
private int registeredNameCount = 0;
private int suspiciousCount = 0;
private int suspiciousPercent = 0;
private int consolidationCount = 0;
private int bulkIssuanceCount = 0;
private int recentSponsorshipCount = 0;
private List<RewardShareTransactionData> sponsorshipRewardShares = new ArrayList<>();
private final Map<String, List<TransactionData>> paymentsByAddress = new HashMap<>();
private final Set<String> sponsees = new LinkedHashSet<>();
private Set<String> consolidatedAddresses = new LinkedHashSet<>();
private final Set<String> zeroTransactionAddreses = new LinkedHashSet<>();
private final Set<String> penaltyAddresses = new LinkedHashSet<>();
public SelfSponsorshipAlgoV3(Repository repository, String address, long snapshotTimestampV1, long snapshotTimestampV3, boolean override) throws DataException {
this.repository = repository;
this.address = address;
this.accountData = this.repository.getAccountRepository().getAccount(this.address);
this.snapshotTimestampV1 = snapshotTimestampV1;
this.snapshotTimestampV3 = snapshotTimestampV3;
this.override = override;
}
public String getAddress() {
return this.address;
}
public Set<String> getPenaltyAddresses() {
return this.penaltyAddresses;
}
public void run() throws DataException {
if (this.accountData == null) {
// Nothing to do
return;
}
this.fetchSponsorshipRewardShares();
if (this.sponsorshipRewardShares.isEmpty()) {
// Nothing to do
return;
}
this.findConsolidatedRewards();
this.findBulkIssuance();
this.findRegisteredNameCount();
this.findRecentSponsorshipCount();
int score = this.calculateScore();
if (score <= 0 && !override) {
return;
}
String newAddress = this.getDestinationAccount(this.address);
while (newAddress != null) {
// Found destination account
this.penaltyAddresses.add(newAddress);
// Run algo for this address, but in "override" mode because it has already been flagged
SelfSponsorshipAlgoV3 algoV3 = new SelfSponsorshipAlgoV3(this.repository, newAddress, this.snapshotTimestampV1, this.snapshotTimestampV3, true);
algoV3.run();
this.penaltyAddresses.addAll(algoV3.getPenaltyAddresses());
newAddress = this.getDestinationAccount(newAddress);
}
this.penaltyAddresses.add(this.address);
if (this.override || this.recentSponsorshipCount < 20) {
this.penaltyAddresses.addAll(this.consolidatedAddresses);
this.penaltyAddresses.addAll(this.zeroTransactionAddreses);
}
else {
this.penaltyAddresses.addAll(this.sponsees);
}
}
private String getDestinationAccount(String address) throws DataException {
List<TransactionData> transferPrivsTransactions = fetchTransferPrivsForAddress(address);
if (transferPrivsTransactions.isEmpty()) {
// No TRANSFER_PRIVS transactions for this address
return null;
}
AccountData accountData = this.repository.getAccountRepository().getAccount(address);
if (accountData == null) {
return null;
}
for (TransactionData transactionData : transferPrivsTransactions) {
TransferPrivsTransactionData transferPrivsTransactionData = (TransferPrivsTransactionData) transactionData;
if (Arrays.equals(transferPrivsTransactionData.getSenderPublicKey(), accountData.getPublicKey())) {
return transferPrivsTransactionData.getRecipient();
}
}
return null;
}
private void findConsolidatedRewards() throws DataException {
List<String> sponseesThatSentRewards = new ArrayList<>();
Map<String, Integer> paymentRecipients = new HashMap<>();
// Collect outgoing payments of each sponsee
for (String sponseeAddress : this.sponsees) {
// Firstly fetch all payments for address, since the functions below depend on this data
this.fetchPaymentsForAddress(sponseeAddress);
// Check if the address has zero relevant transactions
if (this.hasZeroTransactions(sponseeAddress)) {
this.zeroTransactionAddreses.add(sponseeAddress);
}
// Get payment recipients
List<String> allPaymentRecipients = this.fetchOutgoingPaymentRecipientsForAddress(sponseeAddress);
if (allPaymentRecipients.isEmpty()) {
continue;
}
sponseesThatSentRewards.add(sponseeAddress);
List<String> addressesPaidByThisSponsee = new ArrayList<>();
for (String paymentRecipient : allPaymentRecipients) {
if (addressesPaidByThisSponsee.contains(paymentRecipient)) {
// We already tracked this association - don't allow multiple to stack up
continue;
}
addressesPaidByThisSponsee.add(paymentRecipient);
// Increment count for this recipient, or initialize to 1 if not present
if (paymentRecipients.computeIfPresent(paymentRecipient, (k, v) -> v + 1) == null) {
paymentRecipients.put(paymentRecipient, 1);
}
}
}
// Exclude addresses with a low number of payments
Map<String, Integer> filteredPaymentRecipients = paymentRecipients.entrySet().stream()
.filter(p -> p.getValue() != null && p.getValue() >= 10)
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
// Now check how many sponsees have sent to this subset of addresses
Map<String, Integer> sponseesThatConsolidatedRewards = new HashMap<>();
for (String sponseeAddress : sponseesThatSentRewards) {
List<String> allPaymentRecipients = this.fetchOutgoingPaymentRecipientsForAddress(sponseeAddress);
// Remove any that aren't to one of the flagged recipients (i.e. consolidation)
allPaymentRecipients.removeIf(r -> !filteredPaymentRecipients.containsKey(r));
int count = allPaymentRecipients.size();
if (count == 0) {
continue;
}
if (sponseesThatConsolidatedRewards.computeIfPresent(sponseeAddress, (k, v) -> v + count) == null) {
sponseesThatConsolidatedRewards.put(sponseeAddress, count);
}
}
// Remove sponsees that have only sent a low number of payments to the filtered addresses
Map<String, Integer> filteredSponseesThatConsolidatedRewards = sponseesThatConsolidatedRewards.entrySet().stream()
.filter(p -> p.getValue() != null && p.getValue() >= 2)
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
this.consolidationCount = sponseesThatConsolidatedRewards.size();
this.consolidatedAddresses = new LinkedHashSet<>(filteredSponseesThatConsolidatedRewards.keySet());
this.suspiciousCount = this.consolidationCount + this.zeroTransactionAddreses.size();
this.suspiciousPercent = (int)(this.suspiciousCount / (float) this.sponsees.size() * 100);
}
private void findBulkIssuance() {
Long lastTimestamp = null;
for (RewardShareTransactionData rewardShareTransactionData : sponsorshipRewardShares) {
long timestamp = rewardShareTransactionData.getTimestamp();
if (timestamp >= this.snapshotTimestampV3) {
continue;
}
if (lastTimestamp != null) {
if (timestamp - lastTimestamp < 3*60*1000L) {
this.bulkIssuanceCount++;
}
}
lastTimestamp = timestamp;
}
}
private void findRegisteredNameCount() throws DataException {
int registeredNameCount = 0;
for (String sponseeAddress : sponsees) {
List<NameData> names = repository.getNameRepository().getNamesByOwner(sponseeAddress);
for (NameData name : names) {
if (name.getRegistered() < this.snapshotTimestampV3) {
registeredNameCount++;
break;
}
}
}
this.registeredNameCount = registeredNameCount;
}
private void findRecentSponsorshipCount() {
int recentSponsorshipCount = 0;
for (RewardShareTransactionData rewardShare : sponsorshipRewardShares) {
if (rewardShare.getTimestamp() >= this.snapshotTimestampV1) {
recentSponsorshipCount++;
}
}
this.recentSponsorshipCount = recentSponsorshipCount;
}
private int calculateScore() {
final int suspiciousMultiplier = (this.suspiciousCount >= 100) ? this.suspiciousPercent : 1;
final int nameMultiplier = (this.sponsees.size() >= 25 && this.registeredNameCount <= 1) ? 21 :
(this.sponsees.size() >= 15 && this.registeredNameCount <= 1) ? 11 :
(this.sponsees.size() >= 5 && this.registeredNameCount <= 1) ? 5 : 1;
final int consolidationMultiplier = Math.max(this.consolidationCount, 1);
final int bulkIssuanceMultiplier = Math.max(this.bulkIssuanceCount / 2, 1);
final int offset = 20;
return suspiciousMultiplier * nameMultiplier * consolidationMultiplier * bulkIssuanceMultiplier - offset;
}
private void fetchSponsorshipRewardShares() throws DataException {
List<RewardShareTransactionData> sponsorshipRewardShares = new ArrayList<>();
// Define relevant transactions
List<TransactionType> txTypes = List.of(TransactionType.REWARD_SHARE);
List<TransactionData> transactionDataList = fetchTransactions(repository, txTypes, this.address, false);
transactionDataList.removeIf(t -> t.getTimestamp() <= this.snapshotTimestampV1 || t.getTimestamp() >= this.snapshotTimestampV3);
for (TransactionData transactionData : transactionDataList) {
if (transactionData.getType() != TransactionType.REWARD_SHARE) {
continue;
}
RewardShareTransactionData rewardShareTransactionData = (RewardShareTransactionData) transactionData;
// Skip removals
if (rewardShareTransactionData.getSharePercent() < 0) {
continue;
}
// Skip if not sponsored by this account
if (!Arrays.equals(rewardShareTransactionData.getCreatorPublicKey(), accountData.getPublicKey())) {
continue;
}
// Skip self shares
if (Objects.equals(rewardShareTransactionData.getRecipient(), this.address)) {
continue;
}
boolean duplicateFound = false;
for (RewardShareTransactionData existingRewardShare : sponsorshipRewardShares) {
if (Objects.equals(existingRewardShare.getRecipient(), rewardShareTransactionData.getRecipient())) {
// Duplicate
duplicateFound = true;
break;
}
}
if (!duplicateFound) {
sponsorshipRewardShares.add(rewardShareTransactionData);
this.sponsees.add(rewardShareTransactionData.getRecipient());
}
}
this.sponsorshipRewardShares = sponsorshipRewardShares;
}
private List<TransactionData> fetchTransferPrivsForAddress(String address) throws DataException {
return fetchTransactions(repository,
List.of(TransactionType.TRANSFER_PRIVS),
address, true);
}
private void fetchPaymentsForAddress(String address) throws DataException {
List<TransactionData> payments = fetchTransactions(repository,
Arrays.asList(TransactionType.PAYMENT, TransactionType.TRANSFER_ASSET),
address, false);
this.paymentsByAddress.put(address, payments);
}
private List<String> fetchOutgoingPaymentRecipientsForAddress(String address) {
List<String> outgoingPaymentRecipients = new ArrayList<>();
List<TransactionData> transactionDataList = this.paymentsByAddress.get(address);
if (transactionDataList == null) transactionDataList = new ArrayList<>();
transactionDataList.removeIf(t -> t.getTimestamp() <= this.snapshotTimestampV1 || t.getTimestamp() >= this.snapshotTimestampV3);
for (TransactionData transactionData : transactionDataList) {
switch (transactionData.getType()) {
case PAYMENT:
PaymentTransactionData paymentTransactionData = (PaymentTransactionData) transactionData;
if (!Objects.equals(paymentTransactionData.getRecipient(), address)) {
// Outgoing payment from this account
outgoingPaymentRecipients.add(paymentTransactionData.getRecipient());
}
break;
case TRANSFER_ASSET:
TransferAssetTransactionData transferAssetTransactionData = (TransferAssetTransactionData) transactionData;
if (transferAssetTransactionData.getAssetId() == Asset.QORT) {
if (!Objects.equals(transferAssetTransactionData.getRecipient(), address)) {
// Outgoing payment from this account
outgoingPaymentRecipients.add(transferAssetTransactionData.getRecipient());
}
}
break;
default:
break;
}
}
return outgoingPaymentRecipients;
}
private boolean hasZeroTransactions(String address) {
List<TransactionData> transactionDataList = this.paymentsByAddress.get(address);
if (transactionDataList == null) {
return true;
}
transactionDataList.removeIf(t -> t.getTimestamp() <= this.snapshotTimestampV1 || t.getTimestamp() >= this.snapshotTimestampV3);
return transactionDataList.isEmpty();
}
private static List<TransactionData> fetchTransactions(Repository repository, List<TransactionType> txTypes, String address, boolean reverse) throws DataException {
// Fetch all relevant transactions for this account
List<byte[]> signatures = repository.getTransactionRepository()
.getSignaturesMatchingCriteria(null, null, null, txTypes,
null, null, address, TransactionsResource.ConfirmationStatus.CONFIRMED,
null, null, reverse);
List<TransactionData> transactionDataList = new ArrayList<>();
for (byte[] signature : signatures) {
// Fetch transaction data
TransactionData transactionData = repository.getTransactionRepository().fromSignature(signature);
if (transactionData == null) {
continue;
}
transactionDataList.add(transactionData);
}
return transactionDataList;
}
}

View File

@@ -1,10 +1,9 @@
package org.qortal.api;
import java.math.BigDecimal;
import org.qortal.utils.Amounts;
import javax.xml.bind.annotation.adapters.XmlAdapter;
import org.qortal.utils.Amounts;
import java.math.BigDecimal;
public class AmountTypeAdapter extends XmlAdapter<String, Long> {

View File

@@ -1,13 +1,12 @@
package org.qortal.api;
import static java.util.Arrays.stream;
import static java.util.stream.Collectors.toMap;
import java.util.Map;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlRootElement;
import java.util.Map;
import static java.util.Arrays.stream;
import static java.util.stream.Collectors.toMap;
@XmlAccessorType(XmlAccessType.NONE)
@XmlRootElement
@@ -15,7 +14,7 @@ public enum ApiError {
// COMMON
// UNKNOWN(0, 500),
JSON(1, 400),
// NO_BALANCE(2, 422),
INSUFFICIENT_BALANCE(2, 402),
// NOT_YET_RELEASED(3, 422),
UNAUTHORIZED(4, 403),
REPOSITORY_ISSUE(5, 500),
@@ -79,7 +78,7 @@ public enum ApiError {
// BUYER_ALREADY_OWNER(411, 422),
// POLLS
// POLL_NO_EXISTS(501, 404),
POLL_NO_EXISTS(501, 404),
// POLL_ALREADY_EXISTS(502, 422),
// DUPLICATE_OPTION(503, 422),
// POLL_OPTION_NO_EXISTS(504, 404),
@@ -126,10 +125,17 @@ public enum ApiError {
// Groups
GROUP_UNKNOWN(1101, 404),
// Bitcoin
BTC_NETWORK_ISSUE(1201, 500),
BTC_BALANCE_ISSUE(1202, 402),
BTC_TOO_SOON(1203, 408);
// Foreign blockchain
FOREIGN_BLOCKCHAIN_NETWORK_ISSUE(1201, 500),
FOREIGN_BLOCKCHAIN_BALANCE_ISSUE(1202, 402),
FOREIGN_BLOCKCHAIN_TOO_SOON(1203, 408),
// Trade portal
ORDER_SIZE_TOO_SMALL(1300, 402),
// Data
FILE_NOT_FOUND(1401, 404),
NO_REPLY(1402, 404);
private static final Map<Integer, ApiError> map = stream(ApiError.values()).collect(toMap(apiError -> apiError.code, apiError -> apiError));
@@ -157,4 +163,4 @@ public enum ApiError {
return this.status;
}
}
}

View File

@@ -1,18 +1,17 @@
package org.qortal.api;
import java.io.IOException;
import javax.servlet.RequestDispatcher;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.eclipse.jetty.server.Request;
import org.eclipse.jetty.server.handler.ErrorHandler;
import org.qortal.settings.Settings;
import javax.servlet.RequestDispatcher;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.io.IOException;
public class ApiErrorHandler extends ErrorHandler {
private static final Logger LOGGER = LogManager.getLogger(ApiErrorHandler.class);

View File

@@ -1,9 +1,9 @@
package org.qortal.api;
import javax.servlet.http.HttpServletRequest;
import org.qortal.globalization.Translator;
import javax.servlet.http.HttpServletRequest;
public enum ApiExceptionFactory {
INSTANCE;
@@ -16,4 +16,8 @@ public enum ApiExceptionFactory {
return createException(request, apiError, null);
}
public ApiException createCustomException(HttpServletRequest request, ApiError apiError, String message) {
return new ApiException(apiError.getStatus(), apiError.getCode(), message, null);
}
}

View File

@@ -0,0 +1,107 @@
package org.qortal.api;
import org.qortal.settings.Settings;
import org.qortal.utils.Base58;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.security.SecureRandom;
public class ApiKey {
private String apiKey;
public ApiKey() throws IOException {
this.load();
}
public void generate() throws IOException {
byte[] apiKey = new byte[16];
new SecureRandom().nextBytes(apiKey);
this.apiKey = Base58.encode(apiKey);
this.save();
}
/* Filesystem */
private Path getFilePath() {
return Paths.get(Settings.getInstance().getApiKeyPath(), "apikey.txt");
}
private boolean load() throws IOException {
Path path = this.getFilePath();
File apiKeyFile = new File(path.toString());
if (!apiKeyFile.exists()) {
// Try settings - to allow legacy API keys to be supported
return this.loadLegacyApiKey();
}
try {
this.apiKey = new String(Files.readAllBytes(path));
} catch (IOException e) {
throw new IOException(String.format("Couldn't read contents from file %s", path.toString()));
}
return true;
}
private boolean loadLegacyApiKey() {
String legacyApiKey = Settings.getInstance().getApiKey();
if (legacyApiKey != null && !legacyApiKey.isEmpty()) {
this.apiKey = Settings.getInstance().getApiKey();
try {
// Save it to the apikey file
this.save();
} catch (IOException e) {
// Ignore failures as it will be reloaded from settings next time
}
return true;
}
return false;
}
public void save() throws IOException {
if (this.apiKey == null || this.apiKey.isEmpty()) {
throw new IllegalStateException("Unable to save a blank API key");
}
Path filePath = this.getFilePath();
BufferedWriter writer = new BufferedWriter(new FileWriter(filePath.toString()));
writer.write(this.apiKey);
writer.close();
}
public void delete() throws IOException {
this.apiKey = null;
Path filePath = this.getFilePath();
if (Files.exists(filePath)) {
Files.delete(filePath);
}
}
public boolean generated() {
return (this.apiKey != null);
}
public boolean exists() {
return this.getFilePath().toFile().exists();
}
@Override
public String toString() {
return this.apiKey;
}
}

View File

@@ -1,35 +1,24 @@
package org.qortal.api;
import org.eclipse.persistence.exceptions.XMLMarshalException;
import org.eclipse.persistence.jaxb.JAXBContextFactory;
import org.eclipse.persistence.jaxb.MarshallerProperties;
import org.eclipse.persistence.jaxb.UnmarshallerProperties;
import javax.net.ssl.*;
import javax.xml.bind.*;
import javax.xml.transform.stream.StreamSource;
import java.io.IOException;
import java.io.InputStream;
import java.io.UnsupportedEncodingException;
import java.net.HttpURLConnection;
import java.net.InetSocketAddress;
import java.net.Socket;
import java.net.SocketAddress;
import java.net.URL;
import java.net.URLEncoder;
import java.io.Writer;
import java.net.*;
import java.util.Collections;
import java.util.Map;
import java.util.Scanner;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import javax.net.ssl.HttpsURLConnection;
import javax.net.ssl.SNIHostName;
import javax.net.ssl.SNIServerName;
import javax.net.ssl.SSLParameters;
import javax.net.ssl.SSLSocket;
import javax.xml.bind.JAXBContext;
import javax.xml.bind.JAXBException;
import javax.xml.bind.UnmarshalException;
import javax.xml.bind.Unmarshaller;
import javax.xml.transform.stream.StreamSource;
import org.eclipse.persistence.exceptions.XMLMarshalException;
import org.eclipse.persistence.jaxb.JAXBContextFactory;
import org.eclipse.persistence.jaxb.UnmarshallerProperties;
public class ApiRequest {
private static final Pattern proxyUrlPattern = Pattern.compile("(https://)([^@:/]+)@([0-9.]{7,15})(/.*)");
@@ -107,6 +96,36 @@ public class ApiRequest {
}
}
private static Marshaller createMarshaller(Class<?> objectClass) {
try {
// Create JAXB context aware of object's class
JAXBContext jc = JAXBContextFactory.createContext(new Class[] { objectClass }, null);
// Create marshaller
Marshaller marshaller = jc.createMarshaller();
// Set the marshaller media type to JSON
marshaller.setProperty(MarshallerProperties.MEDIA_TYPE, "application/json");
// Tell marshaller not to include JSON root element in the output
marshaller.setProperty(MarshallerProperties.JSON_INCLUDE_ROOT, false);
return marshaller;
} catch (JAXBException e) {
throw new RuntimeException("Unable to create API marshaller", e);
}
}
public static void marshall(Writer writer, Object object) throws IOException {
Marshaller marshaller = createMarshaller(object.getClass());
try {
marshaller.marshal(object, writer);
} catch (JAXBException e) {
throw new IOException("Unable to create marshall object for API", e);
}
}
public static String getParamsString(Map<String, String> params) {
StringBuilder result = new StringBuilder();
@@ -122,7 +141,7 @@ public class ApiRequest {
}
String resultString = result.toString();
return resultString.length() > 0 ? resultString.substring(0, resultString.length() - 1) : resultString;
return !resultString.isEmpty() ? resultString.substring(0, resultString.length() - 1) : resultString;
}
/**

View File

@@ -1,32 +1,10 @@
package org.qortal.api;
import io.swagger.v3.jaxrs2.integration.resources.OpenApiResource;
import java.io.InputStream;
import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.security.KeyStore;
import java.security.SecureRandom;
import javax.net.ssl.KeyManagerFactory;
import javax.net.ssl.SSLContext;
import org.eclipse.jetty.http.HttpVersion;
import org.eclipse.jetty.rewrite.handler.RedirectPatternRule;
import org.eclipse.jetty.rewrite.handler.RewriteHandler;
import org.eclipse.jetty.server.CustomRequestLog;
import org.eclipse.jetty.server.DetectorConnectionFactory;
import org.eclipse.jetty.server.HttpConfiguration;
import org.eclipse.jetty.server.HttpConnectionFactory;
import org.eclipse.jetty.server.RequestLog;
import org.eclipse.jetty.server.RequestLogWriter;
import org.eclipse.jetty.server.SecureRequestCustomizer;
import org.eclipse.jetty.server.Server;
import org.eclipse.jetty.server.ServerConnector;
import org.eclipse.jetty.server.SslConnectionFactory;
import org.eclipse.jetty.server.*;
import org.eclipse.jetty.server.handler.ErrorHandler;
import org.eclipse.jetty.server.handler.InetAccessHandler;
import org.eclipse.jetty.servlet.DefaultServlet;
@@ -39,24 +17,36 @@ import org.glassfish.jersey.server.ResourceConfig;
import org.glassfish.jersey.servlet.ServletContainer;
import org.qortal.api.resource.AnnotationPostProcessor;
import org.qortal.api.resource.ApiDefinition;
import org.qortal.api.websocket.ActiveChatsWebSocket;
import org.qortal.api.websocket.AdminStatusWebSocket;
import org.qortal.api.websocket.BlocksWebSocket;
import org.qortal.api.websocket.ChatMessagesWebSocket;
import org.qortal.api.websocket.TradeBotWebSocket;
import org.qortal.api.websocket.TradeOffersWebSocket;
import org.qortal.api.websocket.*;
import org.qortal.network.Network;
import org.qortal.settings.Settings;
import javax.net.ssl.KeyManagerFactory;
import javax.net.ssl.SSLContext;
import javax.servlet.http.HttpServletRequest;
import java.io.InputStream;
import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.security.KeyStore;
import java.security.SecureRandom;
public class ApiService {
private static ApiService instance;
private final ResourceConfig config;
private Server server;
private ApiKey apiKey;
public static final String API_VERSION_HEADER = "X-API-VERSION";
private ApiService() {
this.config = new ResourceConfig();
this.config.packages("org.qortal.api.resource");
this.config.packages("org.qortal.api.resource", "org.qortal.api.restricted.resource");
this.config.register(org.glassfish.jersey.media.multipart.MultiPartFeature.class);
this.config.register(OpenApiResource.class);
this.config.register(ApiDefinition.class);
this.config.register(AnnotationPostProcessor.class);
@@ -73,6 +63,15 @@ public class ApiService {
return this.config.getClasses();
}
public void setApiKey(ApiKey apiKey) {
this.apiKey = apiKey;
}
public ApiKey getApiKey() {
return this.apiKey;
}
public void start() {
try {
// Create API server
@@ -87,7 +86,7 @@ public class ApiService {
throw new RuntimeException("Failed to start SSL API due to broken keystore");
// BouncyCastle-specific SSLContext build
SSLContext sslContext = SSLContext.getInstance("TLS", "BCJSSE");
SSLContext sslContext = SSLContext.getInstance("TLSv1.3", "BCJSSE");
KeyManagerFactory keyManagerFactory = KeyManagerFactory.getInstance("PKIX", "BCJSSE");
KeyStore keyStore = KeyStore.getInstance(KeyStore.getDefaultType(), "BC");
@@ -117,13 +116,13 @@ public class ApiService {
ServerConnector portUnifiedConnector = new ServerConnector(this.server,
new DetectorConnectionFactory(sslConnectionFactory),
httpConnectionFactory);
portUnifiedConnector.setHost(Settings.getInstance().getBindAddress());
portUnifiedConnector.setHost(Network.getInstance().getBindAddress());
portUnifiedConnector.setPort(Settings.getInstance().getApiPort());
this.server.addConnector(portUnifiedConnector);
} else {
// Non-SSL
InetAddress bindAddr = InetAddress.getByName(Settings.getInstance().getBindAddress());
InetAddress bindAddr = InetAddress.getByName(Network.getInstance().getBindAddress());
InetSocketAddress endpoint = new InetSocketAddress(bindAddr, Settings.getInstance().getApiPort());
this.server = new Server(endpoint);
}
@@ -196,10 +195,16 @@ public class ApiService {
context.addServlet(AdminStatusWebSocket.class, "/websockets/admin/status");
context.addServlet(BlocksWebSocket.class, "/websockets/blocks");
context.addServlet(DataMonitorSocket.class, "/websockets/datamonitor");
context.addServlet(ActiveChatsWebSocket.class, "/websockets/chat/active/*");
context.addServlet(ChatMessagesWebSocket.class, "/websockets/chat/messages");
context.addServlet(UnsignedFeesSocket.class, "/websockets/crosschain/unsignedfees");
context.addServlet(TradeOffersWebSocket.class, "/websockets/crosschain/tradeoffers");
context.addServlet(TradeBotWebSocket.class, "/websockets/crosschain/tradebot");
context.addServlet(TradePresenceWebSocket.class, "/websockets/crosschain/tradepresence");
// Deprecated
context.addServlet(PresenceWebSocket.class, "/websockets/presence");
// Start server
this.server.start();
@@ -220,4 +225,19 @@ public class ApiService {
this.server = null;
}
public static int getApiVersion(HttpServletRequest request) {
// Get API version
String apiVersionString = request.getHeader(API_VERSION_HEADER);
if (apiVersionString == null) {
// Try query string - this is needed to avoid a CORS preflight. See: https://stackoverflow.com/a/43881141
apiVersionString = request.getParameter("apiVersion");
}
int apiVersion = 1;
if (apiVersionString != null) {
apiVersion = Integer.parseInt(apiVersionString);
}
return apiVersion;
}
}

View File

@@ -1,8 +1,8 @@
package org.qortal.api;
import javax.xml.bind.annotation.adapters.XmlAdapter;
import org.qortal.utils.Base58;
import org.bitcoinj.core.Base58;
import javax.xml.bind.annotation.adapters.XmlAdapter;
public class Base58TypeAdapter extends XmlAdapter<String, byte[]> {

View File

@@ -1,8 +1,7 @@
package org.qortal.api;
import java.math.BigDecimal;
import javax.xml.bind.annotation.adapters.XmlAdapter;
import java.math.BigDecimal;
public class BigDecimalTypeAdapter extends XmlAdapter<String, BigDecimal> {

View File

@@ -4,9 +4,11 @@ import io.swagger.v3.oas.models.Operation;
import io.swagger.v3.oas.models.PathItem;
import io.swagger.v3.oas.models.info.Info;
import io.swagger.v3.oas.models.responses.ApiResponse;
import static java.util.Arrays.asList;
import java.util.List;
import static java.util.Arrays.asList;
class Constants {
public static final String APIERROR_CONTEXT_PATH = "/Api";
public static final String APIERROR_KEY = "ApiError/%s";

View File

@@ -0,0 +1,174 @@
package org.qortal.api;
import io.swagger.v3.jaxrs2.integration.resources.OpenApiResource;
import org.eclipse.jetty.http.HttpVersion;
import org.eclipse.jetty.rewrite.handler.RewriteHandler;
import org.eclipse.jetty.server.*;
import org.eclipse.jetty.server.handler.ErrorHandler;
import org.eclipse.jetty.server.handler.InetAccessHandler;
import org.eclipse.jetty.servlet.FilterHolder;
import org.eclipse.jetty.servlet.ServletContextHandler;
import org.eclipse.jetty.servlet.ServletHolder;
import org.eclipse.jetty.servlets.CrossOriginFilter;
import org.eclipse.jetty.util.ssl.SslContextFactory;
import org.glassfish.jersey.server.ResourceConfig;
import org.glassfish.jersey.servlet.ServletContainer;
import org.qortal.api.resource.AnnotationPostProcessor;
import org.qortal.api.resource.ApiDefinition;
import org.qortal.network.Network;
import org.qortal.repository.DataException;
import org.qortal.settings.Settings;
import javax.net.ssl.KeyManagerFactory;
import javax.net.ssl.SSLContext;
import java.io.InputStream;
import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.security.KeyStore;
import java.security.SecureRandom;
public class DevProxyService {
private static DevProxyService instance;
private final ResourceConfig config;
private Server server;
private DevProxyService() {
this.config = new ResourceConfig();
this.config.packages("org.qortal.api.proxy.resource", "org.qortal.api.resource");
this.config.register(org.glassfish.jersey.media.multipart.MultiPartFeature.class);
this.config.register(OpenApiResource.class);
this.config.register(ApiDefinition.class);
this.config.register(AnnotationPostProcessor.class);
}
public static DevProxyService getInstance() {
if (instance == null)
instance = new DevProxyService();
return instance;
}
public Iterable<Class<?>> getResources() {
return this.config.getClasses();
}
public void start() throws DataException {
try {
// Create API server
// SSL support if requested
String keystorePathname = Settings.getInstance().getSslKeystorePathname();
String keystorePassword = Settings.getInstance().getSslKeystorePassword();
if (keystorePathname != null && keystorePassword != null) {
// SSL version
if (!Files.isReadable(Path.of(keystorePathname)))
throw new RuntimeException("Failed to start SSL API due to broken keystore");
// BouncyCastle-specific SSLContext build
SSLContext sslContext = SSLContext.getInstance("TLS", "BCJSSE");
KeyManagerFactory keyManagerFactory = KeyManagerFactory.getInstance("PKIX", "BCJSSE");
KeyStore keyStore = KeyStore.getInstance(KeyStore.getDefaultType(), "BC");
try (InputStream keystoreStream = Files.newInputStream(Paths.get(keystorePathname))) {
keyStore.load(keystoreStream, keystorePassword.toCharArray());
}
keyManagerFactory.init(keyStore, keystorePassword.toCharArray());
sslContext.init(keyManagerFactory.getKeyManagers(), null, new SecureRandom());
SslContextFactory.Server sslContextFactory = new SslContextFactory.Server();
sslContextFactory.setSslContext(sslContext);
this.server = new Server();
HttpConfiguration httpConfig = new HttpConfiguration();
httpConfig.setSecureScheme("https");
httpConfig.setSecurePort(Settings.getInstance().getDevProxyPort());
SecureRequestCustomizer src = new SecureRequestCustomizer();
httpConfig.addCustomizer(src);
HttpConnectionFactory httpConnectionFactory = new HttpConnectionFactory(httpConfig);
SslConnectionFactory sslConnectionFactory = new SslConnectionFactory(sslContextFactory, HttpVersion.HTTP_1_1.asString());
ServerConnector portUnifiedConnector = new ServerConnector(this.server,
new DetectorConnectionFactory(sslConnectionFactory),
httpConnectionFactory);
portUnifiedConnector.setHost(Network.getInstance().getBindAddress());
portUnifiedConnector.setPort(Settings.getInstance().getDevProxyPort());
this.server.addConnector(portUnifiedConnector);
} else {
// Non-SSL
InetAddress bindAddr = InetAddress.getByName(Network.getInstance().getBindAddress());
InetSocketAddress endpoint = new InetSocketAddress(bindAddr, Settings.getInstance().getDevProxyPort());
this.server = new Server(endpoint);
}
// Error handler
ErrorHandler errorHandler = new ApiErrorHandler();
this.server.setErrorHandler(errorHandler);
// Request logging
if (Settings.getInstance().isDevProxyLoggingEnabled()) {
RequestLogWriter logWriter = new RequestLogWriter("devproxy-requests.log");
logWriter.setAppend(true);
logWriter.setTimeZone("UTC");
RequestLog requestLog = new CustomRequestLog(logWriter, CustomRequestLog.EXTENDED_NCSA_FORMAT);
this.server.setRequestLog(requestLog);
}
// Access handler (currently no whitelist is used)
InetAccessHandler accessHandler = new InetAccessHandler();
this.server.setHandler(accessHandler);
// URL rewriting
RewriteHandler rewriteHandler = new RewriteHandler();
accessHandler.setHandler(rewriteHandler);
// Context
ServletContextHandler context = new ServletContextHandler(ServletContextHandler.NO_SESSIONS);
context.setContextPath("/");
rewriteHandler.setHandler(context);
// Cross-origin resource sharing
FilterHolder corsFilterHolder = new FilterHolder(CrossOriginFilter.class);
corsFilterHolder.setInitParameter(CrossOriginFilter.ALLOWED_ORIGINS_PARAM, "*");
corsFilterHolder.setInitParameter(CrossOriginFilter.ALLOWED_METHODS_PARAM, "GET, POST, DELETE");
corsFilterHolder.setInitParameter(CrossOriginFilter.CHAIN_PREFLIGHT_PARAM, "false");
context.addFilter(corsFilterHolder, "/*", null);
// API servlet
ServletContainer container = new ServletContainer(this.config);
ServletHolder apiServlet = new ServletHolder(container);
apiServlet.setInitOrder(1);
context.addServlet(apiServlet, "/*");
// Start server
this.server.start();
} catch (Exception e) {
// Failed to start
throw new DataException("Failed to start developer proxy", e);
}
}
public void stop() {
try {
// Stop server
this.server.stop();
} catch (Exception e) {
// Failed to stop
}
this.server = null;
instance = null;
}
}

View File

@@ -0,0 +1,172 @@
package org.qortal.api;
import io.swagger.v3.jaxrs2.integration.resources.OpenApiResource;
import org.eclipse.jetty.http.HttpVersion;
import org.eclipse.jetty.rewrite.handler.RewriteHandler;
import org.eclipse.jetty.server.*;
import org.eclipse.jetty.server.handler.ErrorHandler;
import org.eclipse.jetty.server.handler.InetAccessHandler;
import org.eclipse.jetty.servlet.FilterHolder;
import org.eclipse.jetty.servlet.ServletContextHandler;
import org.eclipse.jetty.servlet.ServletHolder;
import org.eclipse.jetty.servlets.CrossOriginFilter;
import org.eclipse.jetty.util.ssl.SslContextFactory;
import org.glassfish.jersey.server.ResourceConfig;
import org.glassfish.jersey.servlet.ServletContainer;
import org.qortal.api.resource.AnnotationPostProcessor;
import org.qortal.api.resource.ApiDefinition;
import org.qortal.network.Network;
import org.qortal.settings.Settings;
import javax.net.ssl.KeyManagerFactory;
import javax.net.ssl.SSLContext;
import java.io.InputStream;
import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.security.KeyStore;
import java.security.SecureRandom;
public class DomainMapService {
private static DomainMapService instance;
private final ResourceConfig config;
private Server server;
private DomainMapService() {
this.config = new ResourceConfig();
this.config.packages("org.qortal.api.resource", "org.qortal.api.domainmap.resource");
this.config.register(org.glassfish.jersey.media.multipart.MultiPartFeature.class);
this.config.register(OpenApiResource.class);
this.config.register(ApiDefinition.class);
this.config.register(AnnotationPostProcessor.class);
}
public static DomainMapService getInstance() {
if (instance == null)
instance = new DomainMapService();
return instance;
}
public Iterable<Class<?>> getResources() {
return this.config.getClasses();
}
public void start() {
try {
// Create API server
// SSL support if requested
String keystorePathname = Settings.getInstance().getSslKeystorePathname();
String keystorePassword = Settings.getInstance().getSslKeystorePassword();
if (keystorePathname != null && keystorePassword != null) {
// SSL version
if (!Files.isReadable(Path.of(keystorePathname)))
throw new RuntimeException("Failed to start SSL API due to broken keystore");
// BouncyCastle-specific SSLContext build
SSLContext sslContext = SSLContext.getInstance("TLSv1.3", "BCJSSE");
KeyManagerFactory keyManagerFactory = KeyManagerFactory.getInstance("PKIX", "BCJSSE");
KeyStore keyStore = KeyStore.getInstance(KeyStore.getDefaultType(), "BC");
try (InputStream keystoreStream = Files.newInputStream(Paths.get(keystorePathname))) {
keyStore.load(keystoreStream, keystorePassword.toCharArray());
}
keyManagerFactory.init(keyStore, keystorePassword.toCharArray());
sslContext.init(keyManagerFactory.getKeyManagers(), null, new SecureRandom());
SslContextFactory.Server sslContextFactory = new SslContextFactory.Server();
sslContextFactory.setSslContext(sslContext);
this.server = new Server();
HttpConfiguration httpConfig = new HttpConfiguration();
httpConfig.setSecureScheme("https");
httpConfig.setSecurePort(Settings.getInstance().getDomainMapPort());
SecureRequestCustomizer src = new SecureRequestCustomizer();
httpConfig.addCustomizer(src);
HttpConnectionFactory httpConnectionFactory = new HttpConnectionFactory(httpConfig);
SslConnectionFactory sslConnectionFactory = new SslConnectionFactory(sslContextFactory, HttpVersion.HTTP_1_1.asString());
ServerConnector portUnifiedConnector = new ServerConnector(this.server,
new DetectorConnectionFactory(sslConnectionFactory),
httpConnectionFactory);
portUnifiedConnector.setHost(Network.getInstance().getBindAddress());
portUnifiedConnector.setPort(Settings.getInstance().getDomainMapPort());
this.server.addConnector(portUnifiedConnector);
} else {
// Non-SSL
InetAddress bindAddr = InetAddress.getByName(Network.getInstance().getBindAddress());
InetSocketAddress endpoint = new InetSocketAddress(bindAddr, Settings.getInstance().getDomainMapPort());
this.server = new Server(endpoint);
}
// Error handler
ErrorHandler errorHandler = new ApiErrorHandler();
this.server.setErrorHandler(errorHandler);
// Request logging
if (Settings.getInstance().isDomainMapLoggingEnabled()) {
RequestLogWriter logWriter = new RequestLogWriter("domainmap-requests.log");
logWriter.setAppend(true);
logWriter.setTimeZone("UTC");
RequestLog requestLog = new CustomRequestLog(logWriter, CustomRequestLog.EXTENDED_NCSA_FORMAT);
this.server.setRequestLog(requestLog);
}
// Access handler (currently no whitelist is used)
InetAccessHandler accessHandler = new InetAccessHandler();
this.server.setHandler(accessHandler);
// URL rewriting
RewriteHandler rewriteHandler = new RewriteHandler();
accessHandler.setHandler(rewriteHandler);
// Context
ServletContextHandler context = new ServletContextHandler(ServletContextHandler.NO_SESSIONS);
context.setContextPath("/");
rewriteHandler.setHandler(context);
// Cross-origin resource sharing
FilterHolder corsFilterHolder = new FilterHolder(CrossOriginFilter.class);
corsFilterHolder.setInitParameter(CrossOriginFilter.ALLOWED_ORIGINS_PARAM, "*");
corsFilterHolder.setInitParameter(CrossOriginFilter.ALLOWED_METHODS_PARAM, "GET, POST, DELETE");
corsFilterHolder.setInitParameter(CrossOriginFilter.CHAIN_PREFLIGHT_PARAM, "false");
context.addFilter(corsFilterHolder, "/*", null);
// API servlet
ServletContainer container = new ServletContainer(this.config);
ServletHolder apiServlet = new ServletHolder(container);
apiServlet.setInitOrder(1);
context.addServlet(apiServlet, "/*");
// Start server
this.server.start();
} catch (Exception e) {
// Failed to start
throw new RuntimeException("Failed to start API", e);
}
}
public void stop() {
try {
// Stop server
this.server.stop();
} catch (Exception e) {
// Failed to stop
}
this.server = null;
}
}

View File

@@ -0,0 +1,172 @@
package org.qortal.api;
import io.swagger.v3.jaxrs2.integration.resources.OpenApiResource;
import org.eclipse.jetty.http.HttpVersion;
import org.eclipse.jetty.rewrite.handler.RewriteHandler;
import org.eclipse.jetty.server.*;
import org.eclipse.jetty.server.handler.ErrorHandler;
import org.eclipse.jetty.server.handler.InetAccessHandler;
import org.eclipse.jetty.servlet.FilterHolder;
import org.eclipse.jetty.servlet.ServletContextHandler;
import org.eclipse.jetty.servlet.ServletHolder;
import org.eclipse.jetty.servlets.CrossOriginFilter;
import org.eclipse.jetty.util.ssl.SslContextFactory;
import org.glassfish.jersey.server.ResourceConfig;
import org.glassfish.jersey.servlet.ServletContainer;
import org.qortal.api.resource.AnnotationPostProcessor;
import org.qortal.api.resource.ApiDefinition;
import org.qortal.network.Network;
import org.qortal.settings.Settings;
import javax.net.ssl.KeyManagerFactory;
import javax.net.ssl.SSLContext;
import java.io.InputStream;
import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.security.KeyStore;
import java.security.SecureRandom;
public class GatewayService {
private static GatewayService instance;
private final ResourceConfig config;
private Server server;
private GatewayService() {
this.config = new ResourceConfig();
this.config.packages("org.qortal.api.resource", "org.qortal.api.gateway.resource");
this.config.register(org.glassfish.jersey.media.multipart.MultiPartFeature.class);
this.config.register(OpenApiResource.class);
this.config.register(ApiDefinition.class);
this.config.register(AnnotationPostProcessor.class);
}
public static GatewayService getInstance() {
if (instance == null)
instance = new GatewayService();
return instance;
}
public Iterable<Class<?>> getResources() {
return this.config.getClasses();
}
public void start() {
try {
// Create API server
// SSL support if requested
String keystorePathname = Settings.getInstance().getSslKeystorePathname();
String keystorePassword = Settings.getInstance().getSslKeystorePassword();
if (keystorePathname != null && keystorePassword != null) {
// SSL version
if (!Files.isReadable(Path.of(keystorePathname)))
throw new RuntimeException("Failed to start SSL API due to broken keystore");
// BouncyCastle-specific SSLContext build
SSLContext sslContext = SSLContext.getInstance("TLSv1.3", "BCJSSE");
KeyManagerFactory keyManagerFactory = KeyManagerFactory.getInstance("PKIX", "BCJSSE");
KeyStore keyStore = KeyStore.getInstance(KeyStore.getDefaultType(), "BC");
try (InputStream keystoreStream = Files.newInputStream(Paths.get(keystorePathname))) {
keyStore.load(keystoreStream, keystorePassword.toCharArray());
}
keyManagerFactory.init(keyStore, keystorePassword.toCharArray());
sslContext.init(keyManagerFactory.getKeyManagers(), null, new SecureRandom());
SslContextFactory.Server sslContextFactory = new SslContextFactory.Server();
sslContextFactory.setSslContext(sslContext);
this.server = new Server();
HttpConfiguration httpConfig = new HttpConfiguration();
httpConfig.setSecureScheme("https");
httpConfig.setSecurePort(Settings.getInstance().getGatewayPort());
SecureRequestCustomizer src = new SecureRequestCustomizer();
httpConfig.addCustomizer(src);
HttpConnectionFactory httpConnectionFactory = new HttpConnectionFactory(httpConfig);
SslConnectionFactory sslConnectionFactory = new SslConnectionFactory(sslContextFactory, HttpVersion.HTTP_1_1.asString());
ServerConnector portUnifiedConnector = new ServerConnector(this.server,
new DetectorConnectionFactory(sslConnectionFactory),
httpConnectionFactory);
portUnifiedConnector.setHost(Network.getInstance().getBindAddress());
portUnifiedConnector.setPort(Settings.getInstance().getGatewayPort());
this.server.addConnector(portUnifiedConnector);
} else {
// Non-SSL
InetAddress bindAddr = InetAddress.getByName(Network.getInstance().getBindAddress());
InetSocketAddress endpoint = new InetSocketAddress(bindAddr, Settings.getInstance().getGatewayPort());
this.server = new Server(endpoint);
}
// Error handler
ErrorHandler errorHandler = new ApiErrorHandler();
this.server.setErrorHandler(errorHandler);
// Request logging
if (Settings.getInstance().isGatewayLoggingEnabled()) {
RequestLogWriter logWriter = new RequestLogWriter("gateway-requests.log");
logWriter.setAppend(true);
logWriter.setTimeZone("UTC");
RequestLog requestLog = new CustomRequestLog(logWriter, CustomRequestLog.EXTENDED_NCSA_FORMAT);
this.server.setRequestLog(requestLog);
}
// Access handler (currently no whitelist is used)
InetAccessHandler accessHandler = new InetAccessHandler();
this.server.setHandler(accessHandler);
// URL rewriting
RewriteHandler rewriteHandler = new RewriteHandler();
accessHandler.setHandler(rewriteHandler);
// Context
ServletContextHandler context = new ServletContextHandler(ServletContextHandler.NO_SESSIONS);
context.setContextPath("/");
rewriteHandler.setHandler(context);
// Cross-origin resource sharing
FilterHolder corsFilterHolder = new FilterHolder(CrossOriginFilter.class);
corsFilterHolder.setInitParameter(CrossOriginFilter.ALLOWED_ORIGINS_PARAM, "*");
corsFilterHolder.setInitParameter(CrossOriginFilter.ALLOWED_METHODS_PARAM, "GET, POST, DELETE");
corsFilterHolder.setInitParameter(CrossOriginFilter.CHAIN_PREFLIGHT_PARAM, "false");
context.addFilter(corsFilterHolder, "/*", null);
// API servlet
ServletContainer container = new ServletContainer(this.config);
ServletHolder apiServlet = new ServletHolder(container);
apiServlet.setInitOrder(1);
context.addServlet(apiServlet, "/*");
// Start server
this.server.start();
} catch (Exception e) {
// Failed to start
throw new RuntimeException("Failed to start API", e);
}
}
public void stop() {
try {
// Stop server
this.server.stop();
} catch (Exception e) {
// Failed to stop
}
this.server = null;
}
}

View File

@@ -0,0 +1,99 @@
package org.qortal.api;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;
import org.qortal.arbitrary.misc.Service;
import java.util.Objects;
public class HTMLParser {
private static final Logger LOGGER = LogManager.getLogger(HTMLParser.class);
private String qdnBase;
private String qdnBaseWithPath;
private byte[] data;
private String qdnContext;
private String resourceId;
private Service service;
private String identifier;
private String path;
private String theme;
private String lang;
private boolean usingCustomRouting;
public HTMLParser(String resourceId, String inPath, String prefix, boolean includeResourceIdInPrefix, byte[] data,
String qdnContext, Service service, String identifier, String theme, boolean usingCustomRouting, String lang) {
String inPathWithoutFilename = inPath.contains("/") ? inPath.substring(0, inPath.lastIndexOf('/')) : String.format("/%s",inPath);
this.qdnBase = includeResourceIdInPrefix ? String.format("%s/%s", prefix, resourceId) : prefix;
this.qdnBaseWithPath = includeResourceIdInPrefix ? String.format("%s/%s%s", prefix, resourceId, inPathWithoutFilename) : String.format("%s%s", prefix, inPathWithoutFilename);
this.data = data;
this.qdnContext = qdnContext;
this.resourceId = resourceId;
this.service = service;
this.identifier = identifier;
this.path = inPath;
this.theme = theme;
this.lang = lang;
this.usingCustomRouting = usingCustomRouting;
}
public void addAdditionalHeaderTags() {
String fileContents = new String(data);
Document document = Jsoup.parse(fileContents);
Elements head = document.getElementsByTag("head");
if (!head.isEmpty()) {
// Add q-apps script tag
String qAppsScriptElement = String.format("<script src=\"/apps/q-apps.js?time=%d\">", System.currentTimeMillis());
head.get(0).prepend(qAppsScriptElement);
// Add q-apps gateway script tag if in gateway mode
if (Objects.equals(this.qdnContext, "gateway")) {
String qAppsGatewayScriptElement = String.format("<script src=\"/apps/q-apps-gateway.js?time=%d\">", System.currentTimeMillis());
head.get(0).prepend(qAppsGatewayScriptElement);
}
// Escape and add vars
String qdnContext = this.qdnContext != null ? this.qdnContext.replace("\\", "").replace("\"","\\\"") : "";
String service = this.service.toString().replace("\\", "").replace("\"","\\\"");
String name = this.resourceId != null ? this.resourceId.replace("\\", "").replace("\"","\\\"") : "";
String identifier = this.identifier != null ? this.identifier.replace("\\", "").replace("\"","\\\"") : "";
String path = this.path != null ? this.path.replace("\\", "").replace("\"","\\\"") : "";
String theme = this.theme != null ? this.theme.replace("\\", "").replace("\"","\\\"") : "";
String lang = this.lang != null ? this.lang.replace("\\", "").replace("\"", "\\\"") : "";
String qdnBase = this.qdnBase != null ? this.qdnBase.replace("\\", "").replace("\"","\\\"") : "";
String qdnBaseWithPath = this.qdnBaseWithPath != null ? this.qdnBaseWithPath.replace("\\", "").replace("\"","\\\"") : "";
String qdnContextVar = String.format(
"<script>var _qdnContext=\"%s\"; var _qdnTheme=\"%s\"; var _qdnLang=\"%s\"; var _qdnService=\"%s\"; var _qdnName=\"%s\"; var _qdnIdentifier=\"%s\"; var _qdnPath=\"%s\"; var _qdnBase=\"%s\"; var _qdnBaseWithPath=\"%s\";</script>",
qdnContext, theme, lang, service, name, identifier, path, qdnBase, qdnBaseWithPath
);
head.get(0).prepend(qdnContextVar);
// Add base href tag
// Exclude the path if this request was routed back to the index automatically
String baseHref = this.usingCustomRouting ? this.qdnBase : this.qdnBaseWithPath;
String baseElement = String.format("<base href=\"%s/\">", baseHref);
head.get(0).prepend(baseElement);
// Add meta charset tag
String metaCharsetElement = "<meta charset=\"UTF-8\">";
head.get(0).prepend(metaCharsetElement);
}
String html = document.html();
this.data = html.getBytes();
}
public static boolean isHtmlFile(String path) {
if (path.endsWith(".html") || path.endsWith(".htm") || path.isEmpty()) {
return true;
}
return false;
}
public byte[] getData() {
return this.data;
}
}

View File

@@ -1,8 +1,7 @@
package org.qortal.api;
import java.math.BigDecimal;
import javax.xml.bind.annotation.adapters.XmlAdapter;
import java.math.BigDecimal;
public class RewardSharePercentTypeAdapter extends XmlAdapter<String, Integer> {

View File

@@ -0,0 +1,6 @@
package org.qortal.api;
public enum SearchMode {
LATEST,
ALL
}

View File

@@ -1,33 +1,127 @@
package org.qortal.api;
import java.net.InetAddress;
import java.net.UnknownHostException;
import org.qortal.arbitrary.ArbitraryDataResource;
import org.qortal.arbitrary.misc.Service;
import org.qortal.controller.arbitrary.ArbitraryDataRenderManager;
import org.qortal.settings.Settings;
import javax.servlet.http.HttpServletRequest;
import org.qortal.settings.Settings;
import java.io.IOException;
import java.net.InetAddress;
import java.net.UnknownHostException;
public abstract class Security {
public static final String API_KEY_HEADER = "X-API-KEY";
/**
* Check API call is allowed, retrieving the API key from the request header or GET/POST parameters where required
* @param request
*/
public static void checkApiCallAllowed(HttpServletRequest request) {
String expectedApiKey = Settings.getInstance().getApiKey();
String passedApiKey = request.getHeader(API_KEY_HEADER);
checkApiCallAllowed(request, null);
}
if ((expectedApiKey != null && !expectedApiKey.equals(passedApiKey)) ||
(passedApiKey != null && !passedApiKey.equals(expectedApiKey)))
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.UNAUTHORIZED);
/**
* Check API call is allowed, retrieving the API key first from the passedApiKey parameter, with a fallback
* to the request header or GET/POST parameters when null.
* @param request
* @param passedApiKey - the API key to test, or null if it should be retrieved from the request headers.
*/
public static void checkApiCallAllowed(HttpServletRequest request, String passedApiKey) {
// We may want to allow automatic authentication for local requests, if enabled in settings
boolean localAuthBypassEnabled = Settings.getInstance().isLocalAuthBypassEnabled();
if (localAuthBypassEnabled) {
try {
InetAddress remoteAddr = InetAddress.getByName(request.getRemoteAddr());
if (remoteAddr.isLoopbackAddress()) {
// Request originates from loopback address, so allow it
return;
}
} catch (UnknownHostException e) {
// Ignore failure, and fallback to API key authentication
}
}
InetAddress remoteAddr;
// Retrieve the API key
ApiKey apiKey = Security.getApiKey(request);
if (!apiKey.generated()) {
// Not generated an API key yet, so disallow sensitive API calls
throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.UNAUTHORIZED, "API key not generated");
}
// We require an API key to be passed
if (passedApiKey == null) {
// API call not passed as a parameter, so try the header
passedApiKey = request.getHeader(API_KEY_HEADER);
}
if (passedApiKey == null) {
// Try query string - this is needed to avoid a CORS preflight. See: https://stackoverflow.com/a/43881141
passedApiKey = request.getParameter("apiKey");
}
if (passedApiKey == null) {
throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.UNAUTHORIZED, "Missing 'X-API-KEY' header");
}
// The API keys must match
if (!apiKey.toString().equals(passedApiKey)) {
throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.UNAUTHORIZED, "API key invalid");
}
}
public static void disallowLoopbackRequests(HttpServletRequest request) {
try {
remoteAddr = InetAddress.getByName(request.getRemoteAddr());
InetAddress remoteAddr = InetAddress.getByName(request.getRemoteAddr());
if (remoteAddr.isLoopbackAddress() && !Settings.getInstance().isGatewayLoopbackEnabled()) {
throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.UNAUTHORIZED, "Local requests not allowed");
}
} catch (UnknownHostException e) {
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.UNAUTHORIZED);
}
}
if (!remoteAddr.isLoopbackAddress())
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.UNAUTHORIZED);
public static void disallowLoopbackRequestsIfAuthBypassEnabled(HttpServletRequest request) {
if (Settings.getInstance().isLocalAuthBypassEnabled()) {
try {
InetAddress remoteAddr = InetAddress.getByName(request.getRemoteAddr());
if (remoteAddr.isLoopbackAddress()) {
throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.UNAUTHORIZED, "Local requests not allowed when localAuthBypassEnabled is enabled in settings");
}
} catch (UnknownHostException e) {
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.UNAUTHORIZED);
}
}
}
public static void requirePriorAuthorization(HttpServletRequest request, String resourceId, Service service, String identifier) {
ArbitraryDataResource resource = new ArbitraryDataResource(resourceId, null, service, identifier);
if (!ArbitraryDataRenderManager.getInstance().isAuthorized(resource)) {
throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.UNAUTHORIZED, "Call /render/authorize first");
}
}
public static void requirePriorAuthorizationOrApiKey(HttpServletRequest request, String resourceId, Service service, String identifier, String apiKey) {
try {
Security.checkApiCallAllowed(request, apiKey);
} catch (ApiException e) {
// API call wasn't allowed, but maybe it was pre-authorized
Security.requirePriorAuthorization(request, resourceId, service, identifier);
}
}
public static ApiKey getApiKey(HttpServletRequest request) {
ApiKey apiKey = ApiService.getInstance().getApiKey();
if (apiKey == null) {
try {
apiKey = new ApiKey();
} catch (IOException e) {
// Couldn't load API key - so we need to treat it as not generated, and therefore unauthorized
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.UNAUTHORIZED);
}
ApiService.getInstance().setApiKey(apiKey);
}
return apiKey;
}
}

View File

@@ -1,18 +1,17 @@
package org.qortal.api;
import org.eclipse.persistence.oxm.annotations.XmlVariableNode;
import org.qortal.transaction.Transaction.TransactionType;
import javax.xml.bind.annotation.XmlTransient;
import javax.xml.bind.annotation.XmlValue;
import javax.xml.bind.annotation.adapters.XmlAdapter;
import java.util.ArrayList;
import java.util.EnumMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import javax.xml.bind.annotation.XmlTransient;
import javax.xml.bind.annotation.XmlValue;
import javax.xml.bind.annotation.adapters.XmlAdapter;
import org.eclipse.persistence.oxm.annotations.XmlVariableNode;
import org.qortal.transaction.Transaction.TransactionType;
public class TransactionCountMapXmlAdapter extends XmlAdapter<TransactionCountMapXmlAdapter.StringIntegerMap, Map<TransactionType, Integer>> {
public static class StringIntegerMap {

View File

@@ -0,0 +1,58 @@
package org.qortal.api.domainmap.resource;
import io.swagger.v3.oas.annotations.tags.Tag;
import org.qortal.arbitrary.ArbitraryDataFile.ResourceIdType;
import org.qortal.arbitrary.ArbitraryDataRenderer;
import org.qortal.arbitrary.misc.Service;
import org.qortal.settings.Settings;
import javax.servlet.ServletContext;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import javax.ws.rs.GET;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
import javax.ws.rs.core.Context;
import java.util.Map;
@Path("/")
@Tag(name = "Domain Map")
public class DomainMapResource {
@Context HttpServletRequest request;
@Context HttpServletResponse response;
@Context ServletContext context;
@GET
public HttpServletResponse getIndexByDomainMap() {
return this.getDomainMap("/");
}
@GET
@Path("{path:.*}")
public HttpServletResponse getPathByDomainMap(@PathParam("path") String inPath) {
return this.getDomainMap(inPath);
}
private HttpServletResponse getDomainMap(String inPath) {
Map<String, String> domainMap = Settings.getInstance().getSimpleDomainMap();
if (domainMap != null && domainMap.containsKey(request.getServerName())) {
// Build synchronously, so that we don't need to make the summary API endpoints available over
// the domain map server. This means that there will be no loading screen, but this is potentially
// preferred in this situation anyway (e.g. to avoid confusing search engine robots).
return this.get(domainMap.get(request.getServerName()), ResourceIdType.NAME, Service.WEBSITE, null, inPath, null, "", false, false);
}
return ArbitraryDataRenderer.getResponse(response, 404, "Error 404: File Not Found");
}
private HttpServletResponse get(String resourceId, ResourceIdType resourceIdType, Service service, String identifier,
String inPath, String secret58, String prefix, boolean includeResourceIdInPrefix, boolean async) {
ArbitraryDataRenderer renderer = new ArbitraryDataRenderer(resourceId, resourceIdType, service, identifier, inPath,
secret58, prefix, includeResourceIdInPrefix, async, "domainMap", request, response, context);
return renderer.render();
}
}

View File

@@ -0,0 +1,152 @@
package org.qortal.api.gateway.resource;
import io.swagger.v3.oas.annotations.security.SecurityRequirement;
import io.swagger.v3.oas.annotations.tags.Tag;
import org.apache.commons.lang3.StringUtils;
import org.qortal.api.ApiError;
import org.qortal.api.ApiExceptionFactory;
import org.qortal.api.Security;
import org.qortal.arbitrary.ArbitraryDataFile;
import org.qortal.arbitrary.ArbitraryDataFile.ResourceIdType;
import org.qortal.arbitrary.ArbitraryDataReader;
import org.qortal.arbitrary.ArbitraryDataRenderer;
import org.qortal.arbitrary.ArbitraryDataResource;
import org.qortal.arbitrary.misc.Service;
import org.qortal.data.arbitrary.ArbitraryResourceStatus;
import org.qortal.repository.DataException;
import org.qortal.repository.Repository;
import org.qortal.repository.RepositoryManager;
import javax.servlet.ServletContext;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import javax.ws.rs.GET;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
import javax.ws.rs.core.Context;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.LinkedList;
import java.util.List;
@Path("/")
@Tag(name = "Gateway")
public class GatewayResource {
@Context HttpServletRequest request;
@Context HttpServletResponse response;
@Context ServletContext context;
private ArbitraryResourceStatus getStatus(Service service, String name, String identifier, Boolean build) {
// If "build=true" has been specified in the query string, build the resource before returning its status
if (build != null && build) {
try {
ArbitraryDataReader reader = new ArbitraryDataReader(name, ArbitraryDataFile.ResourceIdType.NAME, service, null);
if (!reader.isBuilding()) {
reader.loadSynchronously(false);
}
} catch (Exception e) {
// No need to handle exception, as it will be reflected in the status
}
}
try (final Repository repository = RepositoryManager.getRepository()) {
ArbitraryDataResource resource = new ArbitraryDataResource(name, ResourceIdType.NAME, service, identifier);
return resource.getStatus(repository);
} catch (DataException e) {
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e);
}
}
@GET
public HttpServletResponse getRoot() {
return ArbitraryDataRenderer.getResponse(response, 200, "");
}
@GET
@Path("{path:.*}")
@SecurityRequirement(name = "apiKey")
public HttpServletResponse getPath(@PathParam("path") String inPath) {
// Block requests from localhost, to prevent websites/apps from running javascript that fetches unvetted data
Security.disallowLoopbackRequests(request);
return this.parsePath(inPath, "gateway", null, true, true);
}
private HttpServletResponse parsePath(String inPath, String qdnContext, String secret58, boolean includeResourceIdInPrefix, boolean async) {
if (inPath == null || inPath.isEmpty()) {
// Assume not a real file
return ArbitraryDataRenderer.getResponse(response, 404, "Error 404: File Not Found");
}
// Default service is WEBSITE
Service service = Service.WEBSITE;
String name = null;
String identifier = null;
String outPath = "";
List<String> prefixParts = new ArrayList<>();
if (!inPath.contains("/")) {
// Assume entire inPath is a registered name
name = inPath;
}
else {
// Parse the path to determine what we need to load
List<String> parts = new LinkedList<>(Arrays.asList(inPath.split("/")));
// Check if the first element is a service
try {
Service parsedService = Service.valueOf(parts.get(0).toUpperCase());
if (parsedService != null) {
// First element matches a service, so we can assume it is one
service = parsedService;
parts.remove(0);
prefixParts.add(service.name());
}
} catch (IllegalArgumentException e) {
// Not a service
}
if (parts.isEmpty()) {
// We need more than just a service
return ArbitraryDataRenderer.getResponse(response, 404, "Error 404: File Not Found");
}
// Service is removed, so assume first element is now a registered name
name = parts.get(0);
parts.remove(0);
if (!parts.isEmpty()) {
// Name is removed, so check if the first element is now an identifier
ArbitraryResourceStatus status = this.getStatus(service, name, parts.get(0), false);
if (status.getTotalChunkCount() > 0) {
// Matched service, name and identifier combination - so assume this is an identifier and can be removed
identifier = parts.get(0);
parts.remove(0);
prefixParts.add(identifier);
}
}
if (!parts.isEmpty()) {
// outPath can be built by combining any remaining parts
outPath = String.join("/", parts);
}
}
String prefix = StringUtils.join(prefixParts, "/");
if (prefix != null && !prefix.isEmpty()) {
prefix = "/" + prefix;
}
ArbitraryDataRenderer renderer = new ArbitraryDataRenderer(name, ResourceIdType.NAME, service, identifier, outPath,
secret58, prefix, includeResourceIdInPrefix, async, qdnContext, request, response, context);
return renderer.render();
}
}

View File

@@ -0,0 +1,54 @@
package org.qortal.api.model;
import org.qortal.block.SelfSponsorshipAlgoV1Block;
import org.qortal.data.account.AccountData;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import java.util.ArrayList;
import java.util.List;
@XmlAccessorType(XmlAccessType.FIELD)
public class AccountPenaltyStats {
public Integer totalPenalties;
public Integer maxPenalty;
public Integer minPenalty;
public String penaltyHash;
protected AccountPenaltyStats() {
}
public AccountPenaltyStats(Integer totalPenalties, Integer maxPenalty, Integer minPenalty, String penaltyHash) {
this.totalPenalties = totalPenalties;
this.maxPenalty = maxPenalty;
this.minPenalty = minPenalty;
this.penaltyHash = penaltyHash;
}
public static AccountPenaltyStats fromAccounts(List<AccountData> accounts) {
int totalPenalties = 0;
Integer maxPenalty = null;
Integer minPenalty = null;
List<String> addresses = new ArrayList<>();
for (AccountData accountData : accounts) {
int penalty = accountData.getBlocksMintedPenalty();
addresses.add(accountData.getAddress());
totalPenalties++;
// Penalties are expressed as a negative number, so the min and the max are reversed here
if (maxPenalty == null || penalty < maxPenalty) maxPenalty = penalty;
if (minPenalty == null || penalty > minPenalty) minPenalty = penalty;
}
String penaltyHash = SelfSponsorshipAlgoV1Block.getHash(addresses);
return new AccountPenaltyStats(totalPenalties, maxPenalty, minPenalty, penaltyHash);
}
@Override
public String toString() {
return String.format("totalPenalties: %d, maxPenalty: %d, minPenalty: %d, penaltyHash: %s", totalPenalties, maxPenalty, minPenalty, penaltyHash == null ? "null" : penaltyHash);
}
}

View File

@@ -1,15 +1,14 @@
package org.qortal.api.model;
import java.util.Collections;
import java.util.EnumMap;
import java.util.Map;
import org.qortal.api.TransactionCountMapXmlAdapter;
import org.qortal.transaction.Transaction.TransactionType;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
import org.qortal.api.TransactionCountMapXmlAdapter;
import org.qortal.transaction.Transaction.TransactionType;
import java.util.Collections;
import java.util.EnumMap;
import java.util.Map;
@XmlAccessorType(XmlAccessType.FIELD)
public class ActivitySummary {

View File

@@ -1,13 +1,13 @@
package org.qortal.api.model;
import org.qortal.data.asset.OrderData;
import javax.xml.bind.Marshaller;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
import org.qortal.data.asset.OrderData;
@XmlAccessorType(XmlAccessType.NONE)
public class AggregatedOrder {

View File

@@ -1,7 +1,13 @@
package org.qortal.api.model;
import org.qortal.account.Account;
import org.qortal.repository.DataException;
import org.qortal.repository.RepositoryManager;
import org.qortal.repository.Repository;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlElement;
// All properties to be converted to JSON via JAXB
@XmlAccessorType(XmlAccessType.FIELD)
@@ -47,4 +53,31 @@ public class ApiOnlineAccount {
return this.recipientAddress;
}
public int getMinterLevelFromPublicKey() {
try (final Repository repository = RepositoryManager.getRepository()) {
return Account.getRewardShareEffectiveMintingLevel(repository, this.rewardSharePublicKey);
} catch (DataException e) {
return 0;
}
}
public boolean getIsMember() {
try (final Repository repository = RepositoryManager.getRepository()) {
return repository.getGroupRepository().memberExists(694, getMinterAddress());
} catch (DataException e) {
return false;
}
}
// JAXB special
@XmlElement(name = "minterLevel")
protected int getMinterLevel() {
return getMinterLevelFromPublicKey();
}
@XmlElement(name = "isMinterMember")
protected boolean getMinterMember() {
return getIsMember();
}
}

View File

@@ -0,0 +1,101 @@
package org.qortal.api.model;
import io.swagger.v3.oas.annotations.media.Schema;
import org.bouncycastle.util.encoders.Base64;
import org.bouncycastle.util.encoders.DecoderException;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlTransient;
@XmlAccessorType(XmlAccessType.FIELD)
public class AtCreationRequest {
@Schema(description = "CIYAM AT version", example = "2")
private short ciyamAtVersion;
@Schema(description = "base64-encoded code bytes", example = "")
private String codeBytesBase64;
@Schema(description = "base64-encoded data bytes", example = "")
private String dataBytesBase64;
private short numCallStackPages;
private short numUserStackPages;
private long minActivationAmount;
// Default constructor for JSON deserialization
public AtCreationRequest() {}
// Getters and setters
public short getCiyamAtVersion() {
return ciyamAtVersion;
}
public void setCiyamAtVersion(short ciyamAtVersion) {
this.ciyamAtVersion = ciyamAtVersion;
}
public String getCodeBytesBase64() {
return this.codeBytesBase64;
}
@XmlTransient
@Schema(hidden = true)
public byte[] getCodeBytes() {
if (this.codeBytesBase64 != null) {
try {
return Base64.decode(this.codeBytesBase64);
}
catch (DecoderException e) {
return null;
}
}
return null;
}
public String getDataBytesBase64() {
return this.dataBytesBase64;
}
@XmlTransient
@Schema(hidden = true)
public byte[] getDataBytes() {
if (this.dataBytesBase64 != null) {
try {
return Base64.decode(this.dataBytesBase64);
}
catch (DecoderException e) {
return null;
}
}
return null;
}
public short getNumCallStackPages() {
return numCallStackPages;
}
public void setNumCallStackPages(short numCallStackPages) {
this.numCallStackPages = numCallStackPages;
}
public short getNumUserStackPages() {
return numUserStackPages;
}
public void setNumUserStackPages(short numUserStackPages) {
this.numUserStackPages = numUserStackPages;
}
public long getMinActivationAmount() {
return minActivationAmount;
}
public void setMinActivationAmount(long minActivationAmount) {
this.minActivationAmount = minActivationAmount;
}
}

View File

@@ -0,0 +1,23 @@
package org.qortal.api.model;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import java.math.BigDecimal;
import java.math.BigInteger;
@XmlAccessorType(XmlAccessType.FIELD)
public class BlockMintingInfo {
public byte[] minterPublicKey;
public String minterAddress;
public int minterLevel;
public int onlineAccountsCount;
public BigDecimal maxDistance;
public BigInteger keyDistance;
public double keyDistanceRatio;
public long timestamp;
public long timeDelta;
public BlockMintingInfo() {
}
}

View File

@@ -1,10 +1,10 @@
package org.qortal.api.model;
import org.qortal.crypto.Crypto;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import org.qortal.crypto.Crypto;
@XmlAccessorType(XmlAccessType.FIELD)
public class BlockSignerSummary {

View File

@@ -1,61 +1,81 @@
package org.qortal.api.model;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import org.qortal.data.network.PeerChainTipData;
import io.swagger.v3.oas.annotations.media.Schema;
import org.qortal.controller.Controller;
import org.qortal.data.block.BlockSummaryData;
import org.qortal.data.network.PeerData;
import org.qortal.network.Handshake;
import org.qortal.network.Peer;
import io.swagger.v3.oas.annotations.media.Schema;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
@XmlAccessorType(XmlAccessType.FIELD)
public class ConnectedPeer {
public enum Direction {
INBOUND,
OUTBOUND;
}
public Direction direction;
public Handshake handshakeStatus;
public Long lastPing;
public Long connectedWhen;
public Long peersConnectedWhen;
public enum Direction {
INBOUND,
OUTBOUND
}
public String address;
public String version;
public Direction direction;
public Handshake handshakeStatus;
public Long lastPing;
public Long connectedWhen;
public Long peersConnectedWhen;
public String nodeId;
public String address;
public String version;
public Integer lastHeight;
@Schema(example = "base58")
public byte[] lastBlockSignature;
public Long lastBlockTimestamp;
public String nodeId;
protected ConnectedPeer() {
}
public Integer lastHeight;
@Schema(example = "base58")
public byte[] lastBlockSignature;
public Long lastBlockTimestamp;
public UUID connectionId;
public String age;
public Boolean isTooDivergent;
public ConnectedPeer(Peer peer) {
this.direction = peer.isOutbound() ? Direction.OUTBOUND : Direction.INBOUND;
this.handshakeStatus = peer.getHandshakeStatus();
this.lastPing = peer.getLastPing();
protected ConnectedPeer() {
}
PeerData peerData = peer.getPeerData();
this.connectedWhen = peer.getConnectionTimestamp();
this.peersConnectedWhen = peer.getPeersConnectionTimestamp();
public ConnectedPeer(Peer peer) {
this.direction = peer.isOutbound() ? Direction.OUTBOUND : Direction.INBOUND;
this.handshakeStatus = peer.getHandshakeStatus();
this.lastPing = peer.getLastPing();
this.address = peerData.getAddress().toString();
PeerData peerData = peer.getPeerData();
this.connectedWhen = peer.getConnectionTimestamp();
this.peersConnectedWhen = peer.getPeersConnectionTimestamp();
this.version = peer.getPeersVersionString();
this.nodeId = peer.getPeersNodeId();
this.address = peerData.getAddress().toString();
PeerChainTipData peerChainTipData = peer.getChainTipData();
if (peerChainTipData != null) {
this.lastHeight = peerChainTipData.getLastHeight();
this.lastBlockSignature = peerChainTipData.getLastBlockSignature();
this.lastBlockTimestamp = peerChainTipData.getLastBlockTimestamp();
}
}
this.version = peer.getPeersVersionString();
this.nodeId = peer.getPeersNodeId();
this.connectionId = peer.getPeerConnectionId();
if (peer.getConnectionEstablishedTime() > 0) {
long age = (System.currentTimeMillis() - peer.getConnectionEstablishedTime());
long minutes = TimeUnit.MILLISECONDS.toMinutes(age);
long seconds = TimeUnit.MILLISECONDS.toSeconds(age) - TimeUnit.MINUTES.toSeconds(minutes);
this.age = String.format("%dm %ds", minutes, seconds);
} else {
this.age = "connecting...";
}
BlockSummaryData peerChainTipData = peer.getChainTipData();
if (peerChainTipData != null) {
this.lastHeight = peerChainTipData.getHeight();
this.lastBlockSignature = peerChainTipData.getSignature();
this.lastBlockTimestamp = peerChainTipData.getTimestamp();
}
// Only include isTooDivergent decision if we've had the opportunity to request block summaries this peer
if (peer.getLastTooDivergentTime() != null) {
this.isTooDivergent = Controller.wasRecentlyTooDivergent.test(peer);
}
}
}

View File

@@ -1,31 +0,0 @@
package org.qortal.api.model;
import java.math.BigDecimal;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import io.swagger.v3.oas.annotations.media.Schema;
@XmlAccessorType(XmlAccessType.FIELD)
public class CrossChainBitcoinP2SHStatus {
@Schema(description = "Bitcoin P2SH address", example = "3CdH27kTpV8dcFHVRYjQ8EEV5FJg9X8pSJ (mainnet), 2fMiRRXVsxhZeyfum9ifybZvaMHbQTmwdZw (testnet)")
public String bitcoinP2shAddress;
@Schema(description = "Bitcoin P2SH balance")
public BigDecimal bitcoinP2shBalance;
@Schema(description = "Can P2SH redeem yet?")
public boolean canRedeem;
@Schema(description = "Can P2SH refund yet?")
public boolean canRefund;
@Schema(description = "Secret extracted by P2SH redeemer")
public byte[] secret;
public CrossChainBitcoinP2SHStatus() {
}
}

View File

@@ -1,11 +1,10 @@
package org.qortal.api.model;
import java.math.BigDecimal;
import io.swagger.v3.oas.annotations.media.Schema;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import io.swagger.v3.oas.annotations.media.Schema;
import java.math.BigDecimal;
@XmlAccessorType(XmlAccessType.FIELD)
public class CrossChainBitcoinRedeemRequest {

View File

@@ -1,11 +1,10 @@
package org.qortal.api.model;
import java.math.BigDecimal;
import io.swagger.v3.oas.annotations.media.Schema;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import io.swagger.v3.oas.annotations.media.Schema;
import java.math.BigDecimal;
@XmlAccessorType(XmlAccessType.FIELD)
public class CrossChainBitcoinRefundRequest {

View File

@@ -1,10 +1,10 @@
package org.qortal.api.model;
import io.swagger.v3.oas.annotations.media.Schema;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import io.swagger.v3.oas.annotations.media.Schema;
@XmlAccessorType(XmlAccessType.FIELD)
public class CrossChainBitcoinTemplateRequest {

View File

@@ -0,0 +1,30 @@
package org.qortal.api.model;
import io.swagger.v3.oas.annotations.media.Schema;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import java.math.BigDecimal;
@XmlAccessorType(XmlAccessType.FIELD)
public class CrossChainBitcoinyHTLCStatus {
@Schema(description = "P2SH address", example = "3CdH27kTpV8dcFHVRYjQ8EEV5FJg9X8pSJ (mainnet), 2fMiRRXVsxhZeyfum9ifybZvaMHbQTmwdZw (testnet)")
public String bitcoinP2shAddress;
@Schema(description = "P2SH balance")
public BigDecimal bitcoinP2shBalance;
@Schema(description = "Can HTLC redeem yet?")
public boolean canRedeem;
@Schema(description = "Can HTLC refund yet?")
public boolean canRefund;
@Schema(description = "Secret used by HTLC redeemer")
public byte[] secret;
public CrossChainBitcoinyHTLCStatus() {
}
}

View File

@@ -1,11 +1,11 @@
package org.qortal.api.model;
import io.swagger.v3.oas.annotations.media.Schema;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
import io.swagger.v3.oas.annotations.media.Schema;
@XmlAccessorType(XmlAccessType.FIELD)
public class CrossChainBuildRequest {

View File

@@ -1,10 +1,10 @@
package org.qortal.api.model;
import io.swagger.v3.oas.annotations.media.Schema;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import io.swagger.v3.oas.annotations.media.Schema;
@XmlAccessorType(XmlAccessType.FIELD)
public class CrossChainCancelRequest {

View File

@@ -0,0 +1,29 @@
package org.qortal.api.model;
import io.swagger.v3.oas.annotations.media.Schema;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
@XmlAccessorType(XmlAccessType.FIELD)
public class CrossChainDualSecretRequest {
@Schema(description = "Public key to match AT's trade 'partner'", example = "C6wuddsBV3HzRrXUtezE7P5MoRXp5m3mEDokRDGZB6ry")
public byte[] partnerPublicKey;
@Schema(description = "Qortal AT address")
public String atAddress;
@Schema(description = "secret-A (32 bytes)", example = "FHMzten4he9jZ4HGb4297Utj6F5g2w7serjq2EnAg2s1")
public byte[] secretA;
@Schema(description = "secret-B (32 bytes)", example = "EN2Bgx3BcEMtxFCewmCVSMkfZjVKYhx3KEXC5A21KBGx")
public byte[] secretB;
@Schema(description = "Qortal address for receiving QORT from AT")
public String receivingAddress;
public CrossChainDualSecretRequest() {
}
}

View File

@@ -1,14 +1,13 @@
package org.qortal.api.model;
import io.swagger.v3.oas.annotations.media.Schema;
import org.qortal.crosschain.AcctMode;
import org.qortal.data.crosschain.CrossChainTradeData;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
import org.qortal.crosschain.BTCACCT;
import org.qortal.data.crosschain.CrossChainTradeData;
import io.swagger.v3.oas.annotations.media.Schema;
// All properties to be converted to JSON via JAXB
@XmlAccessorType(XmlAccessType.FIELD)
public class CrossChainOfferSummary {
@@ -16,26 +15,41 @@ public class CrossChainOfferSummary {
// Properties
@Schema(description = "AT's Qortal address")
public String qortalAtAddress;
private String qortalAtAddress;
@Schema(description = "AT creator's Qortal address")
public String qortalCreator;
private String qortalCreator;
@Schema(description = "AT creator's ephemeral trading key-pair represented as Qortal address")
private String qortalCreatorTradeAddress;
@XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class)
private long qortAmount;
@Schema(description = "Bitcoin amount - DEPRECATED: use foreignAmount")
@XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class)
@Deprecated
private long btcAmount;
@Schema(description = "Foreign blockchain amount")
@XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class)
private long foreignAmount;
@Schema(description = "Suggested trade timeout (minutes)", example = "10080")
private int tradeTimeout;
private BTCACCT.Mode mode;
@Schema(description = "Current AT execution mode")
private AcctMode mode;
private long timestamp;
@Schema(description = "Trade partner's Qortal receiving address")
private String partnerQortalReceivingAddress;
private String foreignBlockchain;
private String acctName;
protected CrossChainOfferSummary() {
/* For JAXB */
}
@@ -43,12 +57,16 @@ public class CrossChainOfferSummary {
public CrossChainOfferSummary(CrossChainTradeData crossChainTradeData, long timestamp) {
this.qortalAtAddress = crossChainTradeData.qortalAtAddress;
this.qortalCreator = crossChainTradeData.qortalCreator;
this.qortalCreatorTradeAddress = crossChainTradeData.qortalCreatorTradeAddress;
this.qortAmount = crossChainTradeData.qortAmount;
this.btcAmount = crossChainTradeData.expectedBitcoin;
this.foreignAmount = crossChainTradeData.expectedForeignAmount;
this.btcAmount = this.foreignAmount; // Duplicate for deprecated field
this.tradeTimeout = crossChainTradeData.tradeTimeout;
this.mode = crossChainTradeData.mode;
this.timestamp = timestamp;
this.partnerQortalReceivingAddress = crossChainTradeData.qortalPartnerReceivingAddress;
this.foreignBlockchain = crossChainTradeData.foreignBlockchain;
this.acctName = crossChainTradeData.acctName;
}
public String getQortalAtAddress() {
@@ -59,6 +77,10 @@ public class CrossChainOfferSummary {
return this.qortalCreator;
}
public String getQortalCreatorTradeAddress() {
return this.qortalCreatorTradeAddress;
}
public long getQortAmount() {
return this.qortAmount;
}
@@ -67,11 +89,15 @@ public class CrossChainOfferSummary {
return this.btcAmount;
}
public long getForeignAmount() {
return this.foreignAmount;
}
public int getTradeTimeout() {
return this.tradeTimeout;
}
public BTCACCT.Mode getMode() {
public AcctMode getMode() {
return this.mode;
}
@@ -83,10 +109,18 @@ public class CrossChainOfferSummary {
return this.partnerQortalReceivingAddress;
}
public String getForeignBlockchain() {
return this.foreignBlockchain;
}
public String getAcctName() {
return this.acctName;
}
// For debugging mostly
public String toString() {
return String.format("%s: %s", this.qortalAtAddress, this.mode.name());
return String.format("%s: %s", this.qortalAtAddress, this.mode);
}
}

View File

@@ -1,24 +1,21 @@
package org.qortal.api.model;
import io.swagger.v3.oas.annotations.media.Schema;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import io.swagger.v3.oas.annotations.media.Schema;
@XmlAccessorType(XmlAccessType.FIELD)
public class CrossChainSecretRequest {
@Schema(description = "Public key to match AT's trade 'partner'", example = "C6wuddsBV3HzRrXUtezE7P5MoRXp5m3mEDokRDGZB6ry")
public byte[] partnerPublicKey;
@Schema(description = "Private key to match AT's trade 'partner'", example = "C6wuddsBV3HzRrXUtezE7P5MoRXp5m3mEDokRDGZB6ry")
public byte[] partnerPrivateKey;
@Schema(description = "Qortal AT address")
public String atAddress;
@Schema(description = "secret-A (32 bytes)", example = "FHMzten4he9jZ4HGb4297Utj6F5g2w7serjq2EnAg2s1")
public byte[] secretA;
@Schema(description = "secret-B (32 bytes)", example = "EN2Bgx3BcEMtxFCewmCVSMkfZjVKYhx3KEXC5A21KBGx")
public byte[] secretB;
@Schema(description = "Secret (32 bytes)", example = "FHMzten4he9jZ4HGb4297Utj6F5g2w7serjq2EnAg2s1")
public byte[] secret;
@Schema(description = "Qortal address for receiving QORT from AT")
public String receivingAddress;

View File

@@ -0,0 +1,72 @@
package org.qortal.api.model;
import io.swagger.v3.oas.annotations.media.Schema;
import org.qortal.data.crosschain.CrossChainTradeData;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
// All properties to be converted to JSON via JAXB
@XmlAccessorType(XmlAccessType.FIELD)
public class CrossChainTradeLedgerEntry {
private String market;
private String currency;
@XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class)
private long quantity;
@XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class)
private long feeAmount;
private String feeCurrency;
@XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class)
private long totalPrice;
private long tradeTimestamp;
protected CrossChainTradeLedgerEntry() {
/* For JAXB */
}
public CrossChainTradeLedgerEntry(String market, String currency, long quantity, long feeAmount, String feeCurrency, long totalPrice, long tradeTimestamp) {
this.market = market;
this.currency = currency;
this.quantity = quantity;
this.feeAmount = feeAmount;
this.feeCurrency = feeCurrency;
this.totalPrice = totalPrice;
this.tradeTimestamp = tradeTimestamp;
}
public String getMarket() {
return market;
}
public String getCurrency() {
return currency;
}
public long getQuantity() {
return quantity;
}
public long getFeeAmount() {
return feeAmount;
}
public String getFeeCurrency() {
return feeCurrency;
}
public long getTotalPrice() {
return totalPrice;
}
public long getTradeTimestamp() {
return tradeTimestamp;
}
}

View File

@@ -1,10 +1,10 @@
package org.qortal.api.model;
import io.swagger.v3.oas.annotations.media.Schema;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import io.swagger.v3.oas.annotations.media.Schema;
@XmlAccessorType(XmlAccessType.FIELD)
public class CrossChainTradeRequest {

View File

@@ -1,11 +1,12 @@
package org.qortal.api.model;
import io.swagger.v3.oas.annotations.media.Schema;
import org.qortal.data.crosschain.CrossChainTradeData;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
import org.qortal.data.crosschain.CrossChainTradeData;
// All properties to be converted to JSON via JAXB
@XmlAccessorType(XmlAccessType.FIELD)
public class CrossChainTradeSummary {
@@ -15,9 +16,20 @@ public class CrossChainTradeSummary {
@XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class)
private long qortAmount;
@Deprecated
@Schema(description = "DEPRECATED: use foreignAmount instead")
@XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class)
private long btcAmount;
@XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class)
private long foreignAmount;
private String atAddress;
private String sellerAddress;
private String buyerReceivingAddress;
protected CrossChainTradeSummary() {
/* For JAXB */
}
@@ -25,7 +37,11 @@ public class CrossChainTradeSummary {
public CrossChainTradeSummary(CrossChainTradeData crossChainTradeData, long timestamp) {
this.tradeTimestamp = timestamp;
this.qortAmount = crossChainTradeData.qortAmount;
this.btcAmount = crossChainTradeData.expectedBitcoin;
this.foreignAmount = crossChainTradeData.expectedForeignAmount;
this.btcAmount = this.foreignAmount;
this.sellerAddress = crossChainTradeData.qortalCreator;
this.buyerReceivingAddress = crossChainTradeData.qortalPartnerReceivingAddress;
this.atAddress = crossChainTradeData.qortalAtAddress;
}
public long getTradeTimestamp() {
@@ -40,4 +56,11 @@ public class CrossChainTradeSummary {
return this.btcAmount;
}
public long getForeignAmount() { return this.foreignAmount; }
public String getAtAddress() { return this.atAddress; }
public String getSellerAddress() { return this.sellerAddress; }
public String getBuyerReceivingAddressAddress() { return this.buyerReceivingAddress; }
}

View File

@@ -0,0 +1,50 @@
package org.qortal.api.model;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import java.util.Objects;
// All properties to be converted to JSON via JAXB
@XmlAccessorType(XmlAccessType.FIELD)
public class DatasetStatus {
private String name;
private long count;
public DatasetStatus() {}
public DatasetStatus(String name, long count) {
this.name = name;
this.count = count;
}
public String getName() {
return name;
}
public long getCount() {
return count;
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
DatasetStatus that = (DatasetStatus) o;
return count == that.count && Objects.equals(name, that.name);
}
@Override
public int hashCode() {
return Objects.hash(name, count);
}
@Override
public String toString() {
return "DatasetStatus{" +
"name='" + name + '\'' +
", count=" + count +
'}';
}
}

View File

@@ -0,0 +1,16 @@
package org.qortal.api.model;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
@XmlAccessorType(XmlAccessType.FIELD)
public class FileProperties {
public String filename;
public String mimeType;
public Long size;
public FileProperties() {
}
}

View File

@@ -1,12 +1,11 @@
package org.qortal.api.model;
import java.util.List;
import io.swagger.v3.oas.annotations.media.Schema;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlElement;
import io.swagger.v3.oas.annotations.media.Schema;
import java.util.List;
@Schema(description = "Group info, maybe including members")
// All properties to be converted to JSON via JAX-RS

View File

@@ -0,0 +1,18 @@
package org.qortal.api.model;
import io.swagger.v3.oas.annotations.media.Schema;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import java.util.List;
@XmlAccessorType(XmlAccessType.FIELD)
public class ListRequest {
@Schema(description = "A list of items")
public List<String> items;
public ListRequest() {
}
}

View File

@@ -1,11 +1,11 @@
package org.qortal.api.model;
import org.qortal.data.naming.NameData;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlElement;
import org.qortal.data.naming.NameData;
@XmlAccessorType(XmlAccessType.NONE)
public class NameSummary {

View File

@@ -12,6 +12,7 @@ public class NodeInfo {
public long buildTimestamp;
public String nodeId;
public boolean isTestNet;
public String type;
public NodeInfo() {
}

View File

@@ -1,11 +1,13 @@
package org.qortal.api.model;
import org.qortal.controller.Controller;
import org.qortal.controller.OnlineAccountsManager;
import org.qortal.controller.Synchronizer;
import org.qortal.network.Network;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import org.qortal.controller.Controller;
import org.qortal.network.Network;
@XmlAccessorType(XmlAccessType.FIELD)
public class NodeStatus {
@@ -20,12 +22,12 @@ public class NodeStatus {
public final int height;
public NodeStatus() {
this.isMintingPossible = Controller.getInstance().isMintingPossible();
this.isMintingPossible = OnlineAccountsManager.getInstance().hasActiveOnlineAccountSignatures();
this.syncPercent = Controller.getInstance().getSyncPercent();
this.isSynchronizing = this.syncPercent != null;
this.syncPercent = Synchronizer.getInstance().getSyncPercent();
this.isSynchronizing = Synchronizer.getInstance().isSynchronizing();
this.numberOfConnections = Network.getInstance().getHandshakedPeers().size();
this.numberOfConnections = Network.getInstance().getImmutableHandshakedPeers().size();
this.height = Controller.getInstance().getChainHeight();
}

View File

@@ -0,0 +1,15 @@
package org.qortal.api.model;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
@XmlAccessorType(XmlAccessType.FIELD)
public class PeersSummary {
public int inboundConnections;
public int outboundConnections;
public PeersSummary() {
}
}

View File

@@ -0,0 +1,83 @@
package org.qortal.api.model;
import io.swagger.v3.oas.annotations.media.Schema;
import org.qortal.data.voting.VoteOnPollData;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlElement;
import java.util.List;
@Schema(description = "Poll vote info, including voters")
// All properties to be converted to JSON via JAX-RS
@XmlAccessorType(XmlAccessType.FIELD)
public class PollVotes {
@Schema(description = "List of individual votes")
@XmlElement(name = "votes")
public List<VoteOnPollData> votes;
@Schema(description = "Total number of votes")
public Integer totalVotes;
@Schema(description = "Total weight of votes")
public Integer totalWeight;
@Schema(description = "List of vote counts for each option")
public List<OptionCount> voteCounts;
@Schema(description = "List of vote weights for each option")
public List<OptionWeight> voteWeights;
// For JAX-RS
protected PollVotes() {
}
public PollVotes(List<VoteOnPollData> votes, Integer totalVotes, Integer totalWeight, List<OptionCount> voteCounts, List<OptionWeight> voteWeights) {
this.votes = votes;
this.totalVotes = totalVotes;
this.totalWeight = totalWeight;
this.voteCounts = voteCounts;
this.voteWeights = voteWeights;
}
@Schema(description = "Vote info")
// All properties to be converted to JSON via JAX-RS
@XmlAccessorType(XmlAccessType.FIELD)
public static class OptionCount {
@Schema(description = "Option name")
public String optionName;
@Schema(description = "Vote count")
public Integer voteCount;
// For JAX-RS
protected OptionCount() {
}
public OptionCount(String optionName, Integer voteCount) {
this.optionName = optionName;
this.voteCount = voteCount;
}
}
@Schema(description = "Vote weights")
// All properties to be converted to JSON via JAX-RS
@XmlAccessorType(XmlAccessType.FIELD)
public static class OptionWeight {
@Schema(description = "Option name")
public String optionName;
@Schema(description = "Vote weight")
public Integer voteWeight;
// For JAX-RS
protected OptionWeight() {
}
public OptionWeight(String optionName, Integer voteWeight) {
this.optionName = optionName;
this.voteWeight = voteWeight;
}
}
}

View File

@@ -1,10 +1,10 @@
package org.qortal.api.model;
import io.swagger.v3.oas.annotations.media.Schema;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import io.swagger.v3.oas.annotations.media.Schema;
@XmlAccessorType(XmlAccessType.FIELD)
public class RewardShareKeyRequest {

View File

@@ -0,0 +1,156 @@
package org.qortal.api.model;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
@XmlAccessorType(XmlAccessType.FIELD)
public class SimpleForeignTransaction {
public static class AddressAmount {
public final String address;
public final long amount;
protected AddressAmount() {
/* For JAXB */
this.address = null;
this.amount = 0;
}
public AddressAmount(String address, long amount) {
this.address = address;
this.amount = amount;
}
}
private String txHash;
private long timestamp;
private List<AddressAmount> inputs;
public static class Output {
public final List<String> addresses;
public final long amount;
protected Output() {
/* For JAXB */
this.addresses = null;
this.amount = 0;
}
public Output(List<String> addresses, long amount) {
this.addresses = addresses;
this.amount = amount;
}
}
private List<Output> outputs;
private long totalAmount;
private long fees;
private Boolean isSentNotReceived;
protected SimpleForeignTransaction() {
/* For JAXB */
}
private SimpleForeignTransaction(Builder builder) {
this.txHash = builder.txHash;
this.timestamp = builder.timestamp;
this.inputs = Collections.unmodifiableList(builder.inputs);
this.outputs = Collections.unmodifiableList(builder.outputs);
Objects.requireNonNull(this.txHash);
if (timestamp <= 0)
throw new IllegalArgumentException("timestamp must be positive");
long totalGrossAmount = this.inputs.stream().map(addressAmount -> addressAmount.amount).reduce(0L, Long::sum);
this.totalAmount = this.outputs.stream().map(addressAmount -> addressAmount.amount).reduce(0L, Long::sum);
this.fees = totalGrossAmount - this.totalAmount;
this.isSentNotReceived = builder.isSentNotReceived;
}
public String getTxHash() {
return this.txHash;
}
public long getTimestamp() {
return this.timestamp;
}
public List<AddressAmount> getInputs() {
return this.inputs;
}
public List<Output> getOutputs() {
return this.outputs;
}
public long getTotalAmount() {
return this.totalAmount;
}
public long getFees() {
return this.fees;
}
public Boolean isSentNotReceived() {
return this.isSentNotReceived;
}
public static class Builder {
private String txHash;
private long timestamp;
private List<AddressAmount> inputs = new ArrayList<>();
private List<Output> outputs = new ArrayList<>();
private Boolean isSentNotReceived;
public Builder txHash(String txHash) {
this.txHash = Objects.requireNonNull(txHash);
return this;
}
public Builder timestamp(long timestamp) {
if (timestamp <= 0)
throw new IllegalArgumentException("timestamp must be positive");
this.timestamp = timestamp;
return this;
}
public Builder input(String address, long amount) {
Objects.requireNonNull(address);
if (amount < 0)
throw new IllegalArgumentException("amount must be zero or positive");
AddressAmount input = new AddressAmount(address, amount);
inputs.add(input);
return this;
}
public Builder output(List<String> addresses, long amount) {
Objects.requireNonNull(addresses);
if (amount < 0)
throw new IllegalArgumentException("amount must be zero or positive");
Output output = new Output(addresses, amount);
outputs.add(output);
return this;
}
public Builder isSentNotReceived(Boolean isSentNotReceived) {
this.isSentNotReceived = isSentNotReceived;
return this;
}
public SimpleForeignTransaction build() {
return new SimpleForeignTransaction(this);
}
}
}

Some files were not shown because too many files have changed in this diff Show More