Added encrypted/chunked publishing and streaming encrypted video playback. #3

Merged
crowetic merged 1 commits from encryptedVideo into main 2025-12-17 18:22:45 +00:00
5 changed files with 987 additions and 314 deletions
+151 -53
View File
@@ -1,4 +1,4 @@
import { useCallback, useEffect, useState } from 'react'; import { useCallback, useEffect, useRef, useState } from 'react';
import { Box, Card, CardContent, Typography, Divider, Skeleton, Chip, Button } from '@mui/material'; import { Box, Card, CardContent, Typography, Divider, Skeleton, Chip, Button } from '@mui/material';
import Grid from '@mui/material/Grid'; import Grid from '@mui/material/Grid';
import { useNavigate } from 'react-router-dom'; import { useNavigate } from 'react-router-dom';
@@ -197,7 +197,9 @@ export default function QAssetsNewsSection() {
const [announcements, setAnnouncements] = useState<NewsSummary[] | null>(null); const [announcements, setAnnouncements] = useState<NewsSummary[] | null>(null);
const [assetNews, setAssetNews] = useState<NewsSummary[] | null>(null); const [assetNews, setAssetNews] = useState<NewsSummary[] | null>(null);
const [promotions, setPromotions] = useState<NewsSummary[] | null>(null); const [promotions, setPromotions] = useState<NewsSummary[] | null>(null);
const [loading, setLoading] = useState(false); const [loadingAnnouncements, setLoadingAnnouncements] = useState(true);
const [loadingAssetNews, setLoadingAssetNews] = useState(true);
const [loadingPromotions, setLoadingPromotions] = useState(true);
const [showArchivedAnnouncements, setShowArchivedAnnouncements] = useState(false); const [showArchivedAnnouncements, setShowArchivedAnnouncements] = useState(false);
const [showArchivedNews, setShowArchivedNews] = useState(false); const [showArchivedNews, setShowArchivedNews] = useState(false);
@@ -212,52 +214,119 @@ export default function QAssetsNewsSection() {
const { memberGroupIds, loading: groupsLoading } = useMemberGroupIds(); const { memberGroupIds, loading: groupsLoading } = useMemberGroupIds();
const theme = useTheme(); const theme = useTheme();
const controllerRef = useRef<AbortController | null>(null);
const createAbortController = useCallback(() => {
controllerRef.current?.abort();
const ctrl = new AbortController();
controllerRef.current = ctrl;
return ctrl;
}, []);
const isSignalCurrent = useCallback(
(signal: AbortSignal) => controllerRef.current?.signal === signal,
[]
);
const isAbortError = (error: unknown) =>
Boolean(
error && typeof error === 'object' && (error as { name?: string }).name === 'AbortError'
);
// Initial load of lists // Initial load of lists
const loadNews = useCallback( const loadNews = useCallback(
async (forceFresh = false) => { (forceFresh = false) => {
const controller = createAbortController();
const signal = controller.signal;
setLoadingAnnouncements(true);
setLoadingAssetNews(true);
setLoadingPromotions(true);
if (forceFresh) { if (forceFresh) {
invalidateAnnouncementCache(); invalidateAnnouncementCache();
} }
try {
setLoading(true);
const announcementLimit = showMoreAnnouncements ? 50 : 5; const announcementLimit = showMoreAnnouncements ? 50 : 5;
const assetNewsLimit = showMoreNews ? 50 : 8; const assetNewsLimit = showMoreNews ? 50 : 8;
const [a, n, p] = await Promise.all([
fetchAnnouncements(announcementLimit, { const loadAnnouncements = async () => {
try {
const results = await fetchAnnouncements(announcementLimit, {
includeExpired: showArchivedAnnouncements, includeExpired: showArchivedAnnouncements,
forceFresh, forceFresh,
}), signal,
fetchLatestAssetNews(assetNewsLimit, { });
if (!isSignalCurrent(signal)) return;
setAnnouncements(results);
} catch (err) {
if (isAbortError(err)) return;
console.error('Failed to load Q-Assets announcements', err);
if (isSignalCurrent(signal)) {
setAnnouncements([]);
}
} finally {
if (isSignalCurrent(signal)) {
setLoadingAnnouncements(false);
}
}
};
const loadAssetNews = async () => {
try {
const results = await fetchLatestAssetNews(assetNewsLimit, {
includeExpired: showArchivedNews, includeExpired: showArchivedNews,
allowedGroupIds: memberGroupIds, allowedGroupIds: memberGroupIds,
}), signal,
fetchActivePromotions(), });
]); if (!isSignalCurrent(signal)) return;
setAnnouncements(a); setAssetNews(results);
setAssetNews(n); } catch (err) {
setPromotions(p); if (isAbortError(err)) return;
} catch (e) { console.error('Failed to load asset news publications', err);
console.error('Failed to load Q-Assets news', e); if (isSignalCurrent(signal)) {
setAnnouncements([]);
setAssetNews([]); setAssetNews([]);
setPromotions([]);
} finally {
setLoading(false);
} }
} finally {
if (isSignalCurrent(signal)) {
setLoadingAssetNews(false);
}
}
};
const loadPromotions = async () => {
try {
const results = await fetchActivePromotions(Date.now(), { signal });
if (!isSignalCurrent(signal)) return;
setPromotions(results);
} catch (err) {
if (isAbortError(err)) return;
console.error('Failed to load promotions', err);
if (isSignalCurrent(signal)) {
setPromotions([]);
}
} finally {
if (isSignalCurrent(signal)) {
setLoadingPromotions(false);
}
}
};
void loadAnnouncements();
void loadAssetNews();
void loadPromotions();
}, },
[ [
createAbortController,
isSignalCurrent,
memberGroupIds,
showArchivedAnnouncements, showArchivedAnnouncements,
showArchivedNews, showArchivedNews,
showMoreAnnouncements, showMoreAnnouncements,
showMoreNews, showMoreNews,
memberGroupIds,
] ]
); );
const loadingLists =
loading || groupsLoading || announcements === null || assetNews === null || promotions === null;
const handleClickItem = (item: NewsSummary) => { const handleClickItem = (item: NewsSummary) => {
setSelected(item); setSelected(item);
}; };
@@ -269,27 +338,26 @@ export default function QAssetsNewsSection() {
}; };
useEffect(() => { useEffect(() => {
let cancelled = false; if (groupsLoading) return;
(async () => { loadNews(true);
if (cancelled) return;
await loadNews(true);
})();
return () => { return () => {
cancelled = true; controllerRef.current?.abort();
controllerRef.current = null;
}; };
}, [loadNews]); }, [groupsLoading, loadNews]);
useEffect(() => { useEffect(() => {
const handler = () => { const handler = () => {
if (groupsLoading) return;
loadNews(true); loadNews(true);
}; };
window.addEventListener(NEWS_REFRESH_EVENT, handler); window.addEventListener(NEWS_REFRESH_EVENT, handler);
return () => { return () => {
window.removeEventListener(NEWS_REFRESH_EVENT, handler); window.removeEventListener(NEWS_REFRESH_EVENT, handler);
}; };
}, [loadNews]); }, [groupsLoading, loadNews]);
const announcementList = announcements || []; const announcementList = announcements ?? [];
const announcementActive = announcementList.filter((item) => !item.isExpired); const announcementActive = announcementList.filter((item) => !item.isExpired);
const announcementArchived = announcementList.filter((item) => item.isExpired); const announcementArchived = announcementList.filter((item) => item.isExpired);
@@ -305,6 +373,32 @@ export default function QAssetsNewsSection() {
showMoreNews ? Number.MAX_SAFE_INTEGER : maxPerList showMoreNews ? Number.MAX_SAFE_INTEGER : maxPerList
); );
const promotionsList = promotions ?? [];
const showAnnouncementSkeleton = loadingAnnouncements && announcements === null;
const showAssetNewsSkeleton = loadingAssetNews && assetNews === null;
const showPromotionsSkeleton = loadingPromotions && promotions === null;
const ColumnSkeleton = () => (
<Card
sx={{
height: '100%',
borderRadius: 3,
display: 'flex',
flexDirection: 'column',
justifyContent: 'center',
boxShadow: '0 8px 24px rgba(0,0,0,0.08)',
border: `1px solid ${alpha(theme.palette.common.white, 0.08)}`,
}}
>
<CardContent>
<Skeleton variant="text" width="60%" sx={{ mb: 1 }} />
<Skeleton variant="text" width="90%" />
<Skeleton variant="text" width="80%" />
</CardContent>
</Card>
);
// Helper for label // Helper for label
const typeLabel = (type: NewsType | undefined) => { const typeLabel = (type: NewsType | undefined) => {
if (!type) return ''; if (!type) return '';
@@ -453,23 +547,12 @@ export default function QAssetsNewsSection() {
Announcements from Q-Assets, latest news from all issuers, and paid promotional content. Announcements from Q-Assets, latest news from all issuers, and paid promotional content.
</Typography> </Typography>
{loadingLists ? (
<Grid container spacing={2}>
{[0, 1, 2].map((i) => (
<Grid key={i} size={{ xs: 12, md: 4 }}>
<Card>
<CardContent>
<Skeleton variant="text" width="60%" />
<Skeleton variant="text" width="90%" />
<Skeleton variant="text" width="80%" />
</CardContent>
</Card>
</Grid>
))}
</Grid>
) : (
<Grid container spacing={2}> <Grid container spacing={2}>
<Grid size={{ xs: 12, md: 4 }}> <Grid size={{ xs: 12, md: 4 }}>
{showAnnouncementSkeleton ? (
<ColumnSkeleton />
) : (
<>
<NewsListColumn <NewsListColumn
title="Q-Assets Announcements" title="Q-Assets Announcements"
items={visibleAnnouncements} items={visibleAnnouncements}
@@ -504,8 +587,14 @@ export default function QAssetsNewsSection() {
</Button> </Button>
)} )}
</Box> </Box>
</>
)}
</Grid> </Grid>
<Grid size={{ xs: 12, md: 4 }}> <Grid size={{ xs: 12, md: 4 }}>
{showAssetNewsSkeleton ? (
<ColumnSkeleton />
) : (
<>
<NewsListColumn <NewsListColumn
title="Asset News Publications" title="Asset News Publications"
items={visibleNews} items={visibleNews}
@@ -519,7 +608,11 @@ export default function QAssetsNewsSection() {
/> />
<Box sx={{ display: 'flex', gap: 1, flexWrap: 'wrap', mt: 1 }}> <Box sx={{ display: 'flex', gap: 1, flexWrap: 'wrap', mt: 1 }}>
{(showArchivedNews ? newsArchived : newsActive).length > maxPerList && ( {(showArchivedNews ? newsArchived : newsActive).length > maxPerList && (
<Button size="small" onClick={() => setShowMoreNews((v) => !v)} variant="outlined"> <Button
size="small"
onClick={() => setShowMoreNews((v) => !v)}
variant="outlined"
>
{showMoreNews ? 'Show less' : 'Show more'} {showMoreNews ? 'Show less' : 'Show more'}
</Button> </Button>
)} )}
@@ -535,18 +628,23 @@ export default function QAssetsNewsSection() {
</Button> </Button>
)} )}
</Box> </Box>
</>
)}
</Grid> </Grid>
<Grid size={{ xs: 12, md: 4 }}> <Grid size={{ xs: 12, md: 4 }}>
{showPromotionsSkeleton ? (
<ColumnSkeleton />
) : (
<NewsListColumn <NewsListColumn
title="Promotions" title="Promotions"
items={promotions ?? []} items={promotionsList}
emptyText="No active promotions." emptyText="No active promotions."
onClickItem={handleClickItem} onClickItem={handleClickItem}
variant="promotion" variant="promotion"
/> />
</Grid>
</Grid>
)} )}
</Grid>
</Grid>
<NewsActionBar <NewsActionBar
treasuryAddress="Q-Assets" // TODO put real address for the treasury account. We do not want to utilize Q-Assets. treasuryAddress="Q-Assets" // TODO put real address for the treasury account. We do not want to utilize Q-Assets.
defaultPromoPriceQort={5} defaultPromoPriceQort={5}
+471 -83
View File
@@ -49,6 +49,7 @@ import {
objectToBase64, objectToBase64,
base64ToUtf8, base64ToUtf8,
base64ToUint8Array, base64ToUint8Array,
uint8ArrayToBase64,
} from '../../../utils/data'; } from '../../../utils/data';
import { uniqueId6 } from '../../../utils/ids'; import { uniqueId6 } from '../../../utils/ids';
import { stripPrivateMagic, PRIVATE_MAGIC_B64 } from '../../../constants/qdeckIdentifiers'; import { stripPrivateMagic, PRIVATE_MAGIC_B64 } from '../../../constants/qdeckIdentifiers';
@@ -83,6 +84,16 @@ import {
getEncryptionInfo, getEncryptionInfo,
resourceIsPrivate, resourceIsPrivate,
} from '../../../utils/qdnEncryption'; } from '../../../utils/qdnEncryption';
import {
MAX_INLINE_FILE_SIZE,
DEFAULT_CHUNK_SIZE,
iterateFileChunks,
buildChunkIdentifier,
createChunkedManifest,
FileChunkDescriptor,
ChunkedFileManifest,
CHUNK_FORCED_THRESHOLD,
} from '../../../utils/fileChunking';
import type { import type {
FolderDescriptor, FolderDescriptor,
FolderNode, FolderNode,
@@ -124,6 +135,8 @@ type PreviewDialogState = {
resource?: QdnResource | null; resource?: QdnResource | null;
zoomed?: boolean; zoomed?: boolean;
expanded?: boolean; expanded?: boolean;
videoUrl?: string;
chunked?: boolean;
}; };
const PREVIEW_STEPS: PreviewStep[] = [ const PREVIEW_STEPS: PreviewStep[] = [
@@ -140,6 +153,8 @@ const createPreviewDialogState = (): PreviewDialogState => ({
zoomed: false, zoomed: false,
expanded: false, expanded: false,
loading: false, loading: false,
videoUrl: undefined,
chunked: false,
}); });
const PUBLISH_MODE_STORAGE_KEY = 'qassets_publish_mode_preference_v1'; const PUBLISH_MODE_STORAGE_KEY = 'qassets_publish_mode_preference_v1';
@@ -150,6 +165,7 @@ const PENDING_FOLDERS_KEY = 'qassets_data_pending_folders_v1';
const PUBLISH_CHUNK_SIZE = 25; const PUBLISH_CHUNK_SIZE = 25;
const MANIFEST_REFRESH_COOLDOWN = 90 * 1000; const MANIFEST_REFRESH_COOLDOWN = 90 * 1000;
const MAX_FILE_IDENTIFIER_LENGTH = QASSETS_FILE_ID_MAX; const MAX_FILE_IDENTIFIER_LENGTH = QASSETS_FILE_ID_MAX;
const CHUNK_METADATA_TAG = 'qassets-chunk';
const MANIFEST_SERVICE = ensurePrivateService('DOCUMENT_PRIVATE'); const MANIFEST_SERVICE = ensurePrivateService('DOCUMENT_PRIVATE');
type ResourceSort = type ResourceSort =
| 'name-asc' | 'name-asc'
@@ -455,6 +471,7 @@ type LoadedResourceContent = {
key: string; key: string;
base64: string; base64: string;
mime: string; mime: string;
chunkedManifest?: ChunkedFileManifest;
}; };
const normalizeData64 = (payload: any): string | null => { const normalizeData64 = (payload: any): string | null => {
@@ -853,6 +870,32 @@ export default function DataExplorer() {
createPreviewDialogState() createPreviewDialogState()
); );
const previewVideoRef = useRef<HTMLVideoElement | null>(null); const previewVideoRef = useRef<HTMLVideoElement | null>(null);
const previewChunkedBlobUrlRef = useRef<string | null>(null);
const chunkedMediaSourceRef = useRef<{ objectUrl: string; cancel: () => void } | null>(null);
const setPreviewChunkedBlobUrl = useCallback((url?: string) => {
if (previewChunkedBlobUrlRef.current && previewChunkedBlobUrlRef.current !== url) {
URL.revokeObjectURL(previewChunkedBlobUrlRef.current);
}
previewChunkedBlobUrlRef.current = url ?? null;
}, []);
const cleanupChunkedBlobUrl = useCallback(() => {
if (previewChunkedBlobUrlRef.current) {
URL.revokeObjectURL(previewChunkedBlobUrlRef.current);
previewChunkedBlobUrlRef.current = null;
}
}, []);
const cleanupChunkedMediaSource = useCallback(() => {
const entry = chunkedMediaSourceRef.current;
chunkedMediaSourceRef.current = null;
if (entry) {
entry.cancel();
URL.revokeObjectURL(entry.objectUrl);
}
}, []);
const cleanupChunkedVideoPreview = useCallback(() => {
cleanupChunkedMediaSource();
cleanupChunkedBlobUrl();
}, [cleanupChunkedBlobUrl, cleanupChunkedMediaSource]);
const [manifestDialog, setManifestDialog] = useState<{ const [manifestDialog, setManifestDialog] = useState<{
open: boolean; open: boolean;
entry: StructuredEntry | null; entry: StructuredEntry | null;
@@ -1023,6 +1066,19 @@ export default function DataExplorer() {
base64, base64,
mime: inferredMime, mime: inferredMime,
}; };
if ((resource.metadata?.qassetsFs as any)?.chunked) {
try {
const manifest = JSON.parse(base64ToUtf8(base64)) as ChunkedFileManifest;
if (manifest && Array.isArray(manifest.chunks)) {
entry.chunkedManifest = manifest;
if (manifest.mimeType) {
entry.mime = manifest.mimeType;
}
}
} catch {
// ignore malformed chunk manifest
}
}
setLoadedContent(entry); setLoadedContent(entry);
setDetectedTypes((prev) => setDetectedTypes((prev) =>
prev[resource.identifier] === inferredMime prev[resource.identifier] === inferredMime
@@ -1445,6 +1501,112 @@ export default function DataExplorer() {
return map; return map;
}, [combinedResources]); }, [combinedResources]);
const createChunkedBlobUrl = useCallback(
async (manifest: ChunkedFileManifest) => {
if (!manifest.chunks?.length) {
throw new Error('Chunk manifest is empty.');
}
const buffers: Uint8Array[] = [];
for (const chunk of manifest.chunks) {
const chunkResource = combinedResourceMap.get(chunk.identifier);
if (!chunkResource) {
throw new Error(`Chunk ${chunk.identifier} is not available yet.`);
}
const chunkBase64 = await resolveResourceBase64(chunkResource);
buffers.push(base64ToUint8Array(chunkBase64));
}
const blob = new Blob(buffers, {
type: manifest.mimeType || 'application/octet-stream',
});
return URL.createObjectURL(blob);
},
[combinedResourceMap, resolveResourceBase64]
);
const streamChunkedVideo = useCallback(
(manifest: ChunkedFileManifest) => {
if (!manifest.chunks?.length) {
throw new Error('Chunk manifest is empty.');
}
if (typeof window === 'undefined' || typeof MediaSource === 'undefined') {
throw new Error('Chunk streaming is not supported in this environment.');
}
cleanupChunkedMediaSource();
const mediaSource = new MediaSource();
const objectUrl = URL.createObjectURL(mediaSource);
const sortedChunks = [...manifest.chunks].sort((a, b) => a.index - b.index);
let resolved = false;
let rejectStream: ((reason?: any) => void) | null = null;
const promise = new Promise<void>((resolve, reject) => {
rejectStream = reject;
const handleSourceOpen = () => {
mediaSource.removeEventListener('sourceopen', handleSourceOpen);
let sourceBuffer: SourceBuffer;
try {
sourceBuffer = mediaSource.addSourceBuffer(manifest.mimeType || 'video/mp4');
} catch (err) {
reject(err);
return;
}
let nextIndex = 0;
const appendNextChunk = async () => {
if (resolved) return;
if (nextIndex >= sortedChunks.length) {
resolved = true;
if (mediaSource.readyState === 'open') {
try {
mediaSource.endOfStream();
} catch (e) {
console.log(e);
}
}
resolve();
return;
}
if (sourceBuffer.updating) {
sourceBuffer.addEventListener('updateend', appendNextChunk, { once: true });
return;
}
const chunkMeta = sortedChunks[nextIndex];
const chunkResource = combinedResourceMap.get(chunkMeta.identifier);
if (!chunkResource) {
reject(new Error(`Chunk ${chunkMeta.identifier} is not available yet.`));
return;
}
try {
const chunkBase64 = await resolveResourceBase64(chunkResource);
const chunkData = base64ToUint8Array(chunkBase64);
sourceBuffer.appendBuffer(chunkData);
nextIndex += 1;
sourceBuffer.addEventListener('updateend', appendNextChunk, { once: true });
} catch (err) {
reject(err);
}
};
appendNextChunk();
};
mediaSource.addEventListener('sourceopen', handleSourceOpen);
});
const cancel = () => {
if (resolved) return;
resolved = true;
if (rejectStream) {
rejectStream(new Error('Chunk streaming cancelled.'));
}
if (mediaSource.readyState === 'open') {
try {
mediaSource.endOfStream();
} catch (e) {
console.log(e);
}
}
};
chunkedMediaSourceRef.current = { objectUrl, cancel };
return { objectUrl, promise };
},
[cleanupChunkedMediaSource, combinedResourceMap, resolveResourceBase64]
);
const resourceStructuredEntries = useMemo( const resourceStructuredEntries = useMemo(
() => () =>
combinedResources combinedResources
@@ -2319,8 +2481,8 @@ export default function DataExplorer() {
groupId, groupId,
groupAdminsOnly, groupAdminsOnly,
directRecipients, directRecipients,
chunkedPublishing,
}: PublishSubmitPayload) => { }: PublishSubmitPayload) => {
console.log('files', files);
if (!activeName) { if (!activeName) {
setPublishStatus('Select a Qortal name before publishing.'); setPublishStatus('Select a Qortal name before publishing.');
return; return;
@@ -2331,18 +2493,104 @@ export default function DataExplorer() {
} }
const normalizedFolder = normalizePathSegments(form.folderPath).join('/'); const normalizedFolder = normalizePathSegments(form.folderPath).join('/');
const hasForcedChunking =
encryptionMode !== 'none' && files.some((file) => file.size > CHUNK_FORCED_THRESHOLD);
const requiresChunkGroup = chunkedPublishing && encryptionMode !== 'none';
const needsForcedGroup = hasForcedChunking;
if (requiresChunkGroup && encryptionMode !== 'group') {
setPublishStatus('Chunked publishing requires group encryption.');
return;
}
if ((requiresChunkGroup || needsForcedGroup) && !groupId) {
setPublishStatus('Select a private group for chunked publishing.');
return;
}
setPublishing(true); setPublishing(true);
setPublishStatus(null); setPublishStatus(null);
const baseId = sanitizeIdentifier(form.identifier || ''); const baseId = sanitizeIdentifier(form.identifier || '');
let success = false; let success = false;
try { try {
const publisherAddress = await resolvePublisherAddress();
const directRecipientList = parseRecipientList(directRecipients);
let directPublicKeys: string[] = [];
if (encryptionMode === 'direct') {
if (!directRecipientList.length) {
alert('No recipients specified. Files will be encrypted for you only.');
}
const { publicKeys } = await collectRecipientPublicKeys({
usersAllowed: directRecipientList,
includeSelf: true,
me: {
name: activeName || authName || undefined,
address: publisherAddress,
},
});
directPublicKeys = publicKeys;
if (!directPublicKeys.length) {
throw new Error('No recipient public keys resolved.');
}
}
const encryptPayload = async (
payload64: string
): Promise<{
base64: string;
service: Service;
tagExtra: string[];
privateMode?: 'group' | 'direct';
}> => {
if (encryptionMode === 'none') {
return {
base64: payload64,
service: form.service,
tagExtra: [],
};
}
if (encryptionMode === 'group') {
if (!groupId) throw new Error('Select a group for encryption.');
const enc64Group = await qortalRequest({
action: 'ENCRYPT_QORTAL_GROUP_DATA',
base64: payload64,
groupId,
isAdmins: groupAdminsOnly,
});
const tagExtra = buildEncryptionTagSet({
mode: 'group',
publisher: publisherAddress,
groupId,
adminsOnly: groupAdminsOnly,
});
return {
base64: enc64Group,
service: resolveServiceForEncryptionMode(form.service, 'group'),
tagExtra,
privateMode: 'group',
};
}
const enc64Direct = await qortalRequest({
action: 'ENCRYPT_DATA',
base64: payload64,
publicKeys: directPublicKeys,
});
const tagExtra = buildEncryptionTagSet({
mode: 'direct',
publisher: publisherAddress,
userCount: directPublicKeys.length || directRecipientList.length || 1,
});
return {
base64: enc64Direct,
service: resolveServiceForEncryptionMode(form.service, 'direct'),
tagExtra,
privateMode: 'direct',
};
};
const publishRequests: BatchPublishResource[] = []; const publishRequests: BatchPublishResource[] = [];
const chunkSize = DEFAULT_CHUNK_SIZE;
for (let i = 0; i < files.length; i += 1) { for (let i = 0; i < files.length; i += 1) {
const file = files[i]; const file = files[i];
console.log('incoming file[i]', file);
const file64 = await fileToBase64(file);
console.log('base64 from file (first 100):', file64.slice(0, 100));
const title = form.title || file.name; const title = form.title || file.name;
const description = form.description || `Published via Q-Assets Data Explorer`; const description = form.description || `Published via Q-Assets Data Explorer`;
let tags: string[] = []; let tags: string[] = [];
@@ -2361,81 +2609,6 @@ export default function DataExplorer() {
metadata.title = file.name; metadata.title = file.name;
} }
const applyEncryption = async (
file64: string
): Promise<{
base64: string;
service: Service;
tagExtra: string[];
privateMode?: 'group' | 'direct';
}> => {
if (encryptionMode === 'none') {
return {
base64: file64,
service: form.service,
tagExtra: [],
};
}
const publisherAddress = await resolvePublisherAddress();
if (encryptionMode === 'group') {
if (!groupId) throw new Error('Select a group for encryption.');
const enc64Group = await qortalRequest({
action: 'ENCRYPT_QORTAL_GROUP_DATA',
base64: file64,
groupId,
isAdmins: groupAdminsOnly,
});
// const privData64 = applyPrivateMagicIfNeeded(enc);
const tagExtra = buildEncryptionTagSet({
mode: 'group',
publisher: publisherAddress,
groupId,
adminsOnly: groupAdminsOnly,
});
return {
base64: enc64Group,
service: resolveServiceForEncryptionMode(form.service, 'group'),
tagExtra,
privateMode: 'group',
};
}
const recipients = parseRecipientList(directRecipients);
if (!recipients.length) {
alert('No recipients specified. Files will be encrypted for you only.');
}
const { publicKeys } = await collectRecipientPublicKeys({
usersAllowed: recipients,
includeSelf: true,
me: { name: activeName || authName || undefined, address: publisherAddress },
});
if (!publicKeys.length) throw new Error('No recipient public keys resolved.');
const enc64Direct = await qortalRequest({
action: 'ENCRYPT_DATA',
base64: file64,
publicKeys,
});
const tagExtra = buildEncryptionTagSet({
mode: 'direct',
publisher: publisherAddress,
userCount: publicKeys.length || recipients.length || 1,
});
return {
base64: enc64Direct,
service: resolveServiceForEncryptionMode(form.service, 'direct'),
tagExtra,
privateMode: 'direct',
};
};
const {
base64: finalData64,
service: finalService,
tagExtra,
privateMode,
} = await applyEncryption(file64);
if (tagExtra.length) {
tags = tagExtra.concat(tags);
}
let identifier: string; let identifier: string;
if (baseId) { if (baseId) {
const suffix = publishDialog.variant === 'multiple' ? `-${i + 1}-${uniqueId6()}` : ''; const suffix = publishDialog.variant === 'multiple' ? `-${i + 1}-${uniqueId6()}` : '';
@@ -2444,9 +2617,22 @@ export default function DataExplorer() {
? `${baseId}${suffix}`.slice(0, MAX_FILE_IDENTIFIER_LENGTH) ? `${baseId}${suffix}`.slice(0, MAX_FILE_IDENTIFIER_LENGTH)
: baseId.slice(0, MAX_FILE_IDENTIFIER_LENGTH); : baseId.slice(0, MAX_FILE_IDENTIFIER_LENGTH);
} else { } else {
identifier = buildQassetsFileIdentifier(finalService as Service, activeName); identifier = buildQassetsFileIdentifier(form.service as Service, activeName);
} }
const shouldChunk =
encryptionMode !== 'none' &&
(file.size > CHUNK_FORCED_THRESHOLD ||
(chunkedPublishing && file.size > MAX_INLINE_FILE_SIZE));
if (!shouldChunk) {
const file64 = await fileToBase64(file);
const {
base64: finalData64,
service: finalService,
tagExtra,
privateMode,
} = await encryptPayload(file64);
const finalTags = tagExtra.length ? tagExtra.concat(tags) : tags;
publishRequests.push({ publishRequests.push({
name: activeName, name: activeName,
service: finalService, service: finalService,
@@ -2454,11 +2640,97 @@ export default function DataExplorer() {
base64: finalData64, base64: finalData64,
title, title,
description, description,
tags, tags: finalTags,
metadata, metadata,
// disableEncrypt: encryptionMode !== 'none',
privateMode, privateMode,
}); });
continue;
}
const chunkDescriptors: FileChunkDescriptor[] = [];
const chunkPublishRequests: BatchPublishResource[] = [];
for await (const chunk of iterateFileChunks(file, chunkSize)) {
const chunk64 = uint8ArrayToBase64(chunk.uint8);
const chunkIndexId = buildChunkIdentifier(identifier, chunk.index).slice(
0,
MAX_FILE_IDENTIFIER_LENGTH
);
const {
base64: chunkData64,
service: chunkService,
tagExtra,
privateMode,
} = await encryptPayload(chunk64);
chunkDescriptors.push({
index: chunk.index,
identifier: chunkIndexId,
size: chunk.size,
});
const chunkTags = Array.from(new Set([...tagExtra, CHUNK_METADATA_TAG]));
chunkPublishRequests.push({
name: activeName,
service: chunkService,
identifier: chunkIndexId,
base64: chunkData64,
title: `${title} (chunk ${chunk.index + 1})`,
description,
tags: chunkTags,
metadata: {
qassetsChunk: {
parentIdentifier: identifier,
index: chunk.index,
chunked: true,
},
},
privateMode,
});
}
publishRequests.push(...chunkPublishRequests);
const chunkEncryptionInfo: ChunkedFileManifest['encryption'] | undefined =
encryptionMode === 'group'
? { mode: 'group', groupId: groupId ?? undefined, adminsOnly: groupAdminsOnly }
: {
mode: 'direct',
recipientCount: directPublicKeys.length || directRecipientList.length || 1,
};
const manifest = createChunkedManifest(
file,
chunkSize,
chunkDescriptors,
chunkEncryptionInfo
);
const manifestJson = await objectToBase64(manifest);
const {
base64: manifestData64,
service: manifestService,
tagExtra: manifestTagExtra,
privateMode: manifestPrivateMode,
} = await encryptPayload(manifestJson);
const manifestTags = Array.from(new Set([...manifestTagExtra, ...tags]));
const manifestMetadata = {
...metadata,
qassetsFs: {
...(metadata.qassetsFs || {}),
chunked: true,
chunkManifestId: identifier,
chunkCount: chunkDescriptors.length,
chunkSize,
},
};
publishRequests.push({
name: activeName,
service: manifestService,
identifier,
base64: manifestData64,
title,
description,
tags: manifestTags,
metadata: manifestMetadata,
privateMode: manifestPrivateMode,
});
} }
await publishResources(publishRequests); await publishResources(publishRequests);
@@ -2595,6 +2867,7 @@ export default function DataExplorer() {
if (resourceArg && resourceArg.identifier !== selectedResourceId) { if (resourceArg && resourceArg.identifier !== selectedResourceId) {
setSelectedResourceId(resourceArg.identifier); setSelectedResourceId(resourceArg.identifier);
} }
cleanupChunkedVideoPreview();
const initialSteps = clonePreviewSteps(); const initialSteps = clonePreviewSteps();
setPreviewDialog({ setPreviewDialog({
open: true, open: true,
@@ -2604,6 +2877,8 @@ export default function DataExplorer() {
resource: target, resource: target,
zoomed: false, zoomed: false,
expanded: false, expanded: false,
videoUrl: undefined,
chunked: false,
}); });
const updateStep = (key: PreviewStepKey, status: PreviewStepStatus, message?: string) => { const updateStep = (key: PreviewStepKey, status: PreviewStepStatus, message?: string) => {
setPreviewDialog((prev) => ({ setPreviewDialog((prev) => ({
@@ -2613,6 +2888,88 @@ export default function DataExplorer() {
}; };
try { try {
const loaded = await ensureResourceContent(target, { onStep: updateStep }); const loaded = await ensureResourceContent(target, { onStep: updateStep });
if (loaded.chunkedManifest) {
const chunkManifest = loaded.chunkedManifest!;
const chunkedMime = chunkManifest.mimeType || loaded.mime;
if (chunkedMime.startsWith('video/')) {
updateStep('analyze', 'active', 'Streaming video from chunks…');
try {
const { objectUrl, promise } = streamChunkedVideo(chunkManifest);
setPreviewDialog((prev) => ({
...prev,
open: true,
loading: false,
title: getResourceLabel(target),
type: 'video',
resource: target,
zoomed: false,
chunked: true,
videoUrl: objectUrl,
error: undefined,
}));
void promise.catch(async (streamError) => {
updateStep('analyze', 'error', streamError?.message);
cleanupChunkedMediaSource();
try {
const fallbackUrl = await createChunkedBlobUrl(chunkManifest);
setPreviewChunkedBlobUrl(fallbackUrl);
setPreviewDialog((prev) => ({
...prev,
videoUrl: fallbackUrl,
error:
streamError?.message ||
'Streaming unavailable; playing buffered video instead.',
}));
} catch (fallbackError: any) {
setPreviewDialog((prev) => ({
...prev,
error:
fallbackError?.message ||
streamError?.message ||
'Unable to load chunked video.',
}));
}
});
updateStep('analyze', 'success');
return;
} catch (startError: any) {
updateStep('analyze', 'error', startError?.message);
cleanupChunkedMediaSource();
const fallbackMessage = startError?.message || 'Streaming unavailable.';
try {
const fallbackUrl = await createChunkedBlobUrl(chunkManifest);
setPreviewChunkedBlobUrl(fallbackUrl);
setPreviewDialog((prev) => ({
...prev,
open: true,
loading: false,
title: getResourceLabel(target),
type: 'video',
resource: target,
zoomed: false,
chunked: true,
videoUrl: fallbackUrl,
error: fallbackMessage,
}));
} catch (fallbackError: any) {
setPreviewDialog((prev) => ({
...prev,
open: true,
loading: false,
title: getResourceLabel(target),
type: 'video',
resource: target,
zoomed: false,
chunked: false,
videoUrl: undefined,
error:
fallbackError?.message || startError?.message || 'Unable to load chunked video.',
}));
}
return;
}
}
}
if (loaded.mime.startsWith('image/')) { if (loaded.mime.startsWith('image/')) {
setPreviewDialog((prev) => ({ setPreviewDialog((prev) => ({
...prev, ...prev,
@@ -2623,6 +2980,8 @@ export default function DataExplorer() {
dataUrl: `data:${loaded.mime};base64,${loaded.base64}`, dataUrl: `data:${loaded.mime};base64,${loaded.base64}`,
resource: target, resource: target,
zoomed: false, zoomed: false,
chunked: false,
videoUrl: undefined,
})); }));
return; return;
} }
@@ -2636,6 +2995,8 @@ export default function DataExplorer() {
type: 'video', type: 'video',
resource: target, resource: target,
zoomed: false, zoomed: false,
chunked: false,
videoUrl: undefined,
})); }));
return; return;
} }
@@ -2652,6 +3013,8 @@ export default function DataExplorer() {
content: text, content: text,
resource: target, resource: target,
zoomed: false, zoomed: false,
chunked: false,
videoUrl: undefined,
})); }));
} else { } else {
setPreviewDialog((prev) => ({ setPreviewDialog((prev) => ({
@@ -2663,6 +3026,8 @@ export default function DataExplorer() {
content: 'This resource appears to be binary. Use Save to system to download it.', content: 'This resource appears to be binary. Use Save to system to download it.',
resource: target, resource: target,
zoomed: false, zoomed: false,
chunked: false,
videoUrl: undefined,
})); }));
} }
} catch { } catch {
@@ -2676,6 +3041,8 @@ export default function DataExplorer() {
content: 'Preview not available. Use Save to system to download this resource.', content: 'Preview not available. Use Save to system to download this resource.',
resource: fallbackResource || null, resource: fallbackResource || null,
zoomed: false, zoomed: false,
chunked: false,
videoUrl: undefined,
})); }));
} }
} catch (e: any) { } catch (e: any) {
@@ -2689,6 +3056,8 @@ export default function DataExplorer() {
error: e?.message || 'Unable to preview this resource.', error: e?.message || 'Unable to preview this resource.',
resource: fallbackResource || null, resource: fallbackResource || null,
zoomed: false, zoomed: false,
chunked: false,
videoUrl: undefined,
})); }));
} }
}; };
@@ -2717,7 +3086,10 @@ export default function DataExplorer() {
})); }));
}; };
const handlePreviewClose = () => setPreviewDialog(createPreviewDialogState()); const handlePreviewClose = () => {
cleanupChunkedVideoPreview();
setPreviewDialog(createPreviewDialogState());
};
const handleManifestDialogOpen = (entry: StructuredEntry) => { const handleManifestDialogOpen = (entry: StructuredEntry) => {
setManifestDialog({ setManifestDialog({
@@ -5085,6 +5457,20 @@ export default function DataExplorer() {
height: previewDialog.expanded ? '70vh' : 420, height: previewDialog.expanded ? '70vh' : 420,
}} }}
> >
{previewDialog.videoUrl ? (
<video
ref={previewVideoRef}
src={previewDialog.videoUrl}
controls
style={{
width: '100%',
height: '100%',
objectFit: 'contain',
borderRadius: 8,
backgroundColor: '#000',
}}
/>
) : (
<VideoPlayer <VideoPlayer
videoRef={previewVideoRef} videoRef={previewVideoRef}
qortalVideoResource={{ qortalVideoResource={{
@@ -5093,6 +5479,7 @@ export default function DataExplorer() {
identifier: previewDialog.resource.identifier, identifier: previewDialog.resource.identifier,
}} }}
/> />
)}
</Box> </Box>
)} )}
{previewDialog.type === 'binary' && ( {previewDialog.type === 'binary' && (
@@ -5139,6 +5526,7 @@ export default function DataExplorer() {
: null; : null;
if (entry) { if (entry) {
void handleDeleteFilesCopy([entry]); void handleDeleteFilesCopy([entry]);
cleanupChunkedVideoPreview();
setPreviewDialog(createPreviewDialogState()); setPreviewDialog(createPreviewDialogState());
} }
}} }}
@@ -21,6 +21,7 @@ import type { GroupSummary } from '../../../../utils/qortalApi';
import { ALL_QDN_SERVICES } from '../constants'; import { ALL_QDN_SERVICES } from '../constants';
import { formatBytes } from '../viewHelpers'; import { formatBytes } from '../viewHelpers';
import { getServiceLimit } from '../../../../utils/useQdnBatchPublisher'; import { getServiceLimit } from '../../../../utils/useQdnBatchPublisher';
import { MAX_INLINE_FILE_SIZE, CHUNK_FORCED_THRESHOLD } from '../../../../utils/fileChunking';
export type PublishFormState = { export type PublishFormState = {
service: Service; service: Service;
@@ -38,6 +39,7 @@ export type PublishSubmitPayload = {
groupId: number | null; groupId: number | null;
groupAdminsOnly: boolean; groupAdminsOnly: boolean;
directRecipients: string; directRecipients: string;
chunkedPublishing: boolean;
}; };
type PublishDialogProps = { type PublishDialogProps = {
@@ -71,6 +73,7 @@ export function PublishDialog({
const [groupId, setGroupId] = useState<number | null>(null); const [groupId, setGroupId] = useState<number | null>(null);
const [groupAdminsOnly, setGroupAdminsOnly] = useState(false); const [groupAdminsOnly, setGroupAdminsOnly] = useState(false);
const [directRecipients, setDirectRecipients] = useState(''); const [directRecipients, setDirectRecipients] = useState('');
const [chunkedPublishing, setChunkedPublishing] = useState(false);
useEffect(() => { useEffect(() => {
if (!open) return; if (!open) return;
@@ -80,6 +83,7 @@ export function PublishDialog({
setGroupId(null); setGroupId(null);
setGroupAdminsOnly(false); setGroupAdminsOnly(false);
setDirectRecipients(''); setDirectRecipients('');
setChunkedPublishing(false);
onStatusChange(null); onStatusChange(null);
}, [defaults, open, onStatusChange]); }, [defaults, open, onStatusChange]);
@@ -88,6 +92,27 @@ export function PublishDialog({
[form.service] [form.service]
); );
const hasOptionalChunkable = files.some((file) => file.size > MAX_INLINE_FILE_SIZE);
const hasForcedChunking = files.some((file) => file.size > CHUNK_FORCED_THRESHOLD);
useEffect(() => {
if (hasForcedChunking) {
setChunkedPublishing(true);
}
}, [hasForcedChunking]);
useEffect(() => {
if (chunkedPublishing && encryptionMode !== 'group') {
setEncryptionMode('group');
}
}, [chunkedPublishing, encryptionMode]);
useEffect(() => {
if (encryptionMode === 'none' && chunkedPublishing) {
setChunkedPublishing(false);
}
}, [encryptionMode, chunkedPublishing]);
const handleSelectFiles = (event: React.ChangeEvent<HTMLInputElement>) => { const handleSelectFiles = (event: React.ChangeEvent<HTMLInputElement>) => {
const nextFiles = Array.from(event.target.files ?? []); const nextFiles = Array.from(event.target.files ?? []);
setFiles(nextFiles); setFiles(nextFiles);
@@ -102,6 +127,7 @@ export function PublishDialog({
groupId, groupId,
groupAdminsOnly, groupAdminsOnly,
directRecipients, directRecipients,
chunkedPublishing,
}); });
}; };
@@ -183,9 +209,13 @@ export function PublishDialog({
value={encryptionMode} value={encryptionMode}
onChange={(_event, value) => value && setEncryptionMode(value)} onChange={(_event, value) => value && setEncryptionMode(value)}
> >
<ToggleButton value="none">None</ToggleButton> <ToggleButton value="none" disabled={chunkedPublishing}>
None
</ToggleButton>
<ToggleButton value="group">Group</ToggleButton> <ToggleButton value="group">Group</ToggleButton>
<ToggleButton value="direct">Direct</ToggleButton> <ToggleButton value="direct" disabled={chunkedPublishing}>
Direct
</ToggleButton>
</ToggleButtonGroup> </ToggleButtonGroup>
{encryptionMode === 'group' && ( {encryptionMode === 'group' && (
<> <>
@@ -227,9 +257,32 @@ export function PublishDialog({
value={directRecipients} value={directRecipients}
onChange={(event) => setDirectRecipients(event.target.value)} onChange={(event) => setDirectRecipients(event.target.value)}
helperText="Direct encryption will use resolved public keys for the listed recipients." helperText="Direct encryption will use resolved public keys for the listed recipients."
disabled={chunkedPublishing}
/> />
)} )}
{encryptionMode !== 'none' && hasOptionalChunkable && (
<FormControlLabel
control={
<Switch
checked={chunkedPublishing}
onChange={(_event, checked) => setChunkedPublishing(checked)}
disabled={hasForcedChunking}
/>
}
label={
hasForcedChunking
? 'Chunked upload required for files over 100MB'
: 'Use chunked publishing for large files'
}
/>
)}
{chunkedPublishing && !groupId && (
<Typography variant="caption" color="warning.main">
Chunked publishing requires a private group and group encryption.
</Typography>
)}
<Button variant="outlined" component="label"> <Button variant="outlined" component="label">
{files.length ? 'Replace selection' : 'Select files'} {files.length ? 'Replace selection' : 'Select files'}
<input <input
+63
View File
@@ -0,0 +1,63 @@
export const MAX_INLINE_FILE_SIZE = 25 * 1024 * 1024; // 25 MB for inlined files
export const DEFAULT_CHUNK_SIZE = 25 * 1024 * 1024; // 25 MB per chunk (adjust if needed)
export const CHUNK_FORCED_THRESHOLD = 100 * 1024 * 1024; // 100 MB enforce chunked
export interface FileChunkDescriptor {
index: number;
identifier: string;
size: number;
}
export interface ChunkedFileManifest {
version: 1;
fileName: string;
size: number;
mimeType: string;
chunkSize: number;
chunks: FileChunkDescriptor[];
encryption?: {
mode: 'none' | 'group' | 'direct';
groupId?: number;
adminsOnly?: boolean;
recipientCount?: number;
};
}
export async function* iterateFileChunks(
file: File,
chunkSize: number = DEFAULT_CHUNK_SIZE
): AsyncGenerator<{ index: number; uint8: Uint8Array; size: number }, void, void> {
let offset = 0;
let index = 0;
while (offset < file.size) {
const end = Math.min(offset + chunkSize, file.size);
const slice = file.slice(offset, end);
const buf = await slice.arrayBuffer();
const uint8 = new Uint8Array(buf);
yield { index, uint8, size: uint8.length };
offset = end;
index += 1;
}
}
export function buildChunkIdentifier(baseIdentifier: string, index: number): string {
return `${baseIdentifier}__chunk__${String(index).padStart(4, '0')}`;
}
export function createChunkedManifest(
file: File,
chunkSize: number,
chunks: FileChunkDescriptor[],
encryption?: ChunkedFileManifest['encryption']
): ChunkedFileManifest {
const mimeType = file.type || 'application/octet-stream';
return {
version: 1,
fileName: file.name,
size: file.size,
mimeType,
chunkSize,
chunks,
encryption,
};
}
+93 -22
View File
@@ -1,4 +1,5 @@
import { Service } from 'qapp-core'; import { Service } from 'qapp-core';
import pLimit from 'p-limit';
import { qaAnnouncementPrefix, assetNewsGlobalPrefix } from '../constants/qdnConstants'; import { qaAnnouncementPrefix, assetNewsGlobalPrefix } from '../constants/qdnConstants';
import { NewsSummary } from '../types/newsAndPromos'; import { NewsSummary } from '../types/newsAndPromos';
import { fetchPromotionApprovals } from './promotions'; import { fetchPromotionApprovals } from './promotions';
@@ -60,6 +61,7 @@ type FetchNewsOptions = {
includeExpired?: boolean; includeExpired?: boolean;
forceFresh?: boolean; forceFresh?: boolean;
allowedGroupIds?: number[]; // membership list to gate private asset news allowedGroupIds?: number[]; // membership list to gate private asset news
signal?: AbortSignal;
}; };
const LIST_CACHE_MS = 60_000; const LIST_CACHE_MS = 60_000;
@@ -71,6 +73,12 @@ export async function fetchAnnouncements(
options?: FetchNewsOptions options?: FetchNewsOptions
): Promise<NewsSummary[]> { ): Promise<NewsSummary[]> {
try { try {
const ensureNotAborted = () => {
if (options?.signal?.aborted) {
throw new DOMException('AbortError', 'AbortError');
}
};
ensureNotAborted();
const includeExpired = options?.includeExpired ?? false; const includeExpired = options?.includeExpired ?? false;
const listKey = `ann:list:${includeExpired}:${limit}`; const listKey = `ann:list:${includeExpired}:${limit}`;
if (!options?.forceFresh) { if (!options?.forceFresh) {
@@ -81,8 +89,10 @@ export async function fetchAnnouncements(
const expiryDays = Number(await getNewsPromoExpiryDays()); const expiryDays = Number(await getNewsPromoExpiryDays());
const expiryCutoff = const expiryCutoff =
Number.isFinite(expiryDays) && expiryDays > 0 ? Date.now() - expiryDays * 86_400_000 : null; Number.isFinite(expiryDays) && expiryDays > 0 ? Date.now() - expiryDays * 86_400_000 : null;
ensureNotAborted();
const approvalDoc = await loadAnnouncementApprovalDoc(); const approvalDoc = await loadAnnouncementApprovalDoc();
const approvedEntries = approvalDoc.items || []; const approvedEntries = approvalDoc.items || [];
ensureNotAborted();
const items: NewsSummary[] = []; const items: NewsSummary[] = [];
const seen = new Set<string>(); const seen = new Set<string>();
@@ -96,12 +106,14 @@ export async function fetchAnnouncements(
createdHint?: number createdHint?: number
) => { ) => {
try { try {
ensureNotAborted();
const svc = service || ('DOCUMENT' as Service); const svc = service || ('DOCUMENT' as Service);
const cacheKey = `ann:item:${(publisher || '').toLowerCase()}:${svc}:${identifier}`; const cacheKey = `ann:item:${(publisher || '').toLowerCase()}:${svc}:${identifier}`;
let payload: { html: string; title?: string; createdAt?: number } | null | undefined = let payload: { html: string; title?: string; createdAt?: number } | null | undefined =
getCached(cacheKey); getCached(cacheKey);
if (!payload) { if (!payload) {
ensureNotAborted();
const res = await qortalRequest({ const res = await qortalRequest({
action: 'FETCH_QDN_RESOURCE', action: 'FETCH_QDN_RESOURCE',
name: publisher, name: publisher,
@@ -109,10 +121,13 @@ export async function fetchAnnouncements(
identifier, identifier,
encoding: 'base64', encoding: 'base64',
}); });
ensureNotAborted();
payload = await decodeAnnouncementResource(res?.data64 ?? res); payload = await decodeAnnouncementResource(res?.data64 ?? res);
ensureNotAborted();
if (payload) setCached(cacheKey, payload, ITEM_CACHE_MS); if (payload) setCached(cacheKey, payload, ITEM_CACHE_MS);
} }
ensureNotAborted();
if (!payload) return false; if (!payload) return false;
const html = payload.html; const html = payload.html;
const title = payload.title || extractTitleFromHtml(html, 'Q-Assets Announcement'); const title = payload.title || extractTitleFromHtml(html, 'Q-Assets Announcement');
@@ -147,6 +162,7 @@ export async function fetchAnnouncements(
.slice(0, limit * 2); .slice(0, limit * 2);
for (const entry of ordered) { for (const entry of ordered) {
ensureNotAborted();
const dedupeKey = keyFor(entry.publisher, entry.identifier); const dedupeKey = keyFor(entry.publisher, entry.identifier);
if (seen.has(dedupeKey)) continue; if (seen.has(dedupeKey)) continue;
const createdHint = entry.approvedAt || entry.createdAt || Date.now(); const createdHint = entry.approvedAt || entry.createdAt || Date.now();
@@ -165,9 +181,11 @@ export async function fetchAnnouncements(
// Also surface admin-published announcements even if not explicitly approved // Also surface admin-published announcements even if not explicitly approved
let docHits: Awaited<ReturnType<typeof searchSimpleByIdentifierPrefix>> = []; let docHits: Awaited<ReturnType<typeof searchSimpleByIdentifierPrefix>> = [];
ensureNotAborted();
try { try {
const services = await getGroupResourceServices(); const services = await getGroupResourceServices();
ensureNotAborted();
[docHits] = await Promise.all([ [docHits] = await Promise.all([
searchSimpleByIdentifierPrefix(services, qaAnnouncementPrefix, limit * 2), searchSimpleByIdentifierPrefix(services, qaAnnouncementPrefix, limit * 2),
]); ]);
@@ -191,6 +209,7 @@ export async function fetchAnnouncements(
if (items.length >= limit * 2) break; if (items.length >= limit * 2) break;
} }
ensureNotAborted();
const finalList = items.sort((a, b) => b.created - a.created).slice(0, limit); const finalList = items.sort((a, b) => b.created - a.created).slice(0, limit);
setCached(listKey, finalList, LIST_CACHE_MS); setCached(listKey, finalList, LIST_CACHE_MS);
return finalList; return finalList;
@@ -217,7 +236,19 @@ export async function fetchLatestAssetNews(
limit = 10, limit = 10,
options?: FetchNewsOptions options?: FetchNewsOptions
): Promise<NewsSummary[]> { ): Promise<NewsSummary[]> {
const isAbortError = (error: unknown) =>
Boolean(
error && typeof error === 'object' && (error as { name?: string }).name === 'AbortError'
);
try { try {
const ensureNotAborted = () => {
if (options?.signal?.aborted) {
throw new DOMException('AbortError', 'AbortError');
}
};
ensureNotAborted();
const includeExpired = options?.includeExpired ?? false; const includeExpired = options?.includeExpired ?? false;
const allowedGroupIds = options?.allowedGroupIds ?? []; const allowedGroupIds = options?.allowedGroupIds ?? [];
const listKey = `assetnews:list:${includeExpired}:${limit}:${allowedGroupIds const listKey = `assetnews:list:${includeExpired}:${limit}:${allowedGroupIds
@@ -232,24 +263,38 @@ export async function fetchLatestAssetNews(
typeof expiryDays === 'number' && expiryDays > 0 typeof expiryDays === 'number' && expiryDays > 0
? Date.now() - expiryDays * 86_400_000 ? Date.now() - expiryDays * 86_400_000
: null; : null;
ensureNotAborted();
let hits: Awaited<ReturnType<typeof searchSimpleByIdentifierPrefix>> = []; let hits: Awaited<ReturnType<typeof searchSimpleByIdentifierPrefix>> = [];
try { try {
const services = await getGroupResourceServices(); const services = await getGroupResourceServices();
hits = await searchSimpleByIdentifierPrefix(services, assetNewsGlobalPrefix, limit); ensureNotAborted();
[hits] = await Promise.all([
searchSimpleByIdentifierPrefix(services, assetNewsGlobalPrefix, limit),
]);
} catch (e) { } catch (e) {
console.warn('Failed to fetch asset news list', e); console.warn('Failed to fetch asset news list', e);
return []; return [];
} }
ensureNotAborted();
if (!hits.length) return []; if (!hits.length) return [];
const items: NewsSummary[] = []; const dedupedHits: typeof hits = [];
const seen = new Set<string>(); const seenIds = new Set<string>();
const privacyCache = new Map<number, { isPrivate: boolean; groupId?: number }>(); for (const hit of hits) {
ensureNotAborted();
const dedupeKey = `${hit.name}::${hit.identifier}`;
if (seenIds.has(dedupeKey)) continue;
seenIds.add(dedupeKey);
dedupedHits.push(hit);
}
const privacyCache = new Map<number, { isPrivate: boolean; groupId?: number }>();
const getPrivacy = async (assetId: number) => { const getPrivacy = async (assetId: number) => {
ensureNotAborted();
if (privacyCache.has(assetId)) return privacyCache.get(assetId)!; if (privacyCache.has(assetId)) return privacyCache.get(assetId)!;
try { try {
const { publication } = await resolveAssetPublicationById(assetId); const { publication } = await resolveAssetPublicationById(assetId);
ensureNotAborted();
const groupIdRaw = publication?.privateGroupId ?? publication?.primaryGroup?.id; const groupIdRaw = publication?.privateGroupId ?? publication?.primaryGroup?.id;
const groupIdNum = groupIdRaw != null ? Number(groupIdRaw) : undefined; const groupIdNum = groupIdRaw != null ? Number(groupIdRaw) : undefined;
const info = { const info = {
@@ -265,16 +310,17 @@ export async function fetchLatestAssetNews(
} }
}; };
for (const hit of hits) { const limiter = pLimit(3);
const dedupeKey = `${hit.name}::${hit.identifier}`; const tasks = dedupedHits.map((hit) =>
if (seen.has(dedupeKey)) continue; limiter(async () => {
seen.add(dedupeKey); ensureNotAborted();
const finalService = hit.service ? (hit.service as Service) : ('DOCUMENT' as Service);
try { try {
const finalService = hit.service ? (hit.service as Service) : ('DOCUMENT' as Service);
const payloadKey = `assetnews:item:${hit.name.toLowerCase()}:${finalService}:${hit.identifier}`; const payloadKey = `assetnews:item:${hit.name.toLowerCase()}:${finalService}:${hit.identifier}`;
let payload = getCached<{ html: string; title?: string; createdAt?: number }>(payloadKey); let payload = getCached<{ html: string; title?: string; createdAt?: number }>(payloadKey);
if (!payload) { if (!payload) {
ensureNotAborted();
const res = await qortalRequest({ const res = await qortalRequest({
action: 'FETCH_QDN_RESOURCE', action: 'FETCH_QDN_RESOURCE',
name: hit.name, name: hit.name,
@@ -282,7 +328,7 @@ export async function fetchLatestAssetNews(
identifier: hit.identifier, identifier: hit.identifier,
encoding: 'base64', encoding: 'base64',
}); });
ensureNotAborted();
const raw = res?.data64 ?? res; const raw = res?.data64 ?? res;
let html = ''; let html = '';
let title: string | undefined; let title: string | undefined;
@@ -310,12 +356,15 @@ export async function fetchLatestAssetNews(
} }
} }
} }
if (!html) console.log('wtfnohtml', html); if (!html) {
if (!html) continue; console.log('wtfnohtml', html);
return null;
}
payload = { html, title, createdAt }; payload = { html, title, createdAt };
setCached(payloadKey, payload, ITEM_CACHE_MS); setCached(payloadKey, payload, ITEM_CACHE_MS);
} }
ensureNotAborted();
const html = payload.html; const html = payload.html;
let title = payload.title; let title = payload.title;
const createdAt = payload.createdAt; const createdAt = payload.createdAt;
@@ -323,7 +372,6 @@ export async function fetchLatestAssetNews(
const text = stripHtml(html); const text = stripHtml(html);
const excerpt = text.slice(0, 220) + (text.length > 220 ? '…' : ''); const excerpt = text.slice(0, 220) + (text.length > 220 ? '…' : '');
// Try to derive assetId from identifier: asset_news_pub__<assetId>__<id6>
let assetId: number | undefined; let assetId: number | undefined;
const m = hit.identifier.match(/^asset_news_pub__([0-9]+)__/); const m = hit.identifier.match(/^asset_news_pub__([0-9]+)__/);
if (m && m[1]) { if (m && m[1]) {
@@ -332,12 +380,12 @@ export async function fetchLatestAssetNews(
const assetName = assetId != null ? `Asset #${assetId}` : undefined; const assetName = assetId != null ? `Asset #${assetId}` : undefined;
// Visibility: hide private asset news unless viewer is in the allowed group
if (assetId != null) { if (assetId != null) {
const privacy = await getPrivacy(assetId); const privacy = await getPrivacy(assetId);
ensureNotAborted();
if (privacy.isPrivate) { if (privacy.isPrivate) {
if (!privacy.groupId) continue; // cannot authorize, skip if (!privacy.groupId) return null;
if (!allowedGroupIds.includes(privacy.groupId)) continue; if (!allowedGroupIds.includes(privacy.groupId)) return null;
} }
} }
@@ -345,13 +393,12 @@ export async function fetchLatestAssetNews(
html, html,
assetId != null ? `News for ${assetName}` : 'Asset news' assetId != null ? `News for ${assetName}` : 'Asset news'
); );
// const finalTitle = typeof title === 'string' && title.trim() ? title : titleExtracted;
const created = createdAt || hit.updated || hit.created || Date.now(); const created = createdAt || hit.updated || hit.created || Date.now();
const isExpired = expiryCutoff != null && created < expiryCutoff; const isExpired = expiryCutoff != null && created < expiryCutoff;
if (!includeExpired && isExpired) continue; if (!includeExpired && isExpired) return null;
items.push({ return {
type: 'assetNews', type: 'assetNews',
identifier: hit.identifier, identifier: hit.identifier,
title, title,
@@ -363,27 +410,50 @@ export async function fetchLatestAssetNews(
fullHtml: html, fullHtml: html,
publisherName: hit.name, publisherName: hit.name,
service: finalService, service: finalService,
}); };
} catch (err) { } catch (err) {
if (isAbortError(err)) throw err;
console.warn('Failed to fetch asset news item', err); console.warn('Failed to fetch asset news item', err);
return null;
} }
})
);
const results = await Promise.all(tasks);
const items: NewsSummary[] = [];
for (const entry of results) {
if (entry) items.push(entry);
} }
// Sort latest first and trim to limit ensureNotAborted();
const finalList = items.sort((a, b) => b.created - a.created).slice(0, limit); const finalList = items.sort((a, b) => b.created - a.created).slice(0, limit);
setCached(listKey, finalList, LIST_CACHE_MS); setCached(listKey, finalList, LIST_CACHE_MS);
return finalList; return finalList;
} catch (e) { } catch (e) {
if (isAbortError(e)) {
throw e;
}
console.warn('fetchLatestAssetNews failed', e); console.warn('fetchLatestAssetNews failed', e);
return []; return [];
} }
} }
export async function fetchActivePromotions(now = Date.now()): Promise<NewsSummary[]> { export async function fetchActivePromotions(
now = Date.now(),
options?: { signal?: AbortSignal }
): Promise<NewsSummary[]> {
const ensureNotAborted = () => {
if (options?.signal?.aborted) {
throw new DOMException('AbortError', 'AbortError');
}
};
ensureNotAborted();
const approvals = await fetchPromotionApprovals(120); const approvals = await fetchPromotionApprovals(120);
const promos: NewsSummary[] = []; const promos: NewsSummary[] = [];
for (const promo of approvals) { for (const promo of approvals) {
ensureNotAborted();
if (!promo.contentHtml) continue; if (!promo.contentHtml) continue;
if (!promo.isActive) continue; if (!promo.isActive) continue;
if (now < promo.startsAt || now > promo.endsAt) continue; if (now < promo.startsAt || now > promo.endsAt) continue;
@@ -406,5 +476,6 @@ export async function fetchActivePromotions(now = Date.now()): Promise<NewsSumma
}); });
} }
ensureNotAborted();
return promos.sort((a, b) => (b.promotionEndsAt ?? 0) - (a.promotionEndsAt ?? 0)); return promos.sort((a, b) => (b.promotionEndsAt ?? 0) - (a.promotionEndsAt ?? 0));
} }