Twitter API pagination, batching & tweet exports

{
  "route": "/api/v1/x/tweets/search",
  "query": "from:username webhook OR SDK",
  "query_type": "Latest",
  "limit": 100,
  "cursor_sent": null,
  "has_next_page": true,
  "next_cursor": "DAACCgACGRElMJcAAA",
  "unique_tweet_count": 93,
  "last_saved_tweet_id": "1893710452812718080",
  "collected_at": "2026-08-02T20:30:00Z"
}

type Tweet = {
  id: string;
  text?: string;
};

type TweetPage = {
  tweets: Tweet[];
  has_next_page: boolean;
  next_cursor?: string;
};

type SearchCheckpoint = {
  cursor_sent: string | null;
  next_cursor: string | null;
  page_number: number;
  unique_tweet_count: number;
};

type SearchCollectionResult = {
  complete: boolean;
  next_cursor: string | null;
  unique_tweet_count: number;
};

async function collectTweetSearch(
  apiKey: string,
  query: string,
  savePage: (tweets: Tweet[], checkpoint: SearchCheckpoint) => Promise<void>,
): Promise<SearchCollectionResult> {
  const seenTweetIds = new Set<string>();
  const seenCursors = new Set<string>();
  let cursor = "";

  for (let pageNumber = 1; pageNumber <= 50; pageNumber += 1) {
    const params = new URLSearchParams({
      limit: "100",
      q: query,
      queryType: "Latest",
    });
    if (cursor !== "") {
      params.set("cursor", cursor);
    }

    const response = await fetch(
      `https://xquik.com/api/v1/x/tweets/search?${params}`,
      { headers: { "x-api-key": apiKey } },
    );
    if (!response.ok) {
      throw new Error(`Tweet search failed with ${response.status}`);
    }

    const page = (await response.json()) as TweetPage;
    const uniqueTweets = page.tweets.filter((tweet) => {
      if (seenTweetIds.has(tweet.id)) {
        return false;
      }
      seenTweetIds.add(tweet.id);
      return true;
    });

    const nextCursor = page.next_cursor || null;
    if (
      page.has_next_page &&
      nextCursor !== null &&
      (nextCursor === cursor || seenCursors.has(nextCursor))
    ) {
      throw new Error("Pagination stopped because the cursor repeated");
    }

    await savePage(uniqueTweets, {
      cursor_sent: cursor || null,
      next_cursor: nextCursor,
      page_number: pageNumber,
      unique_tweet_count: seenTweetIds.size,
    });

    if (!page.has_next_page) {
      return {
        complete: true,
        next_cursor: null,
        unique_tweet_count: seenTweetIds.size,
      };
    }

    if (nextCursor === null) {
      return {
        complete: false,
        next_cursor: null,
        unique_tweet_count: seenTweetIds.size,
      };
    }

    seenCursors.add(nextCursor);
    cursor = nextCursor;
  }

  return {
    complete: false,
    next_cursor: cursor,
    unique_tweet_count: seenTweetIds.size,
  };
}