0

I have been trying to extract the image url from twitter api's search tweets with Id response. The url are present in the Json response from the api in the includes->media array in the response. But when I try to get it using the twitter java Sdk it is not present their. Following is the link to the Sdk: Twitter official Java Sdk

https://api.twitter.com/2/tweets?ids=1102112466283175936,1553802769207037953,1552279853633835009&tweet.fields=author_id,entities,attachments,conversation_id,created_at,referenced_tweets&expansions=attachments.media_keys&media.fields=url,type

Twitter api request

JSON response

{
"data": [
    {
        "text": "5000 selfies make up this iconic image of @jeremycorbyn in today's @ObserverUK, calling on him to bring courage and leadership to unite @UKLabour MPs behind a #PublicVote on #Brexit. RT to support! ",
        "attachments": {
            "media_keys": [
                "3_1102112450588147712"
            ]
        },
        "entities": {
            "hashtags": [
                {
                    "start": 159,
                    "end": 170,
                    "tag": "PublicVote"
                },
                {
                    "start": 174,
                    "end": 181,
                    "tag": "Brexit"
                }
            ],
            "urls": [
                {
                    "start": 198,
                    "end": 221,
                    
                    "expanded_url": "https://twitter.com/Avaaz/status/1102112466283175936/photo/1",
                    "display_url": "pic.twitter.com/RAQN3rvXOc",
                    "media_key": "3_1102112450588147712"
                }
            ],
            "mentions": [
                {
                    "start": 42,
                    "end": 55,
                    "username": "jeremycorbyn",
                    "id": "117777690"
                },
                {
                    "start": 67,
                    "end": 78,
                    "username": "ObserverUK",
                    "id": "1017383741919023104"
                },
                {
                    "start": 136,
                    "end": 145,
                    "username": "UKLabour",
                    "id": "14291684"
                }
            ]
        },
        "id": "1102112466283175936",
        "created_at": "2019-03-03T07:44:22.000Z",
        "conversation_id": "1102112466283175936",
        "author_id": "2553151"
    },
    {
        "text": "@oldschoolmonk My current favorite doge meme ❤️ ",
        "attachments": {
            "media_keys": [
                "3_1553802763171414016"
            ]
        },
        "id": "1553802769207037953",
        "created_at": "2022-07-31T18:00:23.000Z",
        "conversation_id": "1553801521476669440",
        "referenced_tweets": [
            {
                "type": "replied_to",
                "id": "1553801521476669440"
            }
        ],
        "entities": {
            "urls": [
                {
                    "start": 49,
                    "end": 72,
                    
                    "expanded_url": "https://twitter.com/Pareshaan_aatma/status/1553802769207037953/photo/1",
                    "display_url": "pic.twitter.com/Gp8FBBHmHO",
                    "media_key": "3_1553802763171414016"
                }
            ],
            "mentions": [
                {
                    "start": 0,
                    "end": 14,
                    "username": "oldschoolmonk",
                    "id": "135241813"
                }
            ]
        },
        "author_id": "224239908"
    },
    {
        "text": "I've just found out that 3 of my Samsung phones' batteries have blown up because of the recent UK heatwave \n\nNot a single other brand has  ",
        "attachments": {
            "media_keys": [
                "3_1552279848432844800",
                "3_1552279848416169987"
            ]
        },
        "id": "1552279853633835009",
        "created_at": "2022-07-27T13:08:51.000Z",
        "entities": {
            "annotations": [
                {
                    "start": 33,
                    "end": 39,
                    "probability": 0.8414,
                    "type": "Product",
                    "normalized_text": "Samsung"
                },
                {
                    "start": 95,
                    "end": 96,
                    "probability": 0.5409,
                    "type": "Place",
                    "normalized_text": "UK"
                }
            ],
            "urls": [
                {
                    "start": 140,
                    "end": 163,
                    
                    "expanded_url": "https://twitter.com/Mrwhosetheboss/status/1552279853633835009/photo/1",
                    "display_url": "pic.twitter.com/LI6Qrz8xSv",
                    "media_key": "3_1552279848432844800"
                },
                {
                    "start": 140,
                    "end": 163,
                    
                    "expanded_url": "https://twitter.com/Mrwhosetheboss/status/1552279853633835009/photo/1",
                    "display_url": "pic.twitter.com/LI6Qrz8xSv",
                    "media_key": "3_1552279848416169987"
                }
            ]
        },
        "conversation_id": "1552279853633835009",
        "author_id": "3363946175"
    }
],
"includes": {
    "media": [
        {
            "media_key": "3_1102112450588147712",
            "type": "photo",
            "url": "https://pbs.twimg.com/media/D0t9bz_XgAAl3hD.jpg"
        },
        {
            "media_key": "3_1553802763171414016",
            "type": "photo",
            "url": "https://pbs.twimg.com/media/FZA3YXXaQAAOnma.jpg"
        },
        {
            "media_key": "3_1552279848432844800",
            "type": "photo",
            "url": "https://pbs.twimg.com/media/FYrOTD3WQAA0tvt.jpg"
        },
        {
            "media_key": "3_1552279848416169987",
            "type": "photo",
            "url": "https://pbs.twimg.com/media/FYrOTDzX0AMBSnE.jpg"
        }
    ]
}

}

Shivam Kumar
  • 7
  • 1

1 Answers1

1

tl;dr

As of writing (v2.0.1), there is no direct way to retrieve a fully hydrated Tweet object.

Long version

To retrieve a fully hydrated Tweet object, you first need to enable field expansion on the client.

            final Get2TweetsSearchRecentResponse response =
                    twitterApi.tweets()
                            .tweetsRecentSearch(queryString.toString())
                            .expansions(TwitterEntityFields.getExpansions())
                            .mediaFields(TwitterEntityFields.getMediaFields())
                            .placeFields(TwitterEntityFields.getPlaceFields())
                            .userFields(TwitterEntityFields.getUserFields())
                            .tweetFields(TwitterEntityFields.getTweetFields())
                            .maxResults(maxNoOfTweets)
                            .execute(connectionRetries);

TwitterEntityFields is something like, which encapsulates all required fields for expansion.

public abstract class TwitterEntityFields {

private final static Set<String> TWEET_FIELDS = new HashSet<>();
private final static Set<String> PLACE_FIELDS = new HashSet<>();
private final static Set<String> MEDIA_FIELDS = new HashSet<>();
private final static Set<String> USER_FIELDS = new HashSet<>();
private final static Set<String> EXPANSIONS = new HashSet<>();

static {
    TWEET_FIELDS.add("author_id");
    TWEET_FIELDS.add("id");
    TWEET_FIELDS.add("created_at");
    TWEET_FIELDS.add("public_metrics");
    TWEET_FIELDS.add("entities");
    TWEET_FIELDS.add("geo");
    TWEET_FIELDS.add("lang");
    TWEET_FIELDS.add("source");
    TWEET_FIELDS.add("referenced_tweets");
    TWEET_FIELDS.add("in_reply_to_user_id");

    PLACE_FIELDS.add("country");
    PLACE_FIELDS.add("geo");
    PLACE_FIELDS.add("name");
    PLACE_FIELDS.add("place_type");
    PLACE_FIELDS.add("contained_within");

    MEDIA_FIELDS.add("duration_ms");
    MEDIA_FIELDS.add("height");
    MEDIA_FIELDS.add("media_key");
    MEDIA_FIELDS.add("preview_image_url");
    MEDIA_FIELDS.add("public_metrics");
    MEDIA_FIELDS.add("width");
    MEDIA_FIELDS.add("url");

    EXPANSIONS.add("author_id");
    EXPANSIONS.add("attachments.media_keys");
    EXPANSIONS.add("geo.place_id");

    USER_FIELDS.add("created_at");
    USER_FIELDS.add("description");
    USER_FIELDS.add("location");
    USER_FIELDS.add("profile_image_url");
    USER_FIELDS.add("protected");
    USER_FIELDS.add("public_metrics");
    USER_FIELDS.add("url");
    USER_FIELDS.add("verified");
}

public static Set<String> getTweetFields() {
    return Collections.unmodifiableSet(TWEET_FIELDS);
}

public static Set<String> getPlaceFields() {
    return Collections.unmodifiableSet(PLACE_FIELDS);
}

public static Set<String> getMediaFields() {
    return Collections.unmodifiableSet(MEDIA_FIELDS);
}

public static Set<String> getUserFields() {
    return Collections.unmodifiableSet(USER_FIELDS);
}

public static Set<String> getExpansions() {
    return Collections.unmodifiableSet(EXPANSIONS);
}

}

Next, you need to map the media keys (contained in the attachments of a Tweet to the corresponding expanded Media objects.

To do so, you can do something like

public class TweetAdapter {

private final Tweet tweet;
private final Set<CashtagEntity> symbols = new HashSet<>();
private final Set<HashtagEntity> hashtags = new HashSet<>();
private final Set<MentionEntity> mentions = new HashSet<>();
private final Set<UrlEntity> urls = new HashSet<>();
private final Set<Media> media = new HashSet<>();
private final User user;
private final Place place;

public TweetAdapter(Tweet tweet, Expansions expansions) {
    assert (tweet != null);
    assert (expansions != null);
    this.tweet = tweet;
    collectTextEntities(tweet);
    this.place = collectPlaces(expansions);
    this.media.addAll(collectMedia(expansions));
    this.user = collectUser(expansions);
}

private User collectUser(Expansions expansions) {
    if (expansions != null && expansions.getUsers() != null) {
        for (User user : expansions.getUsers()) {
            if (user.getId().equals(tweet.getAuthorId())) {
                return user;
            }
        }
    }
    throw new RuntimeException("Could not extract Tweet's author. Tweet with ID = " + tweet.getId());
}

private Set<Media> collectMedia(Expansions expansions) {
    final Set<Media> result = new HashSet<>();
    if (expansions != null) {
        final TweetAttachments attachments = tweet.getAttachments();
        final List<Media> media = expansions.getMedia();

        if (attachments != null && media != null) {
            if (attachments.getMediaKeys() != null) {
                for (String mKey : attachments.getMediaKeys()) {
                    for (Media m : media) {
                        if (m.getMediaKey() != null && m.getMediaKey().equals(mKey)) {
                            result.add(m);
                        }
                    }
                }
            }
        }
    }
    return result;
}

private Place collectPlaces(Expansions expansions) {
    if (expansions != null
            && tweet.getGeo() != null
            && expansions.getPlaces() != null) {
        final String placeId = tweet.getGeo().getPlaceId();

        for (Place place : expansions.getPlaces()) {

            if (place.getId().equals(placeId)) {
                return place;
            }

        }

    }
    return null;
}

private void collectTextEntities(Tweet tweet) {
    final FullTextEntities entities = tweet.getEntities();

    if (entities != null) {
        if (entities.getCashtags() != null) {
            for (CashtagEntity c : entities.getCashtags()) {
                symbols.add(c);
            }
        }

        if (entities.getHashtags() != null) {
            for (HashtagEntity c : entities.getHashtags()) {
                hashtags.add(c);
            }
        }
        if (entities.getMentions() != null) {
            for (MentionEntity c : entities.getMentions()) {
                mentions.add(c);
            }
        }
        if (entities.getUrls() != null) {
            for (UrlEntity c : entities.getUrls()) {
                urls.add(c);
            }
        }
    }
} 
 // getters    
}

This will provide a fully hydrated Tweet.

Retrieving the URL of a Media object then boils down to:

    if (media instanceof Video v) {
        return v.getPreviewImageUrl() != null ? v.getPreviewImageUrl().toString() : Constants.EMPTY_STRING;
    }
    if (media instanceof Photo v) {
        return v.getUrl() != null ? v.getUrl().toString() : Constants.EMPTY_STRING;
    }
    if (media instanceof AnimatedGif v) {
        return v.getPreviewImageUrl() != null ? v.getPreviewImageUrl().toString() : Constants.EMPTY_STRING;
    }
rzo1
  • 5,561
  • 3
  • 25
  • 64