Before you all go on a rampage about how this is a duplicate question, I have spent two days working on this issue, watching youtube tutorials on asynchronous programming, surfing similar stackoverflow posts etc, and I cannot for the life of me figure out how to apply Asynchronous Parallel Downloading of files into my project.
First things first, some background:
I am creating a program that, when given a query input via the user, will make a call to the twitch API and download clips.
My program is two parts
1- A web scraper that generates a .json file with all details needed to download files and
2 - A downloader.
Part 1 works perfectly fine and generates the .json files no trouble.
My Downloader contains reference to a Data class that is a handler for common properties and methods like my ClientID, Authentication, OutputPath, JsonFile, QueryURL. It also contains methods to give values to these properties.
Here are the two methods of my FileDownloader.cs that are the problem:
public async static void DownloadAllFiles(Data clientData)
{
data = clientData;
data.OutputFolderExists();
// Deserialize .json file and get ClipInfo list
List<ClipInfo> clips = JsonConvert.DeserializeObject<List<ClipInfo>>(File.ReadAllText(data.JsonFile));
tasks = new List<Task>();
foreach(ClipInfo clip in clips)
{
tasks.Add(DownloadFilesAsync(clip));
}
await Task.WhenAll(tasks);
}
private async static Task DownloadFilesAsync(ClipInfo clip)
{
WebClient client = new WebClient();
string url = GetClipURL(clip);
string filepath = data.OutputPath + clip.id + ".mp4";
await client.DownloadFileTaskAsync(new Uri(url), filepath);
}
This is only one of my many attempts of downloading files, one which I got the idea from this post:
I have also tried methods like the following from a YouTube video by IAmTimCorey:
I have spent many an hour tackling this problem, and I honestly can't figure out why it won't work with any of my attempts. I would vastly appreciate your help.
Thanks,
Ben
Below is the entirety of my code, should anyone need it for any reason.
Code Structure:
The only external libraries I have downloaded is Newtonsoft.Json
ClipInfo.cs
using System;
using System.Collections.Generic;
using System.Text;
namespace Downloader
{
public class ClipInfo
{
public string id { get; set; }
public string url { get; set; }
public string embed_url { get; set; }
public string broadcaster_id { get; set; }
public string broadcaster_name { get; set; }
public string creator_id { get; set; }
public string creator_name { get; set; }
public string video_id { get; set; }
public string game_id { get; set; }
public string language { get; set; }
public string title { get; set; }
public int view_count { get; set; }
public DateTime created_at { get; set; }
public string thumbnail_url { get; set; }
}
}
Pagination.cs
namespace Downloader
{
public class Pagination
{
public string cursor { get; set; }
}
}
Root.cs
using System.Collections.Generic;
namespace Downloader
{
public class Root
{
public List<ClipInfo> data { get; set; }
public Pagination pagination { get; set; }
}
}
Data.cs
using System;
using System.IO;
namespace Downloader
{
public class Data
{
private static string directory = Directory.GetCurrentDirectory();
private readonly static string defaultJsonFile = directory + @"\clips.json";
private readonly static string defaultOutputPath = directory + @"\Clips\";
private readonly static string clipsLink = "https://api.twitch.tv/helix/clips?";
public string OutputPath { get; set; }
public string JsonFile { get; set; }
public string ClientID { get; private set; }
public string Authentication { get; private set; }
public string QueryURL { get; private set; }
public Data()
{
OutputPath = defaultOutputPath;
JsonFile = defaultJsonFile;
}
public Data(string clientID, string authentication)
{
ClientID = clientID;
Authentication = authentication;
OutputPath = defaultOutputPath;
JsonFile = defaultJsonFile;
}
public Data(string clientID, string authentication, string outputPath)
{
ClientID = clientID;
Authentication = authentication;
OutputPath = directory + @"\" + outputPath + @"\";
JsonFile = OutputPath + outputPath + ".json";
}
public void GetQuery()
{
Console.Write("Please enter your query: ");
QueryURL = clipsLink + Console.ReadLine();
}
public void GetClientID()
{
Console.WriteLine("Enter your client ID");
ClientID = Console.ReadLine();
}
public void GetAuthentication()
{
Console.WriteLine("Enter your Authentication");
Authentication = Console.ReadLine();
}
public void OutputFolderExists()
{
if (!Directory.Exists(OutputPath))
{
Directory.CreateDirectory(OutputPath);
}
}
}
}
JsonGenerator.cs
using System;
using System.IO;
using System.Net.Http.Headers;
using System.Net.Http;
using System.Threading.Tasks;
using Newtonsoft.Json;
using System.Linq;
namespace Downloader
{
public static class JsonGenerator
{
// This class has no constructor.
// You call the Generate methods, passing in all required data.
// The file will then be generated.
private static Data data;
public static async Task Generate(Data clientData)
{
data = clientData;
string responseContent = null;
// Loop that runs until the api request goes through
bool authError = true;
while (authError)
{
authError = false;
try
{
responseContent = await GetHttpResponse();
}
catch (HttpRequestException)
{
Console.WriteLine("Invalid authentication, please enter client-ID and authentication again!");
data.GetClientID();
data.GetAuthentication();
authError = true;
}
catch (Exception e)
{
Console.WriteLine(e.Message);
authError = true;
}
}
data.OutputFolderExists();
GenerateJson(responseContent);
}
// Returns the contents of the resopnse to the api call as a string
private static async Task<string> GetHttpResponse()
{
// Creating client
HttpClient client = new HttpClient();
if (data.QueryURL == null)
{
data.GetQuery();
}
// Setting up request
HttpRequestMessage requestMessage = new HttpRequestMessage(HttpMethod.Get, data.QueryURL);
// Adding Headers to request
requestMessage.Headers.Add("client-id", data.ClientID);
requestMessage.Headers.Authorization = new AuthenticationHeaderValue("Bearer", data.Authentication);
// Receiving response to the request
HttpResponseMessage responseMessage = await client.SendAsync(requestMessage);
// Gets the content of the response as a string
string responseContent = await responseMessage.Content.ReadAsStringAsync();
return responseContent;
}
// Generates or adds to the .json file that contains data on each clip
private static void GenerateJson(string responseContent)
{
// Parses the data from the response to the api request
Root responseResult = JsonConvert.DeserializeObject<Root>(responseContent);
// If the file doesn't exist, we need to create it and add a '[' at the start
if (!File.Exists(data.JsonFile))
{
FileStream file = File.Create(data.JsonFile);
file.Close();
// The array of json objects needs to be wrapped inside []
File.AppendAllText(data.JsonFile, "[\n");
}
else
{
// For a pre-existing .json file, The last object won't have a comma at the
// end of it so we need to add it now, before we add more objects
string[] jsonLines = File.ReadAllLines(data.JsonFile);
File.WriteAllLines(data.JsonFile, jsonLines.Take(jsonLines.Length - 1).ToArray());
File.AppendAllText(data.JsonFile, ",");
}
// If the file already exists, but there was no [ at the start for whatever reason,
// we need to add it
if (File.ReadAllText(data.JsonFile).Length == 0 || File.ReadAllText(data.JsonFile)[0] != '[')
{
File.WriteAllText(data.JsonFile, "[\n" + File.ReadAllText(data.JsonFile));
}
string json;
// Loops through each ClipInfo object that the api returned
for (int i = 0; i < responseResult.data.Count; i++)
{
// Serializes the ClipInfo object into a json style string
json = JsonConvert.SerializeObject(responseResult.data[i]);
// Adds the serialized contents of ClipInfo to the .json file
File.AppendAllText(data.JsonFile, json);
if (i != responseResult.data.Count - 1)
{
// All objects except the last require a comma at the end of the
// object in order to correctly format the array of json objects
File.AppendAllText(data.JsonFile, ",");
}
// Adds new line after object entry
File.AppendAllText(data.JsonFile, "\n");
}
// Adds the ] at the end of the file to close off the json objects array
File.AppendAllText(data.JsonFile, "]");
}
}
}
FileDownloader.cs
using Newtonsoft.Json;
using System;
using System.Collections.Generic;
using System.IO;
using System.Net;
using System.Threading.Tasks;
namespace Downloader
{
public class FileDownloader
{
private static Data data;
private static List<Task> tasks;
public async static void DownloadAllFiles(Data clientData)
{
data = clientData;
data.OutputFolderExists();
// Deserialize .json file and get ClipInfo list
List<ClipInfo> clips = JsonConvert.DeserializeObject<List<ClipInfo>>(File.ReadAllText(data.JsonFile));
tasks = new List<Task>();
foreach (ClipInfo clip in clips)
{
tasks.Add(DownloadFilesAsync(clip));
}
await Task.WhenAll(tasks);
}
private static void GetData()
{
if (data.ClientID == null)
{
data.GetClientID();
}
if (data.Authentication == null)
{
data.GetAuthentication();
}
if (data.QueryURL == null)
{
data.GetQuery();
}
}
private static string GetClipURL(ClipInfo clip)
{
// Example thumbnail URL:
// https://clips-media-assets2.twitch.tv/AT-cm%7C902106752-preview-480x272.jpg
// You can get the URL of the location of clip.mp4
// by removing the -preview.... from the thumbnail url */
string url = clip.thumbnail_url;
url = url.Substring(0, url.IndexOf("-preview")) + ".mp4";
return url;
}
private async static Task DownloadFilesAsync(ClipInfo clip)
{
WebClient client = new WebClient();
string url = GetClipURL(clip);
string filepath = data.OutputPath + clip.id + ".mp4";
await client.DownloadFileTaskAsync(new Uri(url), filepath);
}
private static void FileDownloadComplete(object sender, System.ComponentModel.AsyncCompletedEventArgs e)
{
tasks.Remove((Task)sender);
}
}
}
Program.cs
using System;
using System.Threading.Tasks;
using Downloader;
namespace ClipDownloader
{
class Program
{
private static string clientID = "{your_client_id}";
private static string authentication = "{your_authentication}";
async static Task Main(string[] args)
{
Console.WriteLine("Enter your output path");
string outputPath = Console.ReadLine();
Data data = new Data(clientID, authentication, outputPath);
Console.WriteLine(data.OutputPath);
//await JsonGenerator.Generate(data);
FileDownloader.DownloadAllFiles(data);
}
}
}
The example query I usually type in is "game_id=510218"
