0

I have the following pipeline:

        var mlContext = new MLContext();

        var data = mlContext.Data.LoadFromEnumerable(new[]
        {
            new Input {Message = "one two three one two three"},
            new Input {Message = "one two"},
            new Input {Message = "two three"}
        });

        var pipeline =
            mlContext.Transforms.Text.FeaturizeText("TextFeatures", "Message")
                .Append(mlContext.Transforms.Text.TokenizeCharacters(
                    "MessageTokens", "Message"))
                .Append(mlContext.Transforms.Text.ProduceNgrams(
                    "MessageNgrams", "MessageTokens", 2));

        var transformedData = pipeline.Fit(data).Transform(data);

"MessageNgrams" column after pipe execution contains float vectors. How can I get actual Ngrams i.e. "one two", "two tree"?

Aminion
  • 465
  • 1
  • 5
  • 13

2 Answers2

1
List<string> getNgram(string str, int nsize)
{
    var mlContext = new MLContext();
    var dataview = mlContext.Data.LoadFromEnumerable(new List<TextData>() { new TextData { Text = str } });

    var textPipeline = mlContext.Transforms.Text.TokenizeIntoWords("Tokens", "Text")
        .Append(mlContext.Transforms.Conversion.MapValueToKey("Tokens"))
        .Append(mlContext.Transforms.Text.ProduceNgrams("NgramFeatures", "Tokens",
            ngramLength: nsize,
            useAllLengths: false,
            weighting: NgramExtractingEstimator.WeightingCriteria.Tf));

    var textTransformer = textPipeline.Fit(dataview);
    var transformedDataView = textTransformer.Transform(dataview);

    VBuffer<ReadOnlyMemory<char>> slotNames = default;
    transformedDataView.Schema["NgramFeatures"].GetSlotNames(ref slotNames);
    var NgramFeaturesColumn = transformedDataView.GetColumn<VBuffer<float>>(transformedDataView.Schema["NgramFeatures"]);
    var ngrams = slotNames.GetValues().ToArray().Select(x=>x.Span.ToString().Replace('|',' '));

    return ngrams.ToList();
}
ARZ
  • 2,461
  • 3
  • 34
  • 56
0

Here is a code snippet which works for my SCDA, should be very similar or give you hints on how to achieve it.

var slotLabelBuffer = default(VBuffer<ReadOnlyMemory<char>>);
transformedData.Schema["MessageNgrams"].GetSlotNames(ref slotLabelBuffer);

var slotLabels = new Dictionary<int, string>();
for (int i = 0; i < slotLabelBuffer.Length; i++)
   slotLabels.Add(i, slotLabelBuffer.GetItemOrDefault(i).ToString());

Index of slotLabels corresponds to your float vector.

Kale
  • 1