python find max value by mrjob

Question

i would like to find the max value in list by mrjob. when i run this, it always show the error:

No configs found; falling back on auto-configuration; No configs specified for inline runner

i'd like to know what's the meaning

class MRWordCounter(MRJob):

def mapper(self, key, line):
            num = csv_readline(line)
            yield num, 1
def reducer(self, word, compare):
            num_list = []
            for value in compare:
                    if value == max(compare):
                            value=num_list
                            yield word, num_list

score 2 · Answer 1 · answered Aug 08 '19 at 14:38

You can use this method instead:-

#The most occurred word
#Import Dependencies
from mrjob.job import MRJob
from mrjob.step import MRStep
import re

WORD_RE = re.compile(r"[\w']+")


class MRMostUsedWord(MRJob):

    def mapper_get_words(self, _, line):
        # yield each word in the line
        for word in WORD_RE.findall(line):
            yield (word.lower(), 1)

    def combiner_count_words(self, word, counts):
        # sum the words we've seen so far
        yield (word, sum(counts))

    def reducer_count_words(self, word, counts):
        # send all (num_occurrences, word) pairs to the same reducer.
        # num_occurrences is so we can easily use Python's max() function.
        yield None, (sum(counts), word)

    # discard the key; it is just None
    def reducer_find_max_word(self, _, word_count_pairs):
        # each item of word_count_pairs is (count, word),
        # so yielding one results in key=counts, value=word
        yield max(word_count_pairs)

    def steps(self):
        return [
            MRStep(mapper=self.mapper_get_words,
                   combiner=self.combiner_count_words,
                   reducer=self.reducer_count_words),
            MRStep(reducer=self.reducer_find_max_word)
        ]


if __name__ == '__main__':
    MRMostUsedWord.run()

What it simply does is:-

map the words.
combine the count for each word.
flip the key,value pair.
reduce to find the max occurred word.
To run the code,

save the text file and the python script in the same folder, and then:

python3 xyz.py xyz.txt

python find max value by mrjob

1 Answers1