Python getting last line

Question

Working on a monitoring project, I need to count pulses from pulse meters. I've already found some solutions, which I've been trying to adapt to my needs.

Here is the python script, running on a Raspberry Pi :

#!/usr/bin/env python

import RPi.GPIO as GPIO
import datetime
import sys
import signal
from subprocess import check_output

#verbose = True     # global variable

############################################################################################################
############################################################################################################

def printusage(progname):
        print progname + ' <gpio-pin-number> <filename> [debug]'
        print 'Example usage: ' 
    print progname + ' 23 /path/to/mylogfile'
        print progname + ' 23 /path/to/mylogfile debug'
    sys.exit(-1)

def signal_handler(signal, frame):
        if verbose:
        print('You pressed Ctrl+C, so exiting')
    GPIO.cleanup()
        sys.exit(0)


def readvalue(myworkfile):
    try:
        f = open(myworkfile, 'ab+')     # open for reading. If it does not exist, create it
        line=subprocess.check_output(['tail','-f','-1',f])
        elmts=line.split(" ")
        value = int(elmts[2])   
    except:
        value = 0               # if something went wrong, reset to 0
    #print "old value is", value
    f.close()   # close for reading
    return value


def writevalue(myworkfile,value):
    f = open(myworkfile, 'a')
    f.write((str(datetime.datetime.now())+' '+str(value)+'\r\n'))   # timestamp
    f.close()   

############################################################################################################
############################################################################################################

######### Initialization


#### get input parameters:

try:
    mygpiopin = int(sys.argv[1])
    logfile = sys.argv[2]
except:
    printusage(sys.argv[0])

verbose = False
try:
    if sys.argv[3] == 'debug':
        verbose = True
        print "Verbose is On"
    else:
        printusage(sys.argv[0])
except:
    pass

#### if verbose, print some info to stdout

if verbose:
    print "GPIO is " + str(mygpiopin)
    print "Logfile is " + logfile
    print "Current value is " + str(readvalue(logfile))


#### setup

GPIO.setmode(GPIO.BCM)
GPIO.setup(mygpiopin, GPIO.IN, pull_up_down = GPIO.PUD_DOWN)

signal.signal(signal.SIGINT, signal_handler)    # SIGINT = interrupt by CTRL-C


########## Main Loop 

while True:
    # wait for pin going up
    GPIO.wait_for_edge(mygpiopin, GPIO.RISING)

    # read value from file
    counter=readvalue(logfile) + 1
    if verbose:
        print "New value is", counter

    # write value to file
    writevalue(logfile,counter)

    # and wait for pin going down
    GPIO.wait_for_edge(mygpiopin, GPIO.FALLING)

############################################################################################################
############################################################################################################

I want to get the last index registered, and increment it, but everything I've tested so far leaves the loop stuck on an index of 1.

I can't use a "heavier" method to find the last line, for instance browsing the entire file, because it will only get heavier and heavier as time passes, and I can't miss a pulse.

I'm pretty new to programming, so thanks for your help !

EDIT : the result file looks like this :

2016-10-08 16:54:23.072469 1
2016-10-08 16:54:23.462465 1
2016-10-08 16:54:23.777977 1
2016-10-08 16:54:24.010045 1
2016-10-08 16:54:24.194032 1
2016-10-08 16:54:24.388120 1
2016-10-08 16:54:24.549389 1
2016-10-08 16:54:24.737994 1
2016-10-08 16:54:24.959462 1
2016-10-08 16:54:25.164638 1
2016-10-08 16:54:25.351850 1
2016-10-08 16:54:25.536655 1
2016-10-08 16:54:25.716214 1
2016-10-08 16:54:25.794152 1
2016-10-08 17:06:13.506531 1
2016-10-08 17:06:14.097642 1
2016-10-08 17:06:14.211579 1
2016-10-08 17:06:15.237852 1
2016-10-08 17:06:15.752239 1
2016-10-08 17:06:16.320419 1
2016-10-08 17:06:16.842906 1
2016-10-08 17:06:17.391121 1
2016-10-08 17:06:17.851521 1
2016-10-08 17:06:18.444486 1
2016-10-08 17:06:18.858358 1

Why are you using a subprocess to read a file? Worse again why are you passing it a file object? — Padraic Cunningham, Oct 08 '16 at 16:08
You are mixing tabs and spaces. Probably not the cause of you current problems, but definitely a cause for your future problems. Don't do it. Use only spaces. Configure your editor to insert 4 spaces when you press tab. — zvone, Oct 08 '16 at 16:10
Also why are you `line.split(" ")` and getting the third element? All you are writing is `str(datetime.datetime.now())+' '+str(value)+'\r\n')` so there are at most two elements — Padraic Cunningham, Oct 08 '16 at 16:30

score 0 · Answer 1 · answered Oct 08 '16 at 16:21

To read the last line of a big file, pick a position near the end of the file and read. If there are no newlines there, go back a little and read again.

Then find the last newline and the rest is your last line.

EXPECTED_MAX_LINE_SIZE = 500  # this should be both small enough and big enough

def get_last_line(f):
    end = f.seek(0, os.SEEK_END)   # detect file size
    pos = end-EXPECTED_MAX_LINE_SIZE
    if pos < 0: pos = 0
    f.seek(pos, os.SEEK_SET)
    end_of_file = f.read(EXPECTED_MAX_LINE_SIZE)

    # TODO: check if '\n' is not there and read some more

    return end_of_file.splitlines()[-1]

Padraic Cunningham · Answer 2 · 2016-10-08T16:55:01.047

Your logic is way over complicated, if you will be running the script different times just store the last index in a separate file constantly overwriting:

 def next_index(index):
    with open(index, 'a+') as f:  # open for reading. If it does not exist, create it
        val = int(next(f, -1)) + 1 # first run value will be 0
    open(index,"w").write(str(val)) # overwrite
    return val

if the value is created elsewhere just do the writing there, just make sure you overwrite the previous value opening with w.

score 0 · Answer 3 · answered Oct 09 '16 at 09:10

I don't think that was what Padraic Cunningham meant, but I've managed to solve my problem by using two files, one for logging the updated index value, and one for storing the values :

#!/usr/bin/env python

import RPi.GPIO as GPIO
import datetime
import sys
import signal

#verbose = True     # global debug variable

############################################################################################################

def printusage(progname): #show how to use the script
    print progname + ' <gpio-pin-number> <index file> <store file> [debug]'
    print 'Example usage: ' 
    print progname + ' 23 /path/to/mylogfile /path/to/storefile'
    print progname + ' 23 /path/to/mylogfile /path/to/storefile debug'
    sys.exit(-1)

def signal_handler(signal, frame): #exiting the script
    if verbose:
        print('You pressed Ctrl+C, so exiting')
    GPIO.cleanup()
    sys.exit(0)


def readvalue(myworkfile):
    try:
        f = open(myworkfile, 'ab+')     # open for reading. If it does not exist, create it
        line=f.readline() #read the first and only line of the file
        elmts=line.split(" ") # the format is <yyyy-mm-dd hh:mm:ss.ssssss indexvalue>
        value = int(elmts[2]) #get the index value
    except:
        value = 0               # if something went wrong, reset to 0
    #print "old value is", value
    f.close()   # close for reading
    return value


def writevalue(myworkfile,value):
    f = open(myworkfile, 'w') #overwrite the value
    f.write((str(datetime.datetime.now())+' '+str(value)))  # timestamp + data
    f.close()   

def store_value(index_file,index_value):
    f=open(index_file, 'a+')
    f.write(str(datetime.datetime.now())+' '+str(index_value)+'\r\n') #store timestamp + data
    f.close()

############################################################################################################

######### Initialization
#### get input parameters 
try:
    mygpiopin = int(sys.argv[1])
    logfile = sys.argv[2]
    storefile=sys.argv[3]
except:
    printusage(sys.argv[0])

verbose = False
try:
    if sys.argv[4] == 'debug':
        verbose = True
        print "Verbose is On"
    else:
        printusage(sys.argv[0])
except:
    pass

if verbose: #### if verbose, print some info to stdout
    print "GPIO is " + str(mygpiopin)
    print "Logfile is " + logfile
    print "Storefile is " + storefile
    print "Current value is " + str(readvalue(logfile))

#### SETUP
GPIO.setmode(GPIO.BCM)
GPIO.setup(mygpiopin, GPIO.IN, pull_up_down = GPIO.PUD_DOWN)
signal.signal(signal.SIGINT, signal_handler)    # SIGINT = interrupt by CTRL-C

########## Main Loop #####

while True:
    GPIO.wait_for_edge(mygpiopin, GPIO.RISING) # wait for pin going up
    counter=readvalue(logfile) + 1 # read value from file and increment index
    if verbose:
        print "New value is", counter
    writevalue(logfile,counter) # write value to logfile
    store_value(storefile,counter)  # store value in storefile
    GPIO.wait_for_edge(mygpiopin, GPIO.FALLING) # and wait for pin going down
############################################################################################################

Thanks for your help !

*I don't think that was what Padraic Cunningham meant,*, that was exactly what I meant. **store the last index in a separate file** — Padraic Cunningham, Oct 09 '16 at 09:16

Python getting last line

3 Answers3