
# Script block to identify host, user, and kernel
import sys
! hostname; ! whoami; ! pwd; 
print(sys.executable)

atomickitty
sensei
/home/sensei/engr-1330-webroot/1-Lessons/Lesson07
/opt/jupyterhub/bin/python3


%%html
<!-- Script Block to set tables to left alignment -->
<style>
  table {margin-left: 0 !important;}
</style>


import sys
! rm -rf myfirstfile.txt # delete file if it exists
! pwd # list name of working directory, note it includes path, so it is an absolute path

/home/sensei/engr-1330-webroot/1-Lessons/Lesson07


! ls -l # list contents of working directory

total 548
-rw-rw-r-- 1 sensei sensei 122198 Sep 10 20:57 ENGR-1330-Lesson07.ipynb
-rw-rw-r-- 1 sensei sensei 343800 Sep 10 20:31 Filesystem-graphic.png
-rw-rw-r-- 1 sensei sensei  88568 Sep 10 20:31 Filesystem-shell.png
drwxr-xr-x 3 sensei sensei   4096 Jul 21 16:15 OriginalPowerpoint


# create file example
externalfile = open("myfirstfile.txt",'w') # create connection to file, set to write (w), file does not need to exist
mymessage = 'message in a bottle' #some object to write, in this case a string
externalfile.write(mymessage)# write the contents of mymessage to the file
externalfile.close() # close the file connection


! ls -l # list contents of working directory

total 552
-rw-rw-r-- 1 sensei sensei 122198 Sep 10 20:57 ENGR-1330-Lesson07.ipynb
-rw-rw-r-- 1 sensei sensei 343800 Sep 10 20:31 Filesystem-graphic.png
-rw-rw-r-- 1 sensei sensei  88568 Sep 10 20:31 Filesystem-shell.png
drwxr-xr-x 3 sensei sensei   4096 Jul 21 16:15 OriginalPowerpoint
-rw-rw-r-- 1 sensei sensei     19 Sep 10 20:58 myfirstfile.txt


! cat myfirstfile.txt

message in a bottle


# read file example
externalfile = open("myfirstfile.txt",'r') # create connection to file, set to read (r), file must exist
silly_string = externalfile.read() # read the contents
externalfile.close() # close the file connection
print(silly_string)

message in a bottle


# write without a connection -- observe the error message
mymessage = 'message in a bottle' #some object to write, in this case a string
externalfile.write(mymessage)# write the contents of mymessage to the file

---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-9-81084545fc2e> in <module>
      1 # write without a connection
      2 mymessage = 'message in a bottle' #some object to write, in this case a string
----> 3 externalfile.write(mymessage)# write the contents of mymessage to the file

ValueError: I/O operation on closed file.


# read without a connection -- observe the error message
silly_string = externalfile.read() # read the contents
print(silly_string)

---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-10-38ae4e51ad51> in <module>
      1 # read without a connection -- observe the error message
----> 2 silly_string = externalfile.read() # read the contents
      3 print(silly_string)

ValueError: I/O operation on closed file.


externalfile = open("myfirstfile.txt",'a') # create connection to file, set to append (a), file does not need to exist
externalfile.write('\n') # adds a newline character
what_to_add = 'I love rock-and-roll, put another dime in the jukebox baby ... \n' 
externalfile.write(what_to_add) # add a string including the linefeed
what_to_add = '... the waiting is the hardest part \n' 
externalfile.write(what_to_add) # add a string including the linefeed
mylist = [1,2,3,4,5] # a list of numbers
what_to_add = ','.join(map(repr, mylist)) + "\n" # one way to write the list
externalfile.write(what_to_add)
what_to_add = ','.join(map(repr, mylist[0:len(mylist)])) + "\n" # another way to write the list
externalfile.write(what_to_add)
externalfile.close()


externalfile = open("myfirstfile.txt",'a') # create connection to file, set to append (a), file does not need to exist
externalfile.write('\n') # adds a newline character
what_to_add = 'I love rock-and-roll, put another dime in the jukebox baby ... \n' 
externalfile.write(what_to_add) # add a string including the linefeed
externalfile.close()


! cat myfirstfile.txt

message in a bottle
I love rock-and-roll, put another dime in the jukebox baby ... 
... the waiting is the hardest part 
1,2,3,4,5
1,2,3,4,5

I love rock-and-roll, put another dime in the jukebox baby ...


import os
file2kill = "myfirstfile.txt"
try:
    os.remove(file2kill) # file must exist or will generate an exception
except:
    pass # example of using pass to improve readability
print(file2kill, " missing or deleted !")

myfirstfile.txt  missing or deleted !


# create the "My Favorite Quotation" file:
externalfile = open("MyFavoriteQuotation.txt",'w')         # create connection to file, set to write (w)
myquotation = 'The path of the righteous man is beset on all sides by the inequities of the selfish and the tyranny of evil men. Blessed is he who, in the name of charity and good will, shepherds the weak through the valley of darkness. For he is truly his brother’s keeper and the finder of lost children. And I will strike down upon thee with great vengeance and furious anger those who attempt to poison and destroy my brothers. And you will know my name is the Lord when I lay my vengeance upon you.' #My choice: quotation from Pulp Fiction
externalfile.write(myquotation)# write the contents of mymessage to the file
externalfile.close() # close the file connection
#Let's read the file
! cat MyFavoriteQuotation.txt 
# Let's add the string
externalfile = open("MyFavoriteQuotation.txt",'a')  #create connection to file, set to append (a)
externalfile.write('\n') # adds a newline character
what_to_add = "And that's something I wish I had said ... \n"
externalfile.write(what_to_add)
externalfile.close()
#Let's read the file one last time
! cat MyFavoriteQuotation.txt

The path of the righteous man is beset on all sides by the inequities of the selfish and the tyranny of evil men. Blessed is he who, in the name of charity and good will, shepherds the weak through the valley of darkness. For he is truly his brother’s keeper and the finder of lost children. And I will strike down upon thee with great vengeance and furious anger those who attempt to poison and destroy my brothers. And you will know my name is the Lord when I lay my vengeance upon you.The path of the righteous man is beset on all sides by the inequities of the selfish and the tyranny of evil men. Blessed is he who, in the name of charity and good will, shepherds the weak through the valley of darkness. For he is truly his brother’s keeper and the finder of lost children. And I will strike down upon thee with great vengeance and furious anger those who attempt to poison and destroy my brothers. And you will know my name is the Lord when I lay my vengeance upon you.
And that's something I wish I had said ...


# Lets look at the files using shell commands (may need to adapt for windoze)
! echo '--- A.txt ---'
! cat A.txt
! echo
! echo '--- x.txt ---'
! cat x.txt
! echo
! echo '--- B.txt ---'
! cat B.txt

--- A.txt ---
4.0  1.5  0.7  1.2  0.5
1.0  6.0  0.9  1.4  0.7
0.5  1.0  3.9  3.2  0.9
0.2  2.0  0.2  7.5  1.9
1.7  0.9  1.2  2.3  4.9

--- x.txt ---
0.59519
0.50793
0.83171
0.63037
1.03738

--- B.txt ---
5.0
6.0
7.0
8.0
9.0


# Code to read A, X, and b - Notice we need somewhere for the data to go, hence the null lists
amatrix = [] # null list to store matrix read
xvector = [] # null list to store vector read
bvector = [] # null list to store vector read
rowNumA = 0
colNumA = 0
rowNumB = 0
rowNumX = 0


localfile = open("A.txt","r") # connect and read file for MATRIX A
for line in localfile:
    amatrix.append([float(n) for n in line.strip().split()])
    rowNumA += 1
localfile.close() # Disconnect the file
colNumA = len(amatrix[0]) # get the column count


print('A matrix')
for i in range(0,rowNumA,1):
    print ( (amatrix[i][0:colNumA]))

A matrix
[4.0, 1.5, 0.7, 1.2, 0.5]
[1.0, 6.0, 0.9, 1.4, 0.7]
[0.5, 1.0, 3.9, 3.2, 0.9]
[0.2, 2.0, 0.2, 7.5, 1.9]
[1.7, 0.9, 1.2, 2.3, 4.9]


localfile = open("x.txt","r") # connect and read file for VECTOR x
for line in localfile:
    xvector.append(float(line))  # vector read different -- just float the line
    rowNumX += 1
localfile.close() # Disconnect the file


print('x vector')
for i in range(0,rowNumX,1):
    print ( (xvector[i]))

x vector
0.59519
0.50793
0.83171
0.63037
1.03738


localfile = open("B.txt","r") # connect and read file for VECTOR B
for line in localfile:
    bvector.append(float(line))  # vector read different -- just float the line
    rowNumB += 1
localfile.close() # Disconnect the file


print('B vector')
for i in range(0,rowNumB,1):
    print ( (bvector[i]))

B vector
5.0
6.0
7.0
8.0
9.0


rhs = [0 for i in range(rowNumX)] # here we will store Ax = rhs
for i in range(0,rowNumA): # select row
    for j in range(0,colNumA): # dot product current row*xvector
            rhs[i]=rhs[i]+amatrix[i][j]*xvector[j]
for i in range(0,rowNumA,1): # print out the result
    print (rhs[i])

4.999986
5.999993
7.00002
8.000037
9.000025


tolerance = 1.0e-04 # decide that 1 part in 10,000 is enough
same = True # here is our flag, as we compare element by element if we find one that is not close enough we quit and declare we dont have an answer
for i in range(0,rowNumA,1): # just march through the lists
    if abs(rhs[i]-bvector[i]) > tolerance: # too far apart
        same = False
        break # we can exit the loop, 
    else:
        continue # keep checking
if same == True:
    print('The two vectors are the same, so x solves Ax=B')
else:
    print('The two vectors are different, so x does not solve Ax=B')  
    
print('---Ax---','---B---')
for i in range(0,rowNumA,1):
    print (' ',round(rhs[i],3),'    ',round(bvector[i],3))

The two vectors are the same, so x solves Ax=B
---Ax--- ---B---
  5.0      5.0
  6.0      6.0
  7.0      7.0
  8.0      8.0
  9.0      9.0


import requests # Module to process http/https requests


import sys # Module to process commands to/from the OS using a shell-type syntax
! rm -rf all_quads_gross_evaporation.csv # delete file if it exists


remote_url="http://54.243.252.9/engr-1330-webroot/4-Databases/all_quads_gross_evaporation.csv"  # set the url
rget = requests.get(remote_url, allow_redirects=True)  # get the remote resource, follow imbedded links
localfile = open('all_quads_gross_evaporation.csv','wb') # open connection to a local file same name as remote
localfile.write(rget.content) # extract from the remote the contents,insert into the local file same name
localfile.close() # close connection to the local file


# print(type(localfile)) # verify object is an I/O object

<class '_io.BufferedWriter'>


# verify file exists
! pwd # list absolute path to script directory
! ls -lah # list directory contents, owner, file sizes ...

/home/sensei/engr-1330-webroot/1-Lessons/Lesson07
total 924K
drwxr-xr-x  4 sensei sensei 4.0K Sep 12 17:20 .
drwxr-xr-x 28 sensei sensei 4.0K Aug 23 16:10 ..
drwxrwxr-x  2 sensei sensei 4.0K Sep 10 21:43 .ipynb_checkpoints
-rw-rw-r--  1 sensei sensei 121K Sep 12 17:20 ENGR-1330-Lesson07.ipynb
-rw-rw-r--  1 sensei sensei 336K Sep 10 20:31 Filesystem-graphic.png
-rw-rw-r--  1 sensei sensei  87K Sep 10 20:31 Filesystem-shell.png
drwxr-xr-x  3 sensei sensei 4.0K Jul 21 16:15 OriginalPowerpoint
-rw-rw-r--  1 sensei sensei 355K Sep 12 17:18 all_quads_gross_evaporation.csv
-rw-rw-r--  1 sensei sensei  206 Sep 10 21:08 myfirstfile.txt


! less all_quads_gross_evaporation.csv

:,104,105,106,107,108,204,205,206,207,208,304,305,306,307,308,309,404,405,406,407,408,409,410,411,412,413,414,504,505,506,507,508,509,510,511,512,513,514,601,602,603,604,605,606,607,608,609,610,611,612,613,614,701,702,703,704,705,706,707,708,709,710,711,712,713,714,803,804,805,806,807,808,809,810,811,812,813,814,907,908,909,910,911,912,1008,1009,1010,1011,1108,1109,1110,1210^M1954-01,1.8,1.8,2.02,2.24,2.24,2.34,1.89,1.8,1.99,2.02,2.67,2.46,2.11,1.83,1.59,1.17,2.09,2.5,2.22,1.83,1.77,1.62,1.23,1.23,1.27,1.27,1.27,2.98,2.8,2.36,2.16,1.96,1.63,1.52,1.52,1.41,1.38,1.33,2.42,2.54,3.01,2.96,2.81,2.57,2.49,2.22,1.72,1.73,1.66,1.6,1.45,1.45,2.42,2.43,2.45,2.54,2.46,2.29,2.52,2.17,1.78,2.19,2.08,1.87,1.37,1.39,2.45,2.25,2.05,2.3,2.41,2.02,1.94,2.45,1.85,1.53,1.27,1.26,1.93,1.9,2.37,1.91,1.42,1.3,2.5,2.42,1.94,1.29,2.59,2.49,2.22,2.27^M1954-02,4.27,4.27,4.13,3.98,3.9,4.18,4.26,4.27,4.26,4.18,4.1,3.98,3.8,3.9,4.69,3.81,3.83,3.48,3.34,3.24,4.16,3.68,3.13,4.22,4.21,2.53,2.39,4.39,4.01,3.99,4.52,5.01,4.69,4.47,4.34,4.24,3.43,2.78,3.81,3.92,4.47,4.57,5.03,5.42,5.54,5.12,3.81,3.97,4.09,3.71,3.52,3.52,3.81,3.96,4.32,4.52,4.58,4.2,4.28,3.63,3.02,3.66,3.56,3.45,3.33,3.38,4.32,4.16,4,4.01,3.87,3.73,3.64,3.73,3.17,2.76,2.86,3.07,3.25,3.61,3.83,3.53,2.59,2.51,4.71,4.3,3.84,2.5,5.07,4.62,4.05,4.18^M1954-03,4.98,4.98,4.62,4.25,4.2,5.01,4.98,4.98,4.68,4.77,5.45,4.93,4.75,4.36,4.91,3.92,5.2,4.7,4.33,3.83,4.73,4.3,4.56,4.94,4.82,3.07,2.93,6.31,5.58,4.82,5,5.77,5.46,5.36,5.02,4.96,3.92,3.24,6,6.08,6.26,6.28,6.37,5.68,5.47,5.83,4.59,4.35,4.54,4.15,3.83,3.83,6,5.87,5.54,5.77,5.7,5.17,5.39,4.7,3.96,4.05,4.02,4,3.81,3.83,5.54,5.21,4.88,5.85,5.94,4.87,4.42,4.1,3.7,3.36,3.26,3.49,5,4.81,4.78,4.32,3.21,3.21,6.21,6.06,5.02,3.21,6.32,6.2,5.68,5.7^M1954-04,6.09,5.94,5.94,6.07,5.27,6.31,5.98,5.89,5.72,5.22,6.84,6.13,5.19,4.82,5.86,4.95,5.82,all_quads_gross_evaporation.csv


# now lets process the file
localfile = open('all_quads_gross_evaporation.csv','r') # open a connection for reading
aList = [] # null list to store read
rowNumA = 0 # counter to keep track of rows, 
for line in localfile:
    #aList.append([str(n) for n in line.strip().split()])
    aList.append([str(n) for n in line.strip().split(",")]) # process each line, strip whitespace, split on ","
    rowNumA += 1 # increment the counter
localfile.close() #close the connection - amatrix contains the file contents
# print((aList[0])) # print 1st row
for irow in range(0,10):
    print([aList[irow][jcol] for jcol in range(0,10)])  # implied loop constructor syntax

['YYYY-MM', '104', '105', '106', '107', '108', '204', '205', '206', '207']
['1954-01', '1.8', '1.8', '2.02', '2.24', '2.24', '2.34', '1.89', '1.8', '1.99']
['1954-02', '4.27', '4.27', '4.13', '3.98', '3.9', '4.18', '4.26', '4.27', '4.26']
['1954-03', '4.98', '4.98', '4.62', '4.25', '4.2', '5.01', '4.98', '4.98', '4.68']
['1954-04', '6.09', '5.94', '5.94', '6.07', '5.27', '6.31', '5.98', '5.89', '5.72']
['1954-05', '5.41', '5.09', '5.14', '4.4', '3.61', '5.57', '4.56', '4.47', '4.18']
['1954-06', '9.56', '11.75', '12.1', '9.77', '8.06', '9.47', '8.42', '8.66', '8.78']
['1954-07', '8.65', '11.12', '11.33', '11.12', '10.09', '9.44', '9.32', '9.42', '10.14']
['1954-08', '5.81', '7.68', '9.97', '11.34', '9.76', '7.15', '8.56', '8.59', '9.43']
['1954-09', '7.42', '10.41', '10.64', '8.68', '7.67', '7.39', '8.31', '8.65', '8.42']


flag = True
c910 = 0
while flag:
    try:
        if aList[0][c910] == '910': # test if header is 910
            flag = False # switch flag to exit loop
        else :
            c910 += 1 # increment counter if not right header
    except:
        print('No column position found, resetting to 0')
        c910 = 0
        break
    
if c910 != 0:
    for irow in range(0,10): # activate to show first few rows
#    for irow in range(0,rowNumA): # activate to print entire list
        print(aList[irow][0],aList[irow][c910])  # implied loop constructor syntax

YYYY-MM 910
1954-01 1.91
1954-02 3.53
1954-03 4.32
1954-04 4.51
1954-05 4.25
1954-06 6.85
1954-07 7.99
1954-08 7.88
1954-09 6.55

ENGR 1330 Computational Thinking with Data Science¶

Lesson 7 Files:¶

Objectives¶

Files and Filesystems¶

File system¶

Path¶

File Types¶

File Manipulation¶

Example: Examine our local directory¶

Example: Create a file, write to it.¶

Example: Read from an existing file.¶

Example: Update a file.¶

Example: Delete a file¶

Example¶

Reading data from a file.¶

Downloading files from websites (optional)¶

A Method to get the actual file from a remote web server (unencrypted)¶

A Method to get the actual file from a remote web server (SSL/TLS encrypted)¶

References¶