@mastersthesis{Solstad2022,

title = {A comparison of manual and automated quality assessment of Open Educational Resources and their reliability},

author = {Hanna Eide Solstad},

url = {https://ntnuopen.ntnu.no/ntnu-xmlui/bitstream/handle/11250/3024682/no.ntnu%3Ainspera%3A112046434%3A23371129.pdf},

year  = {2022},

date = {2022-06-01},

urldate = {2023-03-03},

institution = {Faculty of Information Technology and Electrical Engineering, Department of Computer Science},

school = {Norwegian University of Science and Technology},

abstract = {The fourth Sustainable Development Goal is to: "Ensure inclusive and equitable quality education and promote lifelong learning opportunities for all". UNESCO considers Open Educational Resources (OER) vital in achieving this. OERs are educational material shared under an open license permitting free access, use, adaption, and redistribution under few restrictions. The OER movement has regained attention, with COVID-19 forcing millions to study from home. However, there are many challenges to continued growth. One of the most crucial challenges is quality control. Current approaches are mainly built on manual reviews, which are time-consuming and expensive.



This thesis proposes a white-box algorithm that combines theoretical quality knowledge with measurable metrics to give a quality score. The algorithm was developed for the educational resource type Interactive Videos created with the framework H5P. I performed a comparative study of the algorithm and the most adopted approach: manual reviews. 23 H5P users were recruited to perform 107 manual reviews of 57 OERs. The manual reviews scored different quality factors, two overall scores, and could add a comment for each resource. The data were then used to find the degree of agreement between the two methods and their reliability.



The result was low to moderate degree of agreement between the manual reviews and algorithm scores. That means that the algorithm can be a suitable approach in certain cases, but mostly as an addition to other methods. However, the most crucial finding was the low reliability of the manual reviews. The reviews were highly subjective and this has significant implications for this study and all research using reviews as a data source. Future studies need to continue to work on automated approaches but consider how they can be evaluated correctly},

howpublished = {Master’s thesis in Master of Technology in Computer Science},

keywords = {},

pubstate = {published},

tppubtype = {mastersthesis}

}